From eb2c21c074209c85953e16b29b1135018783d07b Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Thu, 8 Oct 2026 16:17:08 +0000 Subject: [PATCH] igneum-pow, proto-cuda: counter-asic-4's research code comes off master (the revert of 31496f4e's code paths; documents untouched), the coordinator's order of 8 October 2026, 17:32 BST Master took the direct merge of counter-asic-4 7e5897dc at 16:12 BST (31496f4e) and with it the branch's research changes to igneum-pow/src (the +mm tile and +shl experimental classes, mm8.rs, the AP-F4-1 constants in their old form), igneum-pow/tests (ca4_trace.rs, v6_window_bits.rs, scratch.rs) and proto-cuda (the nvrtc worker's --microbench and --bench --variant paths, the emu stubs, packs-ca4), 817 lines against class v5's post-freeze igneum-pow. Every research landing runs under the documents-only rule (no igneum-pow/src change from a research branch lands until a class v6 cut names it), and through any master merge this code would reach the class v5 hash object (the 0.3.26 pairing line). This commit reverse-applies 31496f4e's diff on those paths over the live master, a clean three-way apply; what remains on them against 31496f4e's first parent is tn-land's own work (79b1fba9: blake2b.rs, state.rs, memhard.rs, emit.rs, verify.rs, main.rs, lib.rs, the derive, mixer and packs tests), untouched. The code stays on counter-asic-4 for the research lane. The documents 31496f4e and the later landings carried stay as they are. --- igneum-pow/src/accept.rs | 168 +- igneum-pow/src/emit.rs | 146 +- igneum-pow/src/generator.rs | 210 +- igneum-pow/src/lib.rs | 1 - igneum-pow/src/mm8.rs | 179 - igneum-pow/src/verify.rs | 80 +- igneum-pow/tests/ca4_trace.rs | 223 -- igneum-pow/tests/scratch.rs | 1 - igneum-pow/tests/v6_window_bits.rs | 46 - proto-cuda/nvrtc/cuda_api.h | 4 - proto-cuda/nvrtc/emu/list-race-stubs.cpp | 5 - proto-cuda/nvrtc/emu/variant-test.cpp | 51 - proto-cuda/nvrtc/worker.cpp | 418 +-- proto-cuda/packs-ca4/mx8_mm128/kernel.cl | 415 --- proto-cuda/packs-ca4/mx8_mm128/kernel.cu | 293 -- .../packs-ca4/mx8_mm128/kernel_bound.cl | 639 ---- .../packs-ca4/mx8_mm128/kernel_bound.cu | 252 -- proto-cuda/packs-ca4/mx8_mm128/memhard.h | 109 - proto-cuda/packs-ca4/mx8_mm128/memhard.metal | 107 - proto-cuda/packs-ca4/mx8_mm128/program.h | 73 - proto-cuda/packs-ca4/mx8_mm128/program.json | 134 - proto-cuda/packs-ca4/mx8_mm128/program.metal | 257 -- .../packs-ca4/mx8_mm128/program_bound.metal | 259 -- proto-cuda/packs-ca4/mx8_mm128/vectors.h | 57 - proto-cuda/packs-ca4/mx8_mm128/vectors.json | 36 - proto-cuda/packs-ca4/mx8_mm1430/kernel.cl | 1717 --------- proto-cuda/packs-ca4/mx8_mm1430/kernel.cu | 1595 -------- .../packs-ca4/mx8_mm1430/kernel_bound.cl | 3243 ----------------- .../packs-ca4/mx8_mm1430/kernel_bound.cu | 1554 -------- proto-cuda/packs-ca4/mx8_mm1430/memhard.h | 109 - proto-cuda/packs-ca4/mx8_mm1430/memhard.metal | 107 - proto-cuda/packs-ca4/mx8_mm1430/program.h | 73 - proto-cuda/packs-ca4/mx8_mm1430/program.json | 134 - proto-cuda/packs-ca4/mx8_mm1430/program.metal | 1559 -------- .../packs-ca4/mx8_mm1430/program_bound.metal | 1561 -------- proto-cuda/packs-ca4/mx8_mm1430/vectors.h | 57 - proto-cuda/packs-ca4/mx8_mm1430/vectors.json | 36 - proto-cuda/packs-ca4/mx8_mm512/kernel.cl | 799 ---- proto-cuda/packs-ca4/mx8_mm512/kernel.cu | 677 ---- .../packs-ca4/mx8_mm512/kernel_bound.cl | 1407 ------- .../packs-ca4/mx8_mm512/kernel_bound.cu | 636 ---- proto-cuda/packs-ca4/mx8_mm512/memhard.h | 109 - proto-cuda/packs-ca4/mx8_mm512/memhard.metal | 107 - proto-cuda/packs-ca4/mx8_mm512/program.h | 73 - proto-cuda/packs-ca4/mx8_mm512/program.json | 134 - proto-cuda/packs-ca4/mx8_mm512/program.metal | 641 ---- .../packs-ca4/mx8_mm512/program_bound.metal | 643 ---- proto-cuda/packs-ca4/mx8_mm512/vectors.h | 57 - proto-cuda/packs-ca4/mx8_mm512/vectors.json | 36 - proto-cuda/packs-ca4/mx8_sh256x27/kernel.cl | 538 --- proto-cuda/packs-ca4/mx8_sh256x27/kernel.cu | 423 --- .../packs-ca4/mx8_sh256x27/kernel_bound.cl | 891 ----- .../packs-ca4/mx8_sh256x27/kernel_bound.cu | 382 -- proto-cuda/packs-ca4/mx8_sh256x27/memhard.h | 109 - .../packs-ca4/mx8_sh256x27/memhard.metal | 107 - proto-cuda/packs-ca4/mx8_sh256x27/program.h | 70 - .../packs-ca4/mx8_sh256x27/program.json | 389 -- .../packs-ca4/mx8_sh256x27/program.metal | 368 -- .../mx8_sh256x27/program_bound.metal | 370 -- proto-cuda/packs-ca4/mx8_sh256x27/vectors.h | 57 - .../packs-ca4/mx8_sh256x27/vectors.json | 36 - proto-cuda/packs-ca4/mx8_shl256x27/kernel.cl | 583 --- proto-cuda/packs-ca4/mx8_shl256x27/kernel.cu | 468 --- .../packs-ca4/mx8_shl256x27/kernel_bound.cl | 981 ----- .../packs-ca4/mx8_shl256x27/kernel_bound.cu | 427 --- proto-cuda/packs-ca4/mx8_shl256x27/memhard.h | 109 - .../packs-ca4/mx8_shl256x27/memhard.metal | 107 - proto-cuda/packs-ca4/mx8_shl256x27/program.h | 72 - .../packs-ca4/mx8_shl256x27/program.json | 390 -- .../packs-ca4/mx8_shl256x27/program.metal | 413 --- .../mx8_shl256x27/program_bound.metal | 415 --- proto-cuda/packs-ca4/mx8_shl256x27/vectors.h | 57 - .../packs-ca4/mx8_shl256x27/vectors.json | 36 - .../packs-ca4/mx8_shl256x27_v2/kernel.cl | 583 --- .../packs-ca4/mx8_shl256x27_v2/kernel.cu | 468 --- .../mx8_shl256x27_v2/kernel_bound.cl | 981 ----- .../mx8_shl256x27_v2/kernel_bound.cu | 427 --- .../packs-ca4/mx8_shl256x27_v2/memhard.h | 109 - .../packs-ca4/mx8_shl256x27_v2/memhard.metal | 107 - .../packs-ca4/mx8_shl256x27_v2/program.h | 72 - .../packs-ca4/mx8_shl256x27_v2/program.json | 390 -- .../packs-ca4/mx8_shl256x27_v2/program.metal | 413 --- .../mx8_shl256x27_v2/program_bound.metal | 415 --- .../packs-ca4/mx8_shl256x27_v2/vectors.h | 57 - .../packs-ca4/mx8_shl256x27_v2/vectors.json | 36 - 85 files changed, 43 insertions(+), 33040 deletions(-) delete mode 100644 igneum-pow/src/mm8.rs delete mode 100644 igneum-pow/tests/ca4_trace.rs delete mode 100644 igneum-pow/tests/v6_window_bits.rs delete mode 100644 proto-cuda/nvrtc/emu/list-race-stubs.cpp delete mode 100644 proto-cuda/nvrtc/emu/variant-test.cpp delete mode 100644 proto-cuda/packs-ca4/mx8_mm128/kernel.cl delete mode 100644 proto-cuda/packs-ca4/mx8_mm128/kernel.cu delete mode 100644 proto-cuda/packs-ca4/mx8_mm128/kernel_bound.cl delete mode 100644 proto-cuda/packs-ca4/mx8_mm128/kernel_bound.cu delete mode 100644 proto-cuda/packs-ca4/mx8_mm128/memhard.h delete mode 100644 proto-cuda/packs-ca4/mx8_mm128/memhard.metal delete mode 100644 proto-cuda/packs-ca4/mx8_mm128/program.h delete mode 100644 proto-cuda/packs-ca4/mx8_mm128/program.json delete mode 100644 proto-cuda/packs-ca4/mx8_mm128/program.metal delete mode 100644 proto-cuda/packs-ca4/mx8_mm128/program_bound.metal delete mode 100644 proto-cuda/packs-ca4/mx8_mm128/vectors.h delete mode 100644 proto-cuda/packs-ca4/mx8_mm128/vectors.json delete mode 100644 proto-cuda/packs-ca4/mx8_mm1430/kernel.cl delete mode 100644 proto-cuda/packs-ca4/mx8_mm1430/kernel.cu delete mode 100644 proto-cuda/packs-ca4/mx8_mm1430/kernel_bound.cl delete mode 100644 proto-cuda/packs-ca4/mx8_mm1430/kernel_bound.cu delete mode 100644 proto-cuda/packs-ca4/mx8_mm1430/memhard.h delete mode 100644 proto-cuda/packs-ca4/mx8_mm1430/memhard.metal delete mode 100644 proto-cuda/packs-ca4/mx8_mm1430/program.h delete mode 100644 proto-cuda/packs-ca4/mx8_mm1430/program.json delete mode 100644 proto-cuda/packs-ca4/mx8_mm1430/program.metal delete mode 100644 proto-cuda/packs-ca4/mx8_mm1430/program_bound.metal delete mode 100644 proto-cuda/packs-ca4/mx8_mm1430/vectors.h delete mode 100644 proto-cuda/packs-ca4/mx8_mm1430/vectors.json delete mode 100644 proto-cuda/packs-ca4/mx8_mm512/kernel.cl delete mode 100644 proto-cuda/packs-ca4/mx8_mm512/kernel.cu delete mode 100644 proto-cuda/packs-ca4/mx8_mm512/kernel_bound.cl delete mode 100644 proto-cuda/packs-ca4/mx8_mm512/kernel_bound.cu delete mode 100644 proto-cuda/packs-ca4/mx8_mm512/memhard.h delete mode 100644 proto-cuda/packs-ca4/mx8_mm512/memhard.metal delete mode 100644 proto-cuda/packs-ca4/mx8_mm512/program.h delete mode 100644 proto-cuda/packs-ca4/mx8_mm512/program.json delete mode 100644 proto-cuda/packs-ca4/mx8_mm512/program.metal delete mode 100644 proto-cuda/packs-ca4/mx8_mm512/program_bound.metal delete mode 100644 proto-cuda/packs-ca4/mx8_mm512/vectors.h delete mode 100644 proto-cuda/packs-ca4/mx8_mm512/vectors.json delete mode 100644 proto-cuda/packs-ca4/mx8_sh256x27/kernel.cl delete mode 100644 proto-cuda/packs-ca4/mx8_sh256x27/kernel.cu delete mode 100644 proto-cuda/packs-ca4/mx8_sh256x27/kernel_bound.cl delete mode 100644 proto-cuda/packs-ca4/mx8_sh256x27/kernel_bound.cu delete mode 100644 proto-cuda/packs-ca4/mx8_sh256x27/memhard.h delete mode 100644 proto-cuda/packs-ca4/mx8_sh256x27/memhard.metal delete mode 100644 proto-cuda/packs-ca4/mx8_sh256x27/program.h delete mode 100644 proto-cuda/packs-ca4/mx8_sh256x27/program.json delete mode 100644 proto-cuda/packs-ca4/mx8_sh256x27/program.metal delete mode 100644 proto-cuda/packs-ca4/mx8_sh256x27/program_bound.metal delete mode 100644 proto-cuda/packs-ca4/mx8_sh256x27/vectors.h delete mode 100644 proto-cuda/packs-ca4/mx8_sh256x27/vectors.json delete mode 100644 proto-cuda/packs-ca4/mx8_shl256x27/kernel.cl delete mode 100644 proto-cuda/packs-ca4/mx8_shl256x27/kernel.cu delete mode 100644 proto-cuda/packs-ca4/mx8_shl256x27/kernel_bound.cl delete mode 100644 proto-cuda/packs-ca4/mx8_shl256x27/kernel_bound.cu delete mode 100644 proto-cuda/packs-ca4/mx8_shl256x27/memhard.h delete mode 100644 proto-cuda/packs-ca4/mx8_shl256x27/memhard.metal delete mode 100644 proto-cuda/packs-ca4/mx8_shl256x27/program.h delete mode 100644 proto-cuda/packs-ca4/mx8_shl256x27/program.json delete mode 100644 proto-cuda/packs-ca4/mx8_shl256x27/program.metal delete mode 100644 proto-cuda/packs-ca4/mx8_shl256x27/program_bound.metal delete mode 100644 proto-cuda/packs-ca4/mx8_shl256x27/vectors.h delete mode 100644 proto-cuda/packs-ca4/mx8_shl256x27/vectors.json delete mode 100644 proto-cuda/packs-ca4/mx8_shl256x27_v2/kernel.cl delete mode 100644 proto-cuda/packs-ca4/mx8_shl256x27_v2/kernel.cu delete mode 100644 proto-cuda/packs-ca4/mx8_shl256x27_v2/kernel_bound.cl delete mode 100644 proto-cuda/packs-ca4/mx8_shl256x27_v2/kernel_bound.cu delete mode 100644 proto-cuda/packs-ca4/mx8_shl256x27_v2/memhard.h delete mode 100644 proto-cuda/packs-ca4/mx8_shl256x27_v2/memhard.metal delete mode 100644 proto-cuda/packs-ca4/mx8_shl256x27_v2/program.h delete mode 100644 proto-cuda/packs-ca4/mx8_shl256x27_v2/program.json delete mode 100644 proto-cuda/packs-ca4/mx8_shl256x27_v2/program.metal delete mode 100644 proto-cuda/packs-ca4/mx8_shl256x27_v2/program_bound.metal delete mode 100644 proto-cuda/packs-ca4/mx8_shl256x27_v2/vectors.h delete mode 100644 proto-cuda/packs-ca4/mx8_shl256x27_v2/vectors.json diff --git a/igneum-pow/src/accept.rs b/igneum-pow/src/accept.rs index 6871141eb..5777be4b6 100644 --- a/igneum-pow/src/accept.rs +++ b/igneum-pow/src/accept.rs @@ -92,12 +92,6 @@ pub enum Reject { OutputBias { bit: u8, ones: u32 }, /// (c): the distinct-address sum was `sum`. DistinctAddresses { sum: u64 }, - /// (c, the per-load shadow class only; Counter ASIC 4.0 research, experimental): the load at `instr` in - /// `iteration` read one address in `dups` pairs of lanes of `unit` (the class v4 shape is not held to this). - DuplicateLanes { iteration: u8, instr: u8, unit: u8, dups: u8 }, - /// (c, the per-load shadow class only; Counter ASIC 4.0 research, the adv-cache-2 requirement): index bit `bit` at - /// load site `site` was set in `ones` of the 16,384 addresses of the 64 units, outside the 6-sigma band. - BiasedIndexBit { site: u8, bit: u8, ones: u32 }, } impl std::fmt::Display for Reject { @@ -108,12 +102,6 @@ impl std::fmt::Display for Reject { } Reject::NoInjectingWrite { reg } => write!(f, "(b) r{reg} has no add, sub, xor, mad, shfl or load write"), Reject::ConstantBit { reg, bits } => write!(f, "(c) r{reg} has {bits} nonce-independent bits"), - Reject::BiasedIndexBit { site, bit, ones } => { - write!(f, "(c, per-load shadow) index bit {bit} at load site {site} set in {ones} of 16,384 addresses") - } - Reject::DuplicateLanes { iteration, instr, unit, dups } => { - write!(f, "(c, per-load shadow) load at iteration {iteration} instruction {instr} reads a duplicate address in {dups} lanes of unit {unit}") - } Reject::LaneConstantSite { iteration, instr, unit } => { write!(f, "(c) load at iteration {iteration} instruction {instr} reads one address in all lanes of unit {unit}") } @@ -243,7 +231,6 @@ pub fn distinct_indices_v4(p: &Program, units: usize) -> Result, Reject saturated: 0, bit_ones: [0; 64], distinct_sum: 0, - site_bit_ones: Vec::new(), }; let mut lane_addrs = vec![0u32; LANES * loads]; for (unit, &base) in accept_base_nonces_n(&p.seed, units).iter().enumerate() { @@ -369,84 +356,6 @@ struct Acc { saturated: u32, bit_ones: [u32; 64], distinct_sum: u64, - /// Counter ASIC 4.0 research (the per-load class only): one-count of every index bit per load site over the units. - site_bit_ones: Vec<[u32; 32]>, -} - -/// One ALU instruction of a shadow sub-block on the register file (the same arithmetic as the arms of `run_unit`; -/// no load, scratch or hot op is ever in a sub-block). Counter ASIC 4.0 research: the per-load shadow class runs its -/// sub-blocks inside the acceptance test, so the test judges the order the class executes. -fn alu_step(ins: &Instr, r: &mut [[u32; LANES]; 8], sel: &[u32; LANES]) { - let d = ins.dst as usize; - let a = ins.src as usize; - match ins.op { - Op::Add => { - let (imm, imm2, bit) = (ins.imm, ins.imm2, ins.bit as u32); - let src = r[a]; - for lane in 0..LANES { - let s = (sel[lane] >> bit) & 1; - let c = if s != 0 { imm2 } else { imm }; - r[d][lane] = r[d][lane].wrapping_add(src[lane]).wrapping_add(c); - } - } - Op::Sub => { - let src = r[a]; - for lane in 0..LANES { - r[d][lane] = r[d][lane].wrapping_sub(src[lane]); - } - } - Op::Mul => { - let src = r[a]; - for lane in 0..LANES { - r[d][lane] = r[d][lane].wrapping_mul(src[lane]); - } - } - Op::MulHi => { - let src = r[a]; - for lane in 0..LANES { - r[d][lane] = mulhi32(r[d][lane], src[lane]); - } - } - Op::Xor => { - let src = r[a]; - for lane in 0..LANES { - r[d][lane] ^= src[lane]; - } - } - Op::Or => { - let src = r[a]; - for lane in 0..LANES { - r[d][lane] |= src[lane]; - } - } - Op::Rotl => { - let n = ins.rot; - for lane in 0..LANES { - r[d][lane] = r[d][lane].rotate_left(n); - } - } - Op::Rotr => { - let src = r[a]; - for lane in 0..LANES { - r[d][lane] = r[d][lane].rotate_right(src[lane] & 31); - } - } - Op::Mad => { - let src = r[a]; - let src2 = r[ins.src2 as usize]; - for lane in 0..LANES { - r[d][lane] = src[lane].wrapping_mul(src2[lane]).wrapping_add(r[d][lane]); - } - } - Op::Shfl => { - let src = r[a]; - let m = ins.mask as usize; - for lane in 0..LANES { - r[d][lane] ^= src[lane ^ m]; - } - } - Op::Load | Op::WLoad | Op::Scratch | Op::Hot => unreachable!("a shadow sub-block holds ALU instructions only"), - } } /// One unit of the dynamic test: the interpreter of `verify.rs` with the closed-form dataset, instrumented. @@ -477,21 +386,13 @@ fn run_unit(p: &Program, unit: usize, base: u32, acc: &mut Acc, lane_addrs: &mut // after instruction 63 of every iteration, `reps` times with the iteration's `sel`, exactly as the hash does // (verify.rs). Until this commit it ran the 64 base instructions only, so every dynamic test (c) judged a class v4 // program the chain never hashes. The shadow block holds no load, so its instructions take the same arms. - // Counter ASIC 4.0 research: under the per-load placement the block runs as sub-blocks inside the loop below - // (per_load_after), so the end-of-iteration pass is empty for that class. - let per_load = p.shadow_per_load(); - let shadow_reps = if per_load { 0 } else { p.shadow_reps() }; + let shadow_reps = p.shadow_reps(); for it in 0..ITERATIONS { let sel = r[0]; - let mut load_j = 0usize; let shadow_pass = (0..shadow_reps).flat_map(|_| p.shadow.iter().enumerate().map(|(k, i)| (INSTR_COUNT + k, i))); for (k, ins) in p.instrs.iter().enumerate().chain(shadow_pass) { let d = ins.dst as usize; let a = ins.src as usize; - // Counter ASIC 4.0 research (the per-load shadow class): sub-block j runs reps times right after the - // j-th memory operation, as the class executes it, so the saturation and distinctness tests below see - // the register file the loads actually read from - let per_load_after = per_load && ins.op.is_load(); match ins.op { Op::Scratch => { // Variant 5: the slot stands in for the address (bit 31 set so it never aliases a dataset word). @@ -593,24 +494,6 @@ fn run_unit(p: &Program, unit: usize, base: u32, acc: &mut Acc, lane_addrs: &mut if idx.iter().all(|&x| x == idx[0]) { return Err(Reject::LaneConstantSite { iteration: it as u8, instr: k as u8, unit: unit as u8 }); } - if per_load { - let mut sorted = idx; - sorted.sort_unstable(); - let dups = sorted.windows(2).filter(|w| w[0] == w[1]).count(); - if dups > 0 { - return Err(Reject::DuplicateLanes { iteration: it as u8, instr: k as u8, unit: unit as u8, dups: dups as u8 }); - } - // the bit one-counts are sized by check_dynamic; the (c'') pass (distinct_indices_v4) runs this - // unit with no table and judges indices only - if !acc.site_bit_ones.is_empty() { - let site = load_j % acc.site_bit_ones.len(); - for &x in &idx { - for b in 0..32 { - acc.site_bit_ones[site][b] += (x >> b) & 1; - } - } - } - } for lane in 0..LANES { if width == 1 { r[d][lane] ^= dataset_elem(idx[lane], d0, d1); @@ -650,14 +533,6 @@ fn run_unit(p: &Program, unit: usize, base: u32, acc: &mut Acc, lane_addrs: &mut nload += 1; } } - if per_load_after { - for _ in 0..p.shadow_reps() { - for sh in p.shadow_sub_block(load_j) { - alu_step(sh, &mut r, &sel); - } - } - load_j += 1; - } } } for i in 0..8 { @@ -703,42 +578,11 @@ pub fn check_dynamic(p: &Program) -> Result { saturated: 0, bit_ones: [0; 64], distinct_sum: 0, - site_bit_ones: Vec::new(), }; - if p.shadow_per_load() { - acc.site_bit_ones = vec![[0u32; 32]; p.instrs.iter().filter(|i| i.op.is_load()).count().max(1)]; - } let mut lane_addrs = vec![0u32; LANES * loads]; for (unit, &base) in accept_base_nonces(&p.seed).iter().enumerate() { run_unit(p, unit, base, &mut acc, &mut lane_addrs)?; } - if p.shadow_per_load() { - // the value-level test (the adv-cache-2 requirement, 7 October 2026): an index bit whose one-count over the - // 64 units' 16,384 addresses at one site sits outside 6 sigma of n / 2 (n / 2 = 8,192, sigma 64, band 384) is a - // biased address bit (a product's low bit placed by the stride rotation reads 1/4 or 3/4: 4,096 off, 64 sigma) - let n = (ACCEPT_UNITS * ITERATIONS * LANES) as u32; - let band = 6 * ((n as f64) / 4.0).sqrt() as u32; - // the bits judged are those inside the site's era window (verify::window): under an era the top k bits of a - // narrow-window site are fixed by design (the invention lane, 8 October 2026: the first form of this test counted - // them as biased and read 0 of 256 under every drawn era); without an era every index bit is judged - let load_sites: Vec<&Instr> = p.instrs.iter().filter(|i| i.op.is_load()).collect(); - let mask: u32 = (1u32 << ACCEPT_DATASET_LOG2) - 1; - for (site, bits) in acc.site_bit_ones.iter().enumerate() { - let judged = match (p.class.era, load_sites.get(site)) { - (Some(_), Some(ins)) => crate::verify::window(ins, mask, ACCEPT_DATASET_LOG2).0, - _ => mask, - }; - for bit in 0..ACCEPT_DATASET_LOG2 as usize { - if (judged >> bit) & 1 == 0 { - continue; - } - let ones = bits[bit]; - if ones.abs_diff(n / 2) > band { - return Err(Reject::BiasedIndexBit { site: site as u8, bit: bit as u8, ones }); - } - } - } - } for reg in 0..8 { let bits = (acc.and_acc[reg] | !acc.or_acc[reg]).count_ones(); if bits != 0 { @@ -812,7 +656,7 @@ mod tests { let ds = DatasetSource::from_key(p.seed, DatasetMode::ClosedForm, ACCEPT_DATASET_LOG2); let bases = accept_base_nonces(&p.seed); let loads = p.loads_per_hash(); - let mut acc = Acc { sources: None, indices: None, sat_source: vec![0; 64], and_acc: [u32::MAX; 8], or_acc: [0; 8], saturated: 0, bit_ones: [0; 64], distinct_sum: 0, site_bit_ones: Vec::new() }; + let mut acc = Acc { sources: None, indices: None, sat_source: vec![0; 64], and_acc: [u32::MAX; 8], or_acc: [0; 8], saturated: 0, bit_ones: [0; 64], distinct_sum: 0 }; let mut la = vec![0u32; LANES * loads]; let mut ones = [0u32; 64]; for (u, &b) in bases.iter().enumerate() { @@ -847,7 +691,7 @@ mod tests { assert_eq!(p.shadow_reps(), 27); let ds = DatasetSource::from_key(p.seed, DatasetMode::ClosedForm, ACCEPT_DATASET_LOG2); let bases = accept_base_nonces(&p.seed); - let mut acc = Acc { sources: None, indices: None, sat_source: vec![0; 64], and_acc: [u32::MAX; 8], or_acc: [0; 8], saturated: 0, bit_ones: [0; 64], distinct_sum: 0, site_bit_ones: Vec::new() }; + let mut acc = Acc { sources: None, indices: None, sat_source: vec![0; 64], and_acc: [u32::MAX; 8], or_acc: [0; 8], saturated: 0, bit_ones: [0; 64], distinct_sum: 0 }; let mut la = vec![0u32; LANES * p.loads_per_hash()]; let mut ones = [0u32; 64]; for (u, &b) in bases.iter().enumerate() { @@ -862,7 +706,7 @@ mod tests { // and the same program with its shadow stripped hashes differently: the shadow is executed, not skipped let mut bare = p.clone(); bare.shadow.clear(); - let mut acc2 = Acc { sources: None, indices: None, sat_source: vec![0; 64], and_acc: [u32::MAX; 8], or_acc: [0; 8], saturated: 0, bit_ones: [0; 64], distinct_sum: 0, site_bit_ones: Vec::new() }; + let mut acc2 = Acc { sources: None, indices: None, sat_source: vec![0; 64], and_acc: [u32::MAX; 8], or_acc: [0; 8], saturated: 0, bit_ones: [0; 64], distinct_sum: 0 }; for (u, &b) in bases.iter().enumerate() { let _ = run_unit(&bare, u, b, &mut acc2, &mut la); } @@ -878,7 +722,7 @@ mod tests { let p = candidate(&s, s.as_bytes(), 0); let ds = DatasetSource::from_key(p.seed, DatasetMode::ClosedForm, ACCEPT_DATASET_LOG2); let bases = accept_base_nonces(&p.seed); - let mut acc = Acc { sources: None, indices: None, sat_source: vec![0; 64], and_acc: [u32::MAX; 8], or_acc: [0; 8], saturated: 0, bit_ones: [0; 64], distinct_sum: 0, site_bit_ones: Vec::new() }; + let mut acc = Acc { sources: None, indices: None, sat_source: vec![0; 64], and_acc: [u32::MAX; 8], or_acc: [0; 8], saturated: 0, bit_ones: [0; 64], distinct_sum: 0 }; let mut la = vec![0u32; LANES * p.loads_per_hash()]; let mut ones = [0u32; 64]; let mut any = false; @@ -921,7 +765,7 @@ mod tests { let p = generate_class("igneum-genesis", LoadClass::hot(96, 4)); let words = p.hot_words(); let loads = p.loads_per_hash(); - let mut acc = Acc { sources: None, indices: None, sat_source: vec![0; 64], and_acc: [u32::MAX; 8], or_acc: [0; 8], saturated: 0, bit_ones: [0; 64], distinct_sum: 0, site_bit_ones: Vec::new() }; + let mut acc = Acc { sources: None, indices: None, sat_source: vec![0; 64], and_acc: [u32::MAX; 8], or_acc: [0; 8], saturated: 0, bit_ones: [0; 64], distinct_sum: 0 }; let mut la = vec![0u32; LANES * loads]; let mut buckets = [0u64; 16]; let mut hot_count = 0u64; diff --git a/igneum-pow/src/emit.rs b/igneum-pow/src/emit.rs index 50d870216..951ffb4a2 100644 --- a/igneum-pow/src/emit.rs +++ b/igneum-pow/src/emit.rs @@ -386,8 +386,8 @@ fn shadow_instr_line(dialect: CoreDialect, ins: &Instr) -> String { /// iteration loop, after the program's last instruction: `reps` passes over the block with the iteration's `sel`. /// Empty for every program without a shadow, so the pinned v2 and v3 packs do not change by a byte. fn shadow_block(p: &Program, dialect: CoreDialect) -> String { - if !p.has_shadow() || p.shadow_per_load() { - return tile_block(p, dialect); + if !p.has_shadow() { + return String::new(); } let reps = p.shadow_reps(); let mut s = String::with_capacity(64 * p.shadow.len() + 200); @@ -401,92 +401,9 @@ fn shadow_block(p: &Program, dialect: CoreDialect) -> String { s.push_str(&format!(" {} // s{k} {}\n", shadow_instr_line(dialect, ins), ins.op.name())); } s.push_str(" }\n"); - s.push_str(&tile_block(p, dialect)); s } -/// Counter ASIC 4.0 research (experimental, `docs/analysis/counter-asic-4-research.md` section 16): sub-block `j` of a -/// per-load shadow, emitted right after the `j`-th load line, `reps` passes with the iteration's `sel`. Empty for every -/// program whose shadow is not placed per load, so no pinned pack changes. -fn shadow_sub_block(p: &Program, dialect: CoreDialect, j: usize) -> String { - if !p.shadow_per_load() { - return String::new(); - } - let reps = p.shadow_reps(); - let sub = p.shadow_sub_block(j); - let n = sub.len(); - let mut s = String::with_capacity(64 * n + 120); - let ty = if dialect == CoreDialect::Cuda { "uint32_t" } else { "uint" }; - s.push_str(&format!(" // per-load shadow sub-block {j} (Counter ASIC 4.0 research): {n} ALU instructions x {reps} passes after load {j}\n")); - s.push_str(&format!(" for ({ty} sh{j} = 0u; sh{j} < {reps}u; ++sh{j}) {{\n")); - for (k, ins) in sub.iter().enumerate() { - s.push_str(&format!(" {} // s{} {}\n", shadow_instr_line(dialect, ins), j * n + k, ins.op.name())); - } - s.push_str(" }\n"); - s -} - -/// Counter ASIC 4.0 research (experimental, section 15.2): the int8 tile block after the shadow, once per iteration. -/// CUDA runs the PTX `mma.m8n8k16 u8` tile natively (the form bit-exact on the 4090 and 5090, `proto-newpow/mma-shadow`); -/// Metal and OpenCL run the shuffle-and-byte-product reference (`mm8_ref`, emitted by [`tile_prelude`]). Both outputs -/// are added after both are computed, so `c` may equal `a` or `b`. Empty without tiles, so no pinned pack changes. -fn tile_block(p: &Program, dialect: CoreDialect) -> String { - if !p.has_tiles() { - return String::new(); - } - let mut s = String::with_capacity(160 * p.tiles.len() + 200); - s.push_str(&format!(" // int8 tile block (Counter ASIC 4.0 research, experimental): {} mma.m8n8k16 u8 tiles per iteration, both outputs consumed\n", p.tiles.len())); - for (k, d) in p.tiles.iter().enumerate() { - let (a, b, c, c2) = (format!("r{}", d[0]), format!("r{}", d[1]), format!("r{}", d[2]), format!("r{}", d[3])); - let line = match dialect { - CoreDialect::Cuda => format!( - "{{ uint32_t d0_, d1_; asm volatile(\"mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {{%0,%1}}, {{%2}}, {{%3}}, {{%4,%5}};\" : \"=r\"(d0_), \"=r\"(d1_) : \"r\"({a}), \"r\"({b}), \"r\"(0u), \"r\"(0u)); {c} += d0_; {c2} += d1_; }}" - ), - CoreDialect::Metal => format!("{{ uint d0_, d1_; mm8_ref({a}, {b}, lane, d0_, d1_); {c} += d0_; {c2} += d1_; }}"), - CoreDialect::OpenCl => format!("{{ uint d0_, d1_; IGNEUM_MM8_REF({a}, {b}, d0_, d1_); {c} += d0_; {c2} += d1_; }}"), - }; - s.push_str(&format!(" {line} // t{k} mm8 a=r{} b=r{} c=r{} c2=r{}\n", d[0], d[1], d[2], d[3])); - } - s -} - -/// The reference tile for the dialects without a native int8 tile (Metal: a function; OpenCL: a macro, because the -/// local-memory exchange needs the kernel's own `xch`, `lid` and `xk`). Empty without tiles. -fn tile_prelude(p: &Program, dialect: CoreDialect) -> String { - if !p.has_tiles() { - return String::new(); - } - match dialect { - CoreDialect::Cuda => String::new(), - CoreDialect::Metal => "// int8 tile reference (Counter ASIC 4.0 research): A = the 32 lanes' av as 8 x 16 bytes (lane l holds row l >> 2, bytes 4 (l & 3)..), -// B = bv as 16 x 8 (lane l holds column l >> 2); lane l gets C[l >> 2][2 (l & 3)] and C[l >> 2][2 (l & 3) + 1]; the PTX mma.m8n8k16 u8 layout. -static inline void mm8_ref(uint av, uint bv, uint lane, thread uint& d0, thread uint& d1) { - uint row = lane >> 2, col0 = 2u * (lane & 3u); - uint acc0 = 0u, acc1 = 0u; - for (uint j = 0u; j < 4u; ++j) { - uint aw = simd_shuffle(av, (ushort)(4u * row + j)); - uint bw0 = simd_shuffle(bv, (ushort)(4u * col0 + j)); - uint bw1 = simd_shuffle(bv, (ushort)(4u * (col0 + 1u) + j)); - for (uint t = 0u; t < 4u; ++t) { - uint ab = (aw >> (8u * t)) & 0xffu; - acc0 += ab * ((bw0 >> (8u * t)) & 0xffu); - acc1 += ab * ((bw1 >> (8u * t)) & 0xffu); - } - } - d0 = acc0; d1 = acc1; -} - -".to_string(), - CoreDialect::OpenCl => "// int8 tile reference (Counter ASIC 4.0 research): 12 index shuffles and 32 byte products per lane; the PTX mma.m8n8k16 u8 layout. -#define IGNEUM_MM8_REF(av, bv, d0, d1) { uint row_ = lane >> 2, col0_ = 2u * (lane & 3u); uint acc0_ = 0u, acc1_ = 0u; \ - for (uint j_ = 0u; j_ < 4u; ++j_) { uint aw_, bw0_, bw1_; IGNEUM_SHFL_IDX(aw_, (av), 4u * row_ + j_); IGNEUM_SHFL_IDX(bw0_, (bv), 4u * col0_ + j_); IGNEUM_SHFL_IDX(bw1_, (bv), 4u * (col0_ + 1u) + j_); \ - for (uint t_ = 0u; t_ < 4u; ++t_) { uint ab_ = (aw_ >> (8u * t_)) & 0xffu; acc0_ += ab_ * ((bw0_ >> (8u * t_)) & 0xffu); acc1_ += ab_ * ((bw1_ >> (8u * t_)) & 0xffu); } } \ - d0 = acc0_; d1 = acc1_; } - -".to_string(), - } -} - /// The shadow lines of program.h (empty without a shadow). fn shadow_header_lines(p: &Program) -> String { let Some(sh) = p.class.shadow else { return String::new() }; @@ -498,15 +415,6 @@ fn shadow_header_lines(p: &Program) -> String { s.push_str(&format!("#define IGNEUM_SHADOW_REPS {}\n", sh.reps)); s.push_str(&format!("#define IGNEUM_SHADOW_INSTRS_PER_HASH {}\n", p.shadow_instrs_per_hash())); s.push_str(&format!("#define IGNEUM_SHADOW_OP_MIX {}\n", jstr(&p.shadow_op_mix()))); - if sh.per_load { - s.push_str("// Counter ASIC 4.0 research (experimental): the block is placed per load, sub-block j (instrs / 16) after the j-th load.\n"); - s.push_str("#define IGNEUM_SHADOW_PER_LOAD 1\n"); - } - if sh.tiles > 0 { - s.push_str("// Counter ASIC 4.0 research (experimental): IGNEUM_MM8_TILES int8 mma.m8n8k16 u8 tiles per iteration after the shadow block.\n"); - s.push_str(&format!("#define IGNEUM_MM8_TILES {}\n", sh.tiles)); - s.push_str(&format!("#define IGNEUM_MM8_TILES_PER_HASH {}\n", p.tiles_per_hash())); - } s } @@ -968,7 +876,6 @@ fn metal_program_impl(p: &Program, dataset_log2: u32, source: LoadSource, bound: let mut s = String::with_capacity(5000); s.push_str("#include \n"); s.push_str("using namespace metal;\n"); - s.push_str(&tile_prelude(p, CoreDialect::Metal)); s.push('\n'); s.push_str(&format!("#define MASK {}\n", hex(mask))); s.push_str(&hot_define(p)); @@ -1023,7 +930,7 @@ fn metal_program_impl(p: &Program, dataset_log2: u32, source: LoadSource, bound: } s.push_str(" uint nonce = baseNonce + gid;\n"); s.push_str(" uint r0, r1, r2, r3, r4, r5, r6, r7;\n"); - if p.has_wide() || p.has_tiles() { + if p.has_wide() { s.push_str(" uint lane = gid & 31u;\n"); } let iw = if bound { "initw" } else { "SEEDW" }; @@ -1050,7 +957,6 @@ fn metal_program_impl(p: &Program, dataset_log2: u32, source: LoadSource, bound: LoadSource::InlineMemhard(_) => format!("mh_word(cache, {idx})"), } }; - let mut load_j = 0usize; for (k, ins) in p.instrs.iter().enumerate() { let d = format!("r{}", ins.dst); let a = format!("r{}", ins.src); @@ -1085,10 +991,6 @@ fn metal_program_impl(p: &Program, dataset_log2: u32, source: LoadSource, bound: Op::Hot => hot_stmt(CoreDialect::Metal, &d, &a), }; s.push_str(&format!(" {line} // {k}\n")); - if p.shadow_per_load() && ins.op.is_load() { - s.push_str(&shadow_sub_block(p, CoreDialect::Metal, load_j)); - load_j += 1; - } } s.push_str(&shadow_block(p, CoreDialect::Metal)); s.push_str(" }\n"); @@ -1129,7 +1031,6 @@ fn init_line(p: &Program, u: &str, i: usize) -> String { fn cuda_instr_lines(p: &Program, dataset_log2: u32) -> String { let mut s = String::with_capacity(6000); let era = p.class.era; - let mut load_j = 0usize; for (k, ins) in p.instrs.iter().enumerate() { let d = format!("r{}", ins.dst); let a = format!("r{}", ins.src); @@ -1160,10 +1061,6 @@ fn cuda_instr_lines(p: &Program, dataset_log2: u32) -> String { Op::Hot => hot_stmt(CoreDialect::Cuda, &d, &a), }; s.push_str(&format!(" {line} // {k} {}\n", ins.op.name())); - if p.shadow_per_load() && ins.op.is_load() { - s.push_str(&shadow_sub_block(p, CoreDialect::Cuda, load_j)); - load_j += 1; - } } s } @@ -1475,7 +1372,6 @@ pub fn cuda_kernel_bound_at(p: &Program, memhard: Option<&MixParams>, dataset_lo fn opencl_instr_lines(p: &Program, dataset_log2: u32) -> String { let mut s = String::with_capacity(6000); let era = p.class.era; - let mut load_j = 0usize; for (k, ins) in p.instrs.iter().enumerate() { let d = format!("r{}", ins.dst); let a = format!("r{}", ins.src); @@ -1505,10 +1401,6 @@ fn opencl_instr_lines(p: &Program, dataset_log2: u32) -> String { Op::Hot => hot_stmt(CoreDialect::OpenCl, &d, &a), }; s.push_str(&format!(" {line} // {k} {}\n", ins.op.name())); - if p.shadow_per_load() && ins.op.is_load() { - s.push_str(&shadow_sub_block(p, CoreDialect::OpenCl, load_j)); - load_j += 1; - } } s } @@ -1560,11 +1452,8 @@ pub fn opencl_kernel_bound_at(p: &Program, memhard: Option<&MixParams>, dataset_ s.push_str(" (void)lid;\n"); s.push_str("#endif\n"); } - if p.has_wide() || p.has_tiles() { - s.push_str(" uint lane = lid & 31u;\n"); - } if p.has_wide() { - s.push_str(" uint wmask = mask & ~31u;\n"); + s.push_str(" uint lane = lid & 31u;\n uint wmask = mask & ~31u;\n"); } for i in 0..8 { s.push_str(&format!( @@ -1626,15 +1515,9 @@ pub fn opencl_kernel_at(p: &Program, memhard: Option<&MixParams>, dataset_log2: s.push_str("#if IGNEUM_EXCHANGE == 1\n"); s.push_str("#define IGNEUM_SHFL_XOR(dst, a, m) dst = sub_group_shuffle_xor((a), (uint)(m))\n"); s.push_str("#define IGNEUM_BCAST0(dst, a) dst = sub_group_broadcast((a), 0u)\n"); - if p.has_tiles() { - s.push_str("#define IGNEUM_SHFL_IDX(dst, a, i) dst = sub_group_shuffle((a), (uint)(i))\n"); - } s.push_str("#elif IGNEUM_EXCHANGE == 2\n"); s.push_str("#define IGNEUM_SHFL_XOR(dst, a, m) dst = intel_sub_group_shuffle_xor((a), (uint)(m))\n"); s.push_str("#define IGNEUM_BCAST0(dst, a) dst = sub_group_broadcast((a), 0u)\n"); - if p.has_tiles() { - s.push_str("#define IGNEUM_SHFL_IDX(dst, a, i) dst = intel_sub_group_shuffle((a), (uint)(i))\n"); - } s.push_str("#else\n"); s.push_str("// Local-memory exchange. Two buffers of IGNEUM_GROUP words alternate (xk counts exchanges), so one barrier per\n"); s.push_str("// exchange is enough: a lane can only overwrite buffer b at exchange k+2 after passing barrier k+1, and every lane\n"); @@ -1644,12 +1527,8 @@ pub fn opencl_kernel_at(p: &Program, memhard: Option<&MixParams>, dataset_log2: ); s.push_str("#define IGNEUM_SHFL_XOR(dst, a, m) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid ^ (uint)(m))]; xk += 1u; }\n"); s.push_str("#define IGNEUM_BCAST0(dst, a) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid & ~31u)]; xk += 1u; }\n"); - if p.has_tiles() { - s.push_str("#define IGNEUM_SHFL_IDX(dst, a, i) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid & ~31u) + (uint)(i)]; xk += 1u; }\n"); - } s.push_str("#endif\n"); s.push('\n'); - s.push_str(&tile_prelude(p, CoreDialect::OpenCl)); s.push_str(&hot_define(p)); s.push_str("static inline uint splitmix32(uint x) {\n"); s.push_str(" x ^= x >> 16; x *= 0x7feb352du;\n"); @@ -1744,11 +1623,8 @@ pub fn opencl_kernel_at(p: &Program, memhard: Option<&MixParams>, dataset_log2: s.push_str(" (void)lid;\n"); s.push_str("#endif\n"); } - if p.has_wide() || p.has_tiles() { - s.push_str(" uint lane = lid & 31u;\n"); - } if p.has_wide() { - s.push_str(" uint wmask = mask & ~31u;\n"); + s.push_str(" uint lane = lid & 31u;\n uint wmask = mask & ~31u;\n"); } for i in 0..8 { s.push_str(&init_line(p, "uint", i)); @@ -2250,18 +2126,6 @@ pub fn program_json(p: &Program, day: &str, ds: &DatasetSource) -> String { )); } s.push_str(" ]},\n"); - // Counter ASIC 4.0 research (experimental): the placement and the tile block, only when set - if sh.per_load { - s.push_str(" \"shadow_placement\": \"per_load\",\n"); - } - if !p.tiles.is_empty() { - s.push_str(&format!( - " \"mm8_tiles\": {{\"tiles\": {}, \"tiles_per_hash\": {}, \"rule\": \"Counter ASIC 4.0 research (docs/analysis/counter-asic-4-research.md 15.2): after the shadow draws the stream draws tiles descriptors a, b, c (below 8 each) and c2 (below 7, skipping c); each tile is the PTX mma.m8n8k16 u8 product of the 32 lanes' r[a] (8 x 16 bytes) and r[b] (16 x 8), lane l adds C[l >> 2][2 (l & 3)] into r[c] and C[l >> 2][2 (l & 3) + 1] into r[c2] modulo 2^32; the block runs once per iteration after the shadow\", \"program_id_suffix\": \"'mm8/' || tiles_le16\", \"descriptors\": [{}]}},\n", - p.tiles.len(), - p.tiles_per_hash(), - p.tiles.iter().map(|d| format!("[{},{},{},{}]", d[0], d[1], d[2], d[3])).collect::>().join(",") - )); - } } s.push_str(" \"instructions\": [\n"); let n = p.instrs.len(); diff --git a/igneum-pow/src/generator.rs b/igneum-pow/src/generator.rs index 4abca5e5b..a6698e89f 100644 --- a/igneum-pow/src/generator.rs +++ b/igneum-pow/src/generator.rs @@ -201,10 +201,6 @@ pub struct Program { /// The latency-shadow block (Counter ASIC 3.0 item 8): `class.shadow.instrs` ALU instructions, run /// `class.shadow.reps` times at the end of every iteration. Empty for every class without a shadow. pub shadow: Vec, - /// Counter ASIC 4.0 research (7 October 2026, experimental, never a chain class): the int8 tile block, one - /// `[a, b, c, c2]` register descriptor per tile (`a != b` is not required; `c != c2`), run once per iteration - /// after the shadow block. Empty for every class with `shadow.tiles == 0`. - pub tiles: Vec<[u8; 4]>, } /// The widths a `load` may read, in words: 4, 16 and 64 bytes. @@ -383,20 +379,10 @@ pub struct HotClass { /// hash and no load. `None` on every other class: version 2 and class v3 draw nothing and emit nothing. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub struct ShadowClass { - /// Instructions in the shadow block (1..=4096; 0 only on a tiles-only experimental class). + /// Instructions in the shadow block (1..=4096). pub instrs: u16, - /// Times the block runs per iteration (1..=1024; 0 only on a tiles-only experimental class). + /// Times the block runs per iteration (1..=1024). pub reps: u16, - /// Counter ASIC 4.0 research (7 October 2026, `docs/analysis/counter-asic-4-research.md` section 16, experimental, - /// never a chain class): the block is split into [`LOAD_SLOTS`] sub-blocks of `instrs / LOAD_SLOTS` consecutive - /// instructions and sub-block `j` runs `reps` times right after the `j`-th load of the base program, instead of - /// the whole block once after instruction 63. The same instructions, the same count per hash, a different - /// placement (the chip's core then sits inside every read's dependency). Name suffix `+shlx`. - pub per_load: bool, - /// Counter ASIC 4.0 research (section 15.2, experimental): `tiles` int8 tile steps per iteration after the shadow - /// block, each the PTX `mma.m8n8k16 u8` tile over two registers with both outputs consumed (the - /// `proto-newpow/mma-shadow` form, bit-exact on the RTX 4090 and 5090 on 6 October 2026). Name suffix `+mm`. - pub tiles: u16, } impl ShadowClass { @@ -404,18 +390,6 @@ impl ShadowClass { pub fn instrs_per_hash(&self) -> usize { ITERATIONS * self.instrs as usize * self.reps as usize } - /// The block of `instrs` run `reps` times at the end of every iteration (the class v4 shape). - pub const fn block(instrs: u16, reps: u16) -> ShadowClass { - ShadowClass { instrs, reps, per_load: false, tiles: 0 } - } - /// Instructions per per-load sub-block (0 unless `per_load`). - pub fn sub_block_len(&self) -> usize { - if self.per_load { self.instrs as usize / LOAD_SLOTS } else { 0 } - } - /// Tile steps per hash: `ITERATIONS x tiles`. - pub fn tiles_per_hash(&self) -> usize { - ITERATIONS * self.tiles as usize - } } /// Scratch geometry (variant 5): 16-byte slots, lane-major, 32 lanes per warp; `scratch_kb` KiB per warp gives @@ -570,27 +544,7 @@ impl LoadClass { /// The class with a latency-shadow block of `instrs` instructions run `reps` times per iteration ("mx8+sh256x13"). pub fn with_shadow(self, instrs: u16, reps: u16) -> LoadClass { assert!((1..=4096).contains(&instrs) && (1..=1024).contains(&reps), "shadow block: 1..=4096 instructions, 1..=1024 reps"); - LoadClass { shadow: Some(ShadowClass::block(instrs, reps)), ..self } - } - - /// Counter ASIC 4.0 research (experimental): the shadow block placed per load, `instrs` a multiple of - /// [`LOAD_SLOTS`] ("mx8+shl256x27": 16 sub-blocks of 16 instructions, each run 27 times after its load). - pub fn with_shadow_per_load(self, instrs: u16, reps: u16) -> LoadClass { - assert!((1..=4096).contains(&instrs) && (1..=1024).contains(&reps) && instrs as usize % LOAD_SLOTS == 0, "per-load shadow: 16..=4096 instructions in multiples of 16, 1..=1024 reps"); - LoadClass { shadow: Some(ShadowClass { instrs, reps, per_load: true, tiles: 0 }), ..self } - } - - /// Counter ASIC 4.0 research (experimental): `tiles` int8 tile steps per iteration over this class (with or - /// without a shadow block; "mx8+mm512", "mx8+sh256x27+mm128"). - pub fn with_tiles(self, tiles: u16) -> LoadClass { - assert!((1..=4096).contains(&tiles), "tile block: 1..=4096 tiles per iteration"); - let sh = self.shadow.unwrap_or(ShadowClass { instrs: 0, reps: 0, per_load: false, tiles: 0 }); - LoadClass { shadow: Some(ShadowClass { tiles, ..sh }), ..self } - } - - /// Tile steps per hash (0 without a tile block). - pub fn tiles_per_hash(&self) -> usize { - self.shadow.map(|s| s.tiles_per_hash()).unwrap_or(0) + LoadClass { shadow: Some(ShadowClass { instrs, reps }), ..self } } /// Shadow instructions per hash (0 without a shadow). @@ -624,23 +578,6 @@ impl LoadClass { /// "mx4": the v3 construction; a trailing "m" and "g" set the mixer multiplier and the growth rule on any /// load class, "w16m4g" for example). pub fn parse(s: &str) -> Option { - // "+mm": the int8 tile block over any class (Counter ASIC 4.0 research, experimental) - if let Some((base, t)) = s.rsplit_once("+mm") { - let tiles: u16 = t.parse().ok()?; - if !(1..=4096).contains(&tiles) { - return None; - } - return Some(LoadClass::parse(base)?.with_tiles(tiles)); - } - // "+shlx": the shadow block placed per load (Counter ASIC 4.0 research, experimental) - if let Some((base, sh)) = s.rsplit_once("+shl") { - let (instrs, reps) = sh.split_once('x')?; - let (instrs, reps): (u16, u16) = (instrs.parse().ok()?, reps.parse().ok()?); - if !(1..=4096).contains(&instrs) || !(1..=1024).contains(&reps) || instrs as usize % LOAD_SLOTS != 0 { - return None; - } - return Some(LoadClass::parse(base)?.with_shadow_per_load(instrs, reps)); - } // "+shx": the latency-shadow block over any class (Counter ASIC 3.0 item 8) if let Some((base, sh)) = s.rsplit_once("+sh") { let (instrs, reps) = sh.split_once('x')?; @@ -765,14 +702,8 @@ impl LoadClass { /// A hot class appends "hotk[a]" ("hot64k4", "scr4k32+hot64k4a"; measured and not adopted). pub fn name(&self) -> String { if let Some(sh) = self.shadow { - // "+shx": the shadow block is a suffix on any class ("mx8+sh256x13"); "+shl" when it - // is placed per load and "+mm" after it for the tile block (Counter ASIC 4.0 research, experimental) - let base = LoadClass { shadow: None, ..*self }.name(); - let mut n = if sh.instrs > 0 { format!("{base}+sh{}{}x{}", if sh.per_load { "l" } else { "" }, sh.instrs, sh.reps) } else { base }; - if sh.tiles > 0 { - n.push_str(&format!("+mm{}", sh.tiles)); - } - return n; + // "+shx": the shadow block is a suffix on any class ("mx8+sh256x13") + return format!("{}+sh{}x{}", LoadClass { shadow: None, ..*self }.name(), sh.instrs, sh.reps); } if let Some(e) = self.era { let base = LoadClass { era: None, ..*self }; @@ -890,7 +821,7 @@ pub const V3_CLASS: LoadClass = LoadClass { era: None, hot: None, ..LoadClass::M /// of 256 ALU instructions run 27 times per iteration ("mx8+sh256x27", 55,296 shadow instructions per hash). The /// base program, the 16 loads, the item construction, the cache growth rule and the era draw are class v3's, draw /// for draw, so a v4 epoch's day cache and dataset are the v3 day's. Composed with the era exactly as V3 is. -pub const V4_CLASS: LoadClass = LoadClass { shadow: Some(ShadowClass::block(V4_SHADOW_INSTRS, V4_SHADOW_REPS)), ..V3_CLASS }; +pub const V4_CLASS: LoadClass = LoadClass { shadow: Some(ShadowClass { instrs: V4_SHADOW_INSTRS, reps: V4_SHADOW_REPS }), ..V3_CLASS }; /// The shadow block size of class v4 at every rung of the latency ladder (`docs/design/latency-ladder.md`): 256 /// instructions. The ladder moves the pass count alone. @@ -906,7 +837,7 @@ pub fn v4_class_at(reps: u16) -> LoadClass { if reps == 0 { V4_CLASS } else { - LoadClass { shadow: Some(ShadowClass::block(V4_SHADOW_INSTRS, reps)), ..V3_CLASS } + LoadClass { shadow: Some(ShadowClass { instrs: V4_SHADOW_INSTRS, reps }), ..V3_CLASS } } } @@ -915,7 +846,7 @@ pub fn v4_class_at(reps: u16) -> LoadClass { pub fn v4_rung_reps(class: &LoadClass) -> Option { let base = LoadClass { era: None, ..*class }; match base.shadow { - Some(ShadowClass { instrs: V4_SHADOW_INSTRS, reps, per_load: false, tiles: 0 }) if LoadClass { shadow: None, ..base } == V3_CLASS => Some(reps), + Some(ShadowClass { instrs: V4_SHADOW_INSTRS, reps }) if LoadClass { shadow: None, ..base } == V3_CLASS => Some(reps), _ => None, } } @@ -1065,26 +996,6 @@ impl Program { pub fn has_shadow(&self) -> bool { self.class.shadow.is_some() && !self.shadow.is_empty() } - /// Whether the shadow block is placed per load (Counter ASIC 4.0 research, experimental). - pub fn shadow_per_load(&self) -> bool { - self.has_shadow() && self.class.shadow.map(|s| s.per_load).unwrap_or(false) - } - /// The `j`-th per-load sub-block of the shadow (empty unless placed per load). - pub fn shadow_sub_block(&self, j: usize) -> &[Instr] { - if !self.shadow_per_load() { - return &[]; - } - let n = self.class.shadow.map(|s| s.sub_block_len()).unwrap_or(0); - &self.shadow[j * n..(j + 1) * n] - } - /// Whether the program carries an int8 tile block (Counter ASIC 4.0 research, experimental). - pub fn has_tiles(&self) -> bool { - !self.tiles.is_empty() - } - /// Tile steps per hash: `ITERATIONS x tiles`. - pub fn tiles_per_hash(&self) -> usize { - self.tiles.len() * ITERATIONS - } /// Times the shadow block runs per iteration (0 without one). pub fn shadow_reps(&self) -> usize { self.class.shadow.map(|s| s.reps as usize).unwrap_or(0) @@ -1295,15 +1206,6 @@ pub fn program_id_class_recipe(generator: u32, seed: &[u32; 8], attempt: u32, cl r.lit(b"shadow/"); r.field(&sh.instrs.to_le_bytes(), "shadow_instrs_le16"); r.field(&sh.reps.to_le_bytes(), "shadow_reps_le16"); - // Counter ASIC 4.0 research (experimental): the placement and the tile block are part of the construction; - // nothing is appended for the class v4 shape, so every v4 id stands - if sh.per_load { - r.lit(b"perload"); - } - if sh.tiles > 0 { - r.lit(b"mm8/"); - r.field(&sh.tiles.to_le_bytes(), "mm8_tiles_le16"); - } } if let Some(h) = class.hot { r.lit(b"hot/"); @@ -1525,14 +1427,7 @@ pub fn candidate_from_words_class( // the era windows included when the class takes them, drawn and ignored) so the stream shape is the program's. let mut shadow = Vec::new(); if let Some(sh) = class.shadow { - // Counter ASIC 4.0 research (the per-load placement, 7 October 2026, 22:3x UTC): the source register of every - // load in program order, so a per-load sub-block can refuse a lossy writer of the NEXT load's source. Found by - // the trace of the first export (tests/ca4_trace.rs): sub-blocks whose last writer of the next load's source was - // `mul` (a 27-pass `d = d * a` with an even `a` clears the low bits) made 11 to 12 percent of a warp's reads land - // on one item (10,728 distinct of 12,288 over three units; sites 8, 10 and 15; shuffles and rotates were clean). - let load_srcs: Vec = instrs.iter().filter(|i| i.op.is_load()).map(|i| i.src as usize).collect(); - let sub_len = sh.sub_block_len(); - for k in 0..sh.instrs as usize { + for _ in 0..sh.instrs { let mut roll = rng.below(75); let mut op = Op::Add; for &(o, w) in &NONLOAD_WEIGHTS { @@ -1543,24 +1438,6 @@ pub fn candidate_from_words_class( roll -= w; } let dst = rng.below(8); - if sh.per_load && sub_len > 0 && !load_srcs.is_empty() { - // the rule: a sub-block instruction that writes the next load's source register is drawn from the - // non-product injecting families only (add, sub, xor, shfl); mul, mulhi, or and mad are redrawn (the biased-product-bit class of 22:3x UTC) from - // the injecting table (sub-version 3's source rule applied to the order this class executes) - let j = k / sub_len; - let next_src = load_srcs[(j + 1) % load_srcs.len()]; - if dst as usize == next_src && matches!(op, Op::Mul | Op::MulHi | Op::Or | Op::Mad) { - let inj: [(Op, u64); 4] = [(Op::Add, 12), (Op::Sub, 6), (Op::Xor, 10), (Op::Shfl, 8)]; - let mut r2 = rng.below(36); - for &(o, w) in &inj { - if r2 < w { - op = o; - break; - } - r2 -= w; - } - } - } let a = rng.below(7); let src = if a >= dst { a + 1 } else { a }; let b = rng.below(8); @@ -1579,20 +1456,6 @@ pub fn candidate_from_words_class( shadow.push(Instr { op, dst: dst as u8, src: src as u8, src2: b as u8, imm, imm2, rot, bit: bit as u8, mask, width: 1, win: 0, off: 0 }); } } - // (4) Counter ASIC 4.0 research (experimental): the tile descriptors, drawn after the shadow block from the same - // stream: a = below(8), b = below(8), c = below(8), c2 = below(7) skipping c (both outputs land in different - // registers, so every tile is load-bearing; the proto-newpow/mma-shadow form). - let mut tiles = Vec::new(); - if let Some(sh) = class.shadow { - for _ in 0..sh.tiles { - let a = rng.below(8) as u8; - let b = rng.below(8) as u8; - let c = rng.below(8) as u8; - let c2r = rng.below(7) as u8; - let c2 = if c2r >= c { c2r + 1 } else { c2r }; - tiles.push([a, b, c, c2]); - } - } Program { seed_string: seed_string.to_string(), seed_bytes: seed_bytes.to_vec(), @@ -1603,7 +1466,6 @@ pub fn candidate_from_words_class( era_bytes: None, instrs, shadow, - tiles, } } @@ -1847,7 +1709,6 @@ pub fn generate_v1_from_words(seed_string: &str, seed: [u32; 8], cfg: &Generator era_bytes: None, instrs, shadow: Vec::new(), - tiles: Vec::new(), } } @@ -2655,56 +2516,3 @@ mod tests { assert_eq!(e.width_words, 1); } } - -#[cfg(test)] -mod ca4_tests { - //! Counter ASIC 4.0 research (experimental classes): the names, the ids and the draws of the per-load shadow and the tile block. - use super::*; - - #[test] - fn per_load_and_tile_classes_parse_name_and_differ_in_id_from_class_v4_shapes() { - let v4 = LoadClass::parse("mx8+sh256x27").unwrap(); - let pl = LoadClass::parse("mx8+shl256x27").unwrap(); - let mm = LoadClass::parse("mx8+mm512").unwrap(); - let both = LoadClass::parse("mx8+sh256x27+mm128").unwrap(); - assert_eq!(pl.name(), "mx8+shl256x27"); - assert_eq!(mm.name(), "mx8+mm512"); - assert_eq!(both.name(), "mx8+sh256x27+mm128"); - assert!(pl.shadow.unwrap().per_load && pl.shadow.unwrap().sub_block_len() == 16); - assert_eq!(mm.shadow.unwrap().instrs, 0); - assert!(LoadClass::parse("mx8+shl250x27").is_none(), "a per-load block is a multiple of 16"); - assert_eq!(V4_CLASS.name(), "mx8+sh256x27", "the class v4 name is unchanged"); - let seed = b"igneum-genesis"; - let p4 = generate_from_seed_bytes_class("t", seed, v4); - // the per-load 16 x 27 construction is refused by the acceptance rule on every attempt once the rule steps the - // sub-blocks in execution order (22:4x UTC, tests/ca4_trace.rs): its candidates are read, never accepted - let pl_attempts = attempts_class("t", seed, pl); - assert!(pl_attempts.len() > 8, "the per-load class accepts 1.4 percent of candidates (22:4x UTC); seed t took {} attempts", pl_attempts.len()); - let pp = pl_attempts[0].0.clone(); - let pm = generate_from_seed_bytes_class("t", seed, mm); - let pb = generate_from_seed_bytes_class("t", seed, both); - // the per-load class takes the dynamic acceptance test in its own execution order (accept.rs DuplicateLanes), - // so a candidate the class v4 shape accepts may be rejected here and the accepted program is a later attempt; - // when the attempt is the same the base program is the class v4 shape's draw for draw - if pp.attempt == p4.attempt { - assert_eq!(p4.instrs, pp.instrs, "the base program is the class's without the shadow, draw for draw"); - } - assert_eq!(p4.shadow.len(), pp.shadow.len(), "the per-load shadow holds a block of the same size"); - assert!(p4.tiles.is_empty() && pp.tiles.is_empty()); - assert_eq!(pm.tiles.len(), 512); - assert_eq!(pb.tiles.len(), 128); - assert_eq!(pb.shadow, p4.shadow, "the tile draws come after the shadow draws"); - assert!(pm.tiles.iter().all(|d| d[2] != d[3] && d.iter().all(|&x| x < 8))); - let ids = [p4.program_id(), pp.program_id(), pm.program_id(), pb.program_id()]; - for i in 0..4 { - for j in 0..4 { - if i != j { - assert_ne!(ids[i], ids[j], "ids {i} {j}"); - } - } - } - for j in 0..LOAD_SLOTS { - assert_eq!(pp.shadow_sub_block(j), &pp.shadow[16 * j..16 * j + 16]); - } - } -} diff --git a/igneum-pow/src/lib.rs b/igneum-pow/src/lib.rs index fca2e06fc..690ddb0bc 100644 --- a/igneum-pow/src/lib.rs +++ b/igneum-pow/src/lib.rs @@ -29,7 +29,6 @@ pub mod derive; pub mod emit; pub mod generator; pub mod memhard; -pub mod mm8; pub mod packcheck; pub mod seed; pub mod state; diff --git a/igneum-pow/src/mm8.rs b/igneum-pow/src/mm8.rs deleted file mode 100644 index d0f084fd2..000000000 --- a/igneum-pow/src/mm8.rs +++ /dev/null @@ -1,179 +0,0 @@ -//! Counter ASIC 4.0 research (7 October 2026, `docs/analysis/counter-asic-4-research.md` section 15.2; experimental, -//! never a chain class): the int8 tile step on a warp's register file, the CPU side of the PTX `mma.m8n8k16 u8` tile -//! as `proto-newpow/mma-shadow` defined it (bit-exact against the RTX 4090 and 5090 on 6 October 2026). -//! -//! The tile: `A` is the 32 lanes' `r[a]` read as an 8 x 16 matrix of bytes (lane `l` holds row `l >> 2`, bytes -//! `4 (l & 3) .. 4 (l & 3) + 3`, byte 0 the lowest), `B` the 32 lanes' `r[b]` as 16 x 8 (lane `l` holds column -//! `l >> 2`, rows `4 (l & 3) ..`), `C = A x B` exact in 32-bit arithmetic (at most 16 x 255 x 255), and lane `l` -//! adds `C[l >> 2][2 (l & 3)]` into `r[c]` and `C[l >> 2][2 (l & 3) + 1]` into `r[c2]` modulo 2^32, both outputs -//! consumed. The scalar form is the reference; the AVX2 form (x86-64, detected at run time) computes the same 64 dot -//! products with `maddubs` on a 4-way split of `B` (`B = 4 (B >> 2) + (B & 3)`, so no 16-bit lane saturates) and is -//! checked against the scalar form by the unit test below. - -use crate::generator::LANES; - -/// One tile step on the register file: `r[c] += C[row][col0]`, `r[c2] += C[row][col0 + 1]` per lane. -pub fn tile_step(r: &mut [[u32; LANES]; 8], d: [u8; 4]) { - let (a, b, c, c2) = (d[0] as usize, d[1] as usize, d[2] as usize, d[3] as usize); - let av = r[a]; - let bv = r[b]; - let mut cm = [[0u32; 8]; 8]; - product(&av, &bv, &mut cm); - for l in 0..LANES { - let row = l >> 2; - let col0 = 2 * (l & 3); - r[c][l] = r[c][l].wrapping_add(cm[row][col0]); - r[c2][l] = r[c2][l].wrapping_add(cm[row][col0 + 1]); - } -} - -/// `C = A x B` for the tile's layouts, into `cm[row][col]`. -pub fn product(av: &[u32; LANES], bv: &[u32; LANES], cm: &mut [[u32; 8]; 8]) { - #[cfg(target_arch = "x86_64")] - { - if avx2_available() { - // SAFETY: the feature was detected at run time. - unsafe { product_avx2(av, bv, cm) }; - return; - } - } - product_scalar(av, bv, cm); -} - -/// The reference: 64 dot products of 16 bytes in plain integer code. -pub fn product_scalar(av: &[u32; LANES], bv: &[u32; LANES], cm: &mut [[u32; 8]; 8]) { - for row in 0..8 { - for col in 0..8 { - let mut acc = 0u32; - for k in 0..16 { - let ab = (av[row * 4 + k / 4] >> (8 * (k % 4))) & 0xff; - let bb = (bv[col * 4 + k / 4] >> (8 * (k % 4))) & 0xff; - acc = acc.wrapping_add(ab * bb); - } - cm[row][col] = acc; - } - } -} - -#[cfg(target_arch = "x86_64")] -fn avx2_available() -> bool { - use std::sync::atomic::{AtomicU8, Ordering}; - static STATE: AtomicU8 = AtomicU8::new(0); - match STATE.load(Ordering::Relaxed) { - 1 => true, - 2 => false, - _ => { - // IGNEUM_MM8_SCALAR=1 forces the reference path (for the bench's scalar row) - let yes = std::is_x86_feature_detected!("avx2") && std::env::var_os("IGNEUM_MM8_SCALAR").is_none(); - STATE.store(if yes { 1 } else { 2 }, Ordering::Relaxed); - yes - } - } -} - -/// Whether the SIMD path is in use on this machine (for the bench's report line). -pub fn simd_path() -> &'static str { - #[cfg(target_arch = "x86_64")] - { - if avx2_available() { - return "avx2"; - } - } - "scalar" -} - -#[cfg(target_arch = "x86_64")] -#[target_feature(enable = "avx2")] -unsafe fn product_avx2(av: &[u32; LANES], bv: &[u32; LANES], cm: &mut [[u32; 8]; 8]) { - use std::arch::x86_64::*; - let a_bytes = av.as_ptr() as *const u8; - let b_bytes = bv.as_ptr() as *const u8; - let ones = _mm256_set1_epi16(1); - let m3 = _mm256_set1_epi8(3); - let m63 = _mm256_set1_epi8(63); - for row in 0..8 { - // the row's 16 bytes in both 128-bit halves - let ar = _mm_loadu_si128(a_bytes.add(16 * row) as *const __m128i); - let a2 = _mm256_broadcastsi128_si256(ar); - for colp in 0..4 { - // columns 2 colp and 2 colp + 1: 32 contiguous bytes of B - let b2 = _mm256_loadu_si256(b_bytes.add(32 * colp) as *const __m256i); - let bq = _mm256_and_si256(_mm256_srli_epi16(b2, 2), m63); // B >> 2 per byte (0..63) - let br = _mm256_and_si256(b2, m3); // B & 3 - // u8 x s8 pairs summed to i16: at most 2 x 255 x 63 = 32,130, no saturation - let pq = _mm256_maddubs_epi16(a2, bq); - let pr = _mm256_maddubs_epi16(a2, br); - let sq = _mm256_madd_epi16(pq, ones); // 8 x i32: 4 per column - let sr = _mm256_madd_epi16(pr, ones); - let s = _mm256_add_epi32(_mm256_slli_epi32(sq, 2), sr); - let mut t = [0i32; 8]; - _mm256_storeu_si256(t.as_mut_ptr() as *mut __m256i, s); - let c0 = (t[0] as u32).wrapping_add(t[1] as u32).wrapping_add(t[2] as u32).wrapping_add(t[3] as u32); - let c1 = (t[4] as u32).wrapping_add(t[5] as u32).wrapping_add(t[6] as u32).wrapping_add(t[7] as u32); - cm[row][2 * colp] = c0; - cm[row][2 * colp + 1] = c1; - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn mix(mut x: u32) -> u32 { - x ^= x >> 16; - x = x.wrapping_mul(0x7feb352d); - x ^= x >> 15; - x = x.wrapping_mul(0x846ca68b); - x ^= x >> 16; - x - } - - #[test] - fn simd_product_equals_the_scalar_reference() { - for seed in 0..64u32 { - let mut av = [0u32; LANES]; - let mut bv = [0u32; LANES]; - for l in 0..LANES { - av[l] = mix(seed * 97 + l as u32); - bv[l] = mix(seed * 131 + 1000 + l as u32); - } - // the saturation edge: all-ones bytes everywhere - if seed == 63 { - av = [0xffff_ffff; LANES]; - bv = [0xffff_ffff; LANES]; - } - let mut c1 = [[0u32; 8]; 8]; - let mut c2 = [[0u32; 8]; 8]; - product_scalar(&av, &bv, &mut c1); - product(&av, &bv, &mut c2); - assert_eq!(c1, c2, "seed {seed}, path {}", simd_path()); - if seed == 63 { - assert_eq!(c1[0][0], 16 * 255 * 255); - } - } - } - - #[test] - fn a_tile_step_adds_both_outputs_and_a_zero_operand_adds_nothing() { - let mut r = [[0u32; LANES]; 8]; - for l in 0..LANES { - r[1][l] = mix(l as u32 + 7); - r[2][l] = mix(l as u32 + 99); - r[3][l] = 5; - r[4][l] = 9; - } - let before = r; - tile_step(&mut r, [1, 2, 3, 4]); - assert!(r[3] != before[3] && r[4] != before[4]); - let mut cm = [[0u32; 8]; 8]; - product_scalar(&before[1], &before[2], &mut cm); - for l in 0..LANES { - assert_eq!(r[3][l], 5u32.wrapping_add(cm[l >> 2][2 * (l & 3)])); - assert_eq!(r[4][l], 9u32.wrapping_add(cm[l >> 2][2 * (l & 3) + 1])); - } - let mut z = before; - tile_step(&mut z, [0, 2, 3, 4]); // r[0] is all zero: C is zero - assert_eq!(z, before); - } -} diff --git a/igneum-pow/src/verify.rs b/igneum-pow/src/verify.rs index c59204edc..9d1281aeb 100644 --- a/igneum-pow/src/verify.rs +++ b/igneum-pow/src/verify.rs @@ -365,62 +365,6 @@ pub fn interpret_warp_init(program: &Program, seed: &[u32; 8], base_nonce: u32, interpret_warp_scratch(program, seed, base_nonce, ds, false).0 } -/// Counter ASIC 4.0 research (diagnostic, experimental classes): the dataset index every lane reads at every load of -/// the unit, in execution order (`ITERATIONS x loads` rows of 32), so a test can count duplicates within a warp per load -/// site and across iterations. Runs the program exactly as [`interpret_warp_init`] does (the per-load sub-blocks and the -/// tile block included); scratch and hot classes are not traced here. -pub fn trace_load_indices(program: &Program, seed: &[u32; 8], base_nonce: u32, ds: &DatasetSource) -> Vec<[u32; LANES]> { - assert!(!program.has_scratch() && !program.has_hot(), "trace_load_indices: ALU, load, shadow and tile programs only"); - let mask = ds.mask; - let log2 = ds.log2_words; - let era = program.class.era; - let layout = program.class.layout(); - let mut r = [[0u32; LANES]; 8]; - for lane in 0..LANES { - let nonce = base_nonce.wrapping_add(lane as u32); - for i in 0..8 { - let mut x = nonce ^ seed[i]; - x = x.wrapping_add(0x9e3779b9u32.wrapping_mul(i as u32 + 1)); - x = splitmix32(x); - r[i][lane] = x ^ seed[(i + 1) & 7]; - } - } - let mut items_derived = 0usize; - let mut idx = [0u32; LANES]; - let mut val = [0u32; LANES]; - let mut out = Vec::new(); - let per_load = program.shadow_per_load(); - for _ in 0..ITERATIONS { - let sel = r[0]; - let mut load_j = 0usize; - for ins in &program.instrs { - step(ins, &mut r, &sel, mask, log2, era.as_ref(), layout, ds, &mut idx, &mut val, &mut items_derived); - if ins.op.is_load() { - out.push(idx); - if per_load { - for _ in 0..program.shadow_reps() { - for sh in program.shadow_sub_block(load_j) { - step(sh, &mut r, &sel, mask, log2, era.as_ref(), layout, ds, &mut idx, &mut val, &mut items_derived); - } - } - load_j += 1; - } - } - } - if !per_load { - for _ in 0..program.shadow_reps() { - for ins in &program.shadow { - step(ins, &mut r, &sel, mask, log2, era.as_ref(), layout, ds, &mut idx, &mut val, &mut items_derived); - } - } - } - for d in &program.tiles { - crate::mm8::tile_step(&mut r, *d); - } - } - out -} - /// [`interpret_warp_init`] that also returns every scratch read-modify-write of the unit in execution order /// (lane-minor within an instruction, as the interpreter runs them) when `trace` is set; empty otherwise and for /// a class without a scratch. For the soundness tests of variant 5 only. @@ -459,22 +403,10 @@ pub fn interpret_warp_scratch( let h = ds.hot.as_ref().expect("a hot-table program needs the epoch's hot table on the dataset source"); assert_eq!(h.n_words(), program.hot_words(), "the hot table's size is the class's"); } - let per_load = program.shadow_per_load(); for _ in 0..ITERATIONS { let sel = r[0]; - let mut load_j = 0usize; for ins in &program.instrs { step(ins, &mut r, &sel, mask, log2, era.as_ref(), layout, ds, &mut idx, &mut val, &mut items_derived); - // Counter ASIC 4.0 research (experimental): the shadow placed per load runs sub-block j right after the - // j-th load, `reps` times, with the iteration's `sel` - if per_load && ins.op.is_load() { - for _ in 0..program.shadow_reps() { - for sh in program.shadow_sub_block(load_j) { - step(sh, &mut r, &sel, mask, log2, era.as_ref(), layout, ds, &mut idx, &mut val, &mut items_derived); - } - } - load_j += 1; - } if ins.op == Op::Scratch { let m = scratch.as_mut().expect("a scratch op needs a scratch class"); let (d, a) = (ins.dst as usize, ins.src as usize); @@ -486,17 +418,11 @@ pub fn interpret_warp_scratch( } // Latency-shadow block (Counter ASIC 3.0 item 8): the block runs `reps` times after instruction 63 with the // iteration's `sel`; it is empty on every class without a shadow, so version 2 and class v3 run nothing here. - if !per_load { - for _ in 0..program.shadow_reps() { - for ins in &program.shadow { - step(ins, &mut r, &sel, mask, log2, era.as_ref(), layout, ds, &mut idx, &mut val, &mut items_derived); - } + for _ in 0..program.shadow_reps() { + for ins in &program.shadow { + step(ins, &mut r, &sel, mask, log2, era.as_ref(), layout, ds, &mut idx, &mut val, &mut items_derived); } } - // Counter ASIC 4.0 research (experimental): the int8 tile block after the shadow, once per iteration - for d in &program.tiles { - crate::mm8::tile_step(&mut r, *d); - } } let mut hashes = [0u64; LANES]; for lane in 0..LANES { diff --git a/igneum-pow/tests/ca4_trace.rs b/igneum-pow/tests/ca4_trace.rs deleted file mode 100644 index 2bc4c02d2..000000000 --- a/igneum-pow/tests/ca4_trace.rs +++ /dev/null @@ -1,223 +0,0 @@ -//! Counter ASIC 4.0 research (diagnostic): where the per-load shadow's duplicate reads come from. Prints, for the class -//! v4 shape and the per-load shape over the same seed and day on a 2^24-word closed-form dataset (the index pattern is -//! the program's, not the dataset's), the distinct indices per load site across the 32 lanes and the distinct items per -//! unit across all 128 reads, plus the shadow sub-block's last writer of each load's source register. -use igneum_pow::generator::{generate_from_seed_bytes_class, LoadClass}; -use igneum_pow::verify::{trace_load_indices, DatasetMode, DatasetSource}; -use std::collections::HashSet; - -#[test] -fn per_load_shadow_duplicate_reads_are_counted_per_site() { - let ds = DatasetSource::new("2026-10-03", DatasetMode::ClosedForm, 24); - let seed = igneum_pow::seed::seed_words_from_bytes(b"igneum-genesis"); - for name in ["mx8+sh256x27", "mx8+shl256x27"] { - let class = LoadClass::parse(name).unwrap(); - // the per-load class is refused by the acceptance rule on every attempt (22:4x UTC): its candidate 0 is read here - // as the known-failed record (the first export's program), the class v4 shape through the accepted program - let p = if name == "mx8+shl256x27" { - let v = igneum_pow::generator::attempts_class("igneum-genesis", b"igneum-genesis", class); - println!("CA4VERDICT igneum-genesis per-load 16 x 27: {} candidates, accepted {}", v.len(), v.iter().filter(|(_, r)| r.is_ok()).count()); - v.into_iter().next().unwrap().0 - } else { - generate_from_seed_bytes_class("igneum-genesis", b"igneum-genesis", class) - }; - let mut total_distinct = 0usize; - let mut per_site = vec![0usize; 16]; - let mut dup_pairs_same_iter = 0usize; - let mut all: HashSet = HashSet::new(); - for base in [0u32, 4096, 1_000_000] { - let rows = trace_load_indices(&p, &seed, base, &ds); - assert_eq!(rows.len(), 128); - let mut unit: HashSet = HashSet::new(); - for (k, row) in rows.iter().enumerate() { - let site = k % 16; - let s: HashSet = row.iter().copied().collect(); - per_site[site] += 32 - s.len(); - dup_pairs_same_iter += 32 - s.len(); - for &x in row { - unit.insert(x); - } - } - total_distinct += unit.len(); - all.extend(unit); - } - // the shadow's last writer of each load's source, in the per-load order (sub-block j precedes load j + 1) - let mut writers = Vec::new(); - if p.shadow_per_load() { - let loads: Vec<_> = p.instrs.iter().filter(|i| i.op.is_load()).collect(); - for (j, ld) in loads.iter().enumerate() { - let prev = if j == 0 { 15 } else { j - 1 }; - let sub = p.shadow_sub_block(prev); - let w = sub.iter().rev().find(|i| i.dst == ld.src).map(|i| i.op.name()).unwrap_or("none"); - writers.push(format!("load{j} src r{} <- {w}", ld.src)); - } - } - if name == "mx8+shl256x27" { - // the known-failed record (the first export, 22:1x UTC): 10,728 distinct of 12,288 over three units and - // 1,482 same-iteration duplicate lanes at sites 8, 10 and 15, whose sub-block last writer of the load's - // source was `mul`; after the 22:3x UTC rule the class must read 0 duplicates like the class v4 shape - // the known-failed record (the first export, id 854050a4293f0615: 10,728 distinct of 12,288, 1,482 duplicate - // lanes) was drawn before the static redraw layer; today's candidate 0 is another program, so its figures are - // printed, not asserted (the file carries the record; the pack proto-cuda/packs-ca4/mx8_shl256x27 is the program) - let _ = (dup_pairs_same_iter, total_distinct); - } - println!( - "CA4TRACE class {name}: distinct items per unit (3 units of 4,096 reads) {total_distinct} of 12,288; within-warp duplicate lanes per site over 3 units x 8 iterations {:?}; same-iteration duplicate lanes {dup_pairs_same_iter}; sub-block last writers of load sources {:?}", - per_site, writers - ); - } -} - -#[test] -fn per_load_shadow_census_64_seeds_is_refused_on_every_attempt() { - // the verdict of 22:4x UTC: with the acceptance rule stepping the per-load sub-blocks in the order the class - // executes (duplicate lanes at a load row, biased index bits at a site over the 64 units), no candidate of the - // 16 x 27 construction passes; the histogram of first failing tests is the record - use igneum_pow::generator::attempts_class; - let class = LoadClass::parse("mx8+shl256x27").unwrap(); - let mut accepted = 0usize; - let mut attempts_total = 0usize; - let mut hist: std::collections::BTreeMap = std::collections::BTreeMap::new(); - for n in 0..64u32 { - let label = format!("ca4-census/{n}"); - let v = attempts_class(&label, label.as_bytes(), class); - attempts_total += v.len(); - for (_, r) in &v { - match r { - Ok(()) => accepted += 1, - Err(e) => { - let s = e.to_string(); - let key = if s.contains("duplicate address") { "(c) per-load duplicate lanes" } else if s.contains("index bit") { "(c) per-load biased index bit" } else if s.starts_with("(a)") { "(a) stale source" } else if s.starts_with("(b)") { "(b) no injecting write" } else { "(c) base rule" }; - *hist.entry(key.to_string()).or_insert(0) += 1; - } - } - } - } - let seeds_without = 64 - accepted; - println!("CA4CENSUS per-load 16 x 27 over 64 seeds: {accepted} accepted of {attempts_total} candidates ({:.1} percent); {seeds_without} of 64 seeds exhaust the chain's 32 attempts; first failing test {hist:?}", accepted as f64 / attempts_total as f64 * 100.0); - // the record of 22:4x UTC: 22 of 1,621 (1.4 percent), 42 seeds without a program; a construction that accepts - // under 5 percent of its candidates is dead as a chain class at the 32-attempt cap - assert!((accepted as f64) < 0.05 * attempts_total as f64, "the acceptance rate moved: {accepted} of {attempts_total}"); -} - -#[test] -fn per_load_shadow_census_over_16_drawn_eras_splits_by_stride_rotation() { - // the Counter lane's ask (adv-cache-2, 7 October 2026, 23:1x UK): read the per-load class's duplicate reads across - // drawn eras, split by the stride rotation R (R 28 and up cuts a product's biased low bits out of the index; - // R 3 to 22 keeps them in), not the devnet era alone - use igneum_pow::generator::{generate_era, EraParams, V3_ALLOWED}; - let ds = DatasetSource::new("2026-10-03", DatasetMode::ClosedForm, 24); - let mut low = (0usize, 0usize, 0usize); // eras, rows, duplicate pairs with R under 28 - let mut high = (0usize, 0usize, 0usize); - let mut lines = Vec::new(); - for n in 0..16u32 { - let label = format!("igneum-era-test/{n}"); - let eb = EraParams::test_era_bytes(&label); - // the per-load class is refused on every attempt (22:4x UTC), so generate_era would panic: the era rows below - // read the class v4 shape as the control of the duplicate-lane statistic across the drawn eras - let p = generate_era("igneum-genesis", b"igneum-genesis", LoadClass::parse("mx8+sh256x27").unwrap(), &eb, &V3_ALLOWED); - let era = p.class.era.expect("an era class"); - let seed = igneum_pow::seed::seed_words_from_bytes(b"igneum-genesis"); - let mut dups = 0usize; - let mut rows = 0usize; - for base in [0u32, 1 << 20] { - for row in trace_load_indices(&p, &seed, base, &ds) { - let s: HashSet = row.iter().copied().collect(); - dups += 32 - s.len(); - rows += 1; - } - } - let bucket = if era.stride_rot >= 28 { &mut high } else { &mut low }; - bucket.0 += 1; - bucket.1 += rows; - bucket.2 += dups; - lines.push(format!("era {n} R={} attempt {} dups {dups}", era.stride_rot, p.attempt)); - } - println!("CA4ERA class v4 shape (the control; the per-load class is refused on every attempt) over 16 drawn eras: R under 28: {} eras, {} rows, {} duplicate pairs; R 28 and up: {} eras, {} rows, {} duplicate pairs; per era {:?}", low.0, low.1, low.2, high.0, high.1, high.2, lines); - assert!(low.2 + high.2 <= 4, "duplicate reads beyond the chance floor across eras"); -} - -/// The value-level test 20.2b names: for every load site, the one-count of each index bit over the units' lanes and -/// iterations; a bit outside the binomial band (mean n/2, tolerance 5 sigma) at any site is a biased address bit. Reads -/// the class v4 shape and the per-load class over the devnet-style seed and 16 drawn eras, split by the stride rotation. -#[test] -fn index_bit_bias_per_site_across_eras() { - use igneum_pow::generator::{generate_era, EraParams, V3_ALLOWED}; - let ds = DatasetSource::new("2026-10-03", DatasetMode::ClosedForm, 24); - let seed = igneum_pow::seed::seed_words_from_bytes(b"igneum-genesis"); - for name in ["mx8+sh256x27", "mx8+shl256x27"] { - let class = LoadClass::parse(name).unwrap(); - let mut report = Vec::new(); - let mut flagged: Vec = Vec::new(); - for n in 0..17u32 { - let (p, r_label) = if n == 16 { - if name == "mx8+shl256x27" { - // candidate 0 of the refused per-load class (the known-failed record) - (igneum_pow::generator::attempts_class("igneum-genesis", b"igneum-genesis", class).into_iter().next().unwrap().0, "none".to_string()) - } else { - (generate_from_seed_bytes_class("igneum-genesis", b"igneum-genesis", class), "none".to_string()) - } - } else { - if name == "mx8+shl256x27" { - continue; // generate_era has no candidate path and the class is refused; the bare-class row stands - } - let eb = EraParams::test_era_bytes(&format!("igneum-era-test/{n}")); - let p = generate_era("igneum-genesis", b"igneum-genesis", class, &eb, &V3_ALLOWED); - let r = p.class.era.unwrap().stride_rot; - (p, r.to_string()) - }; - // ones[site][bit] over 4 units x 8 iterations x 32 lanes = 1,024 samples per site - let mut ones = vec![[0u32; 24]; 16]; - let mut samples = 0u32; - for base in [0u32, 1 << 20, 7 << 20, 0x0123_4560] { - for (k, row) in trace_load_indices(&p, &seed, base, &ds).iter().enumerate() { - for &x in row { - for b in 0..24 { - ones[k % 16][b] += (x >> b) & 1; - } - } - } - samples += 8 * 32; - } - let mean = samples as f64 / 2.0; - let sigma = (samples as f64 / 4.0).sqrt(); - let mut worst = (0.0f64, 0usize, 0usize); - for site in 0..16 { - for b in 0..24 { - let z = (ones[site][b] as f64 - mean).abs() / sigma; - if z > worst.0 { - worst = (z, site, b); - } - } - } - let loads: Vec<_> = p.instrs.iter().filter(|i| i.op.is_load()).collect(); - let ld = loads[worst.1]; - let writer = p.instrs.iter().take_while(|i| !std::ptr::eq(*i, ld)).filter(|i| i.dst == ld.src).last().map(|i| i.op.name()).unwrap_or("none"); - let sub_writer = if p.shadow_per_load() { - let prev = if worst.1 == 0 { 15 } else { worst.1 - 1 }; - p.shadow_sub_block(prev).iter().filter(|i| i.dst == ld.src).last().map(|i| i.op.name()).unwrap_or("none") - } else { "n/a" }; - report.push(format!("R={r_label} worst z {:.1} at site {} bit {} (ones {} of {samples}; base writer {writer}, sub-block writer {sub_writer})", worst.0, worst.1, worst.2, ones[worst.1][worst.2])); - if worst.0 >= 5.0 { - flagged.push(report.last().unwrap().clone()); - } - } - println!("CA4BIAS class {name}: index bit one-counts over 1,024 samples per site, 16 sites x 24 bits, 16 drawn eras plus the bare class: {:?}; flagged (z at or over 5) {}", report, flagged.len()); - // a report, not a gate: on this generator (master, before the sub-version 3 source rule) the class v4 shape itself - // carries the biased product bit at address bit R (adv-cache-2, 7 October 2026); the per-load class is read beside it - if name == "mx8+shl256x27" { - assert!(!flagged.is_empty(), "candidate 0 of the per-load class carries the biased product bit (the record)"); - } - } -} - -#[test] -fn per_load_attempt_verdicts_on_the_test_seed() { - use igneum_pow::generator::attempts_class; - let class = LoadClass::parse("mx8+shl256x27").unwrap(); - for seed in ["t", "igneum-genesis", "ca4-census/0", "ca4-census/1"] { - let v = attempts_class(seed, seed.as_bytes(), class); - let verdicts: Vec = v.iter().map(|(p, r)| format!("a{} {}", p.attempt, match r { Ok(()) => "OK".to_string(), Err(e) => e.to_string() })).collect(); - println!("CA4ATTEMPTS seed {seed}: {} attempts; {:?}", v.len(), verdicts); - } -} diff --git a/igneum-pow/tests/scratch.rs b/igneum-pow/tests/scratch.rs index c03344d2b..5ad1c873d 100644 --- a/igneum-pow/tests/scratch.rs +++ b/igneum-pow/tests/scratch.rs @@ -264,7 +264,6 @@ fn edge(name: &str, c: LoadClass, instrs: Vec) -> Program { era_bytes: None, instrs, shadow: Vec::new(), - tiles: Vec::new(), } } diff --git a/igneum-pow/tests/v6_window_bits.rs b/igneum-pow/tests/v6_window_bits.rs deleted file mode 100644 index 8c32eddf2..000000000 --- a/igneum-pow/tests/v6_window_bits.rs +++ /dev/null @@ -1,46 +0,0 @@ -//! Class v6 invention lane (8 October 2026): the per-load acceptance's value-level bias test must not judge the era -//! window's fixed top index bits. Known-failed first: at counter-asic-4 5984ffab every per-load form under every drawn -//! era was refused with "index bit 26 (or 27) at load site s set in 0 (or 16,384) of 16,384" (0 of 256 seeds on 11 -//! forms, build-1, 11:0x UK); with the window bits skipped the sound form accepts under an era as it does without one. -//! Offered by the invention lane (tools/attack/v6-invention/v6_window_bits.rs.offered on class-v6-invention), landed -//! here by the research lane against 0ab27582's fix. -use igneum_pow::accept::Reject; -use igneum_pow::generator::{attempts_class, LoadClass}; -use igneum_pow::seed::seed_words_from_bytes; - -fn era_class(name: &str, n: u32) -> LoadClass { - let era_seed = seed_words_from_bytes(format!("igneum-era-test/{n}").as_bytes()); - let mut bytes = Vec::new(); - for w in era_seed { - bytes.extend_from_slice(&w.to_le_bytes()); - } - LoadClass::era(LoadClass::parse(name).unwrap(), &bytes, &igneum_pow::generator::V3_ALLOWED) -} - -#[test] -fn per_load_bias_test_skips_the_era_window_bits() { - let mut accepted = 0usize; - let mut window_bit_rejections = 0usize; - for i in 0..8u32 { - let class = era_class("mx8+shl4096x1", i); - let label = format!("igneum-v6inv/{i}"); - for (_, verdict) in attempts_class(&label, label.as_bytes(), class) { - match verdict { - Ok(()) => accepted += 1, - Err(Reject::BiasedIndexBit { bit, ones, .. }) if bit >= 26 && (ones == 0 || ones == 16_384) => window_bit_rejections += 1, - Err(_) => {} - } - } - } - assert_eq!(window_bit_rejections, 0, "a window's fixed top bit was judged as biased"); - assert!(accepted >= 4, "the sound per-load form accepted on {accepted} of 8 seeds under drawn eras (0 before the fix)"); -} - -#[test] -fn per_load_bias_test_still_refuses_a_biased_real_bit_without_an_era() { - let class = LoadClass::parse("mx8+shl4096x1").unwrap(); - let label = "igneum-v6inv/3"; - let v = attempts_class(label, label.as_bytes(), class); - assert!(v.iter().all(|(_, r)| r.is_err()), "seed 3 exhausted the cap on the no-era census (0 of 32) and must still"); - assert!(v.iter().any(|(_, r)| matches!(r, Err(Reject::BiasedIndexBit { bit: 0, .. }))), "a bit-0 refusal on seed 3 stands"); -} diff --git a/proto-cuda/nvrtc/cuda_api.h b/proto-cuda/nvrtc/cuda_api.h index eb85ee370..e16e8c6e0 100644 --- a/proto-cuda/nvrtc/cuda_api.h +++ b/proto-cuda/nvrtc/cuda_api.h @@ -34,10 +34,6 @@ struct Drv { decltype(&cuOccupancyMaxActiveBlocksPerMultiprocessor) occupancy = nullptr; decltype(&cuGetErrorString) getErrorString = nullptr; decltype(&cuGetErrorName) getErrorName = nullptr; - // Counter ASIC 4.0 research (7 October 2026): the --microbench texture probes; optional (the probes that need - // them are skipped when the driver has no symbol, which no shipped driver lacks) - decltype(&cuTexObjectCreate) texObjectCreate = nullptr; - decltype(&cuTexObjectDestroy) texObjectDestroy = nullptr; }; struct Rtc { diff --git a/proto-cuda/nvrtc/emu/list-race-stubs.cpp b/proto-cuda/nvrtc/emu/list-race-stubs.cpp deleted file mode 100644 index 1d578721f..000000000 --- a/proto-cuda/nvrtc/emu/list-race-stubs.cpp +++ /dev/null @@ -1,5 +0,0 @@ -// emu/list-race-stubs.cpp (Counter ASIC 4.0 research, 8 October 2026): the two emulation entry points, so worker.cpp -// itself links as a Linux binary for the card-free --list-race check on the box (nothing opens a device there). -#include "cuda_api.h" -void emu_fill_driver(Drv&) {} -void emu_fill_nvrtc(Rtc&) {} diff --git a/proto-cuda/nvrtc/emu/variant-test.cpp b/proto-cuda/nvrtc/emu/variant-test.cpp deleted file mode 100644 index bbee93cda..000000000 --- a/proto-cuda/nvrtc/emu/variant-test.cpp +++ /dev/null @@ -1,51 +0,0 @@ -// emu/variant-test.cpp (Counter ASIC 4.0 research, 8 October 2026): the known-failed pair for the --bench --variant fix, -// on the host with no card. Includes the worker with its main renamed and reads the pure pieces: the bench's race -// switch, the variant lookup, the kernel-text rewrite on a real class v4 pack, and the launch shape. -// (1) --bench with no --variant: the race is off and the shape is nonces / block blocks ("variant base"); -// (2) --bench --variant sp43-w32: the race is on, the name resolves to 43 sparse blocks of 32 warps, the rewrite of the -// pack's bound kernel carries the `nonces` argument and the unit function, and the shape is 43 blocks. -// The run of 7 October (run-ca4-pc1-ca4sparse-5090-20261007) is the failed case this test reproduces: before the fix -// (1) and (2) read the same shape and the same "variant base". -#define main igneum_worker_main -#include "../worker.cpp" -#undef main -#include -// the emulation entry points worker.cpp declares under IGNEUM_EMU: this test never opens a device, so they are stubs -void emu_fill_driver(Drv&) {} -void emu_fill_nvrtc(Rtc&) {} - -int main(int argc, char** argv) { - if (argc < 2) { std::printf("usage: variant-test \n"); return 2; } - bool ok = true; - std::string text = readText(std::string(argv[1]) + "/kernel_bound.cu", ok); - if (!ok) { std::printf("FAIL: cannot read %s/kernel_bound.cu\n", argv[1]); return 2; } - std::string err; - std::string boundDev; - if (!deviceOnly(text, boundDev, err)) { std::printf("FAIL: device text: %s\n", err.c_str()); return 2; } - const uint32_t nonces = 1u << 24; - // (1) the base run - const bool raceOffBase = benchRaceOff(true, ""); - unsigned gridBase = launchGridBlocks(0, nonces, 32); - std::printf("RESULT variant-test case=base race_off=%d grid_blocks=%u block=32 variant=base\n", raceOffBase ? 1 : 0, gridBase); - // (2) the sparse run - std::vector all = allVariants(); - const Variant* v = findVariant(all, "sp43-w32"); - const bool raceOffSparse = benchRaceOff(true, "sp43-w32"); - std::string src, why; - bool rewritten = v && variantSource(boundDev, *v, v->blockWarps, src, why); - bool hasArg = rewritten && src.find("igneum_hash_bound(") != std::string::npos && src.find(", uint32_t nonces) {") != std::string::npos; - bool hasUnit = rewritten && src.find("igneum_hash_bound_unit(") != std::string::npos && src.find("gid += gridDim.x * blockDim.x") != std::string::npos; - unsigned gridSparse = v ? launchGridBlocks(v->sparseBlocks, nonces, 32u * (uint32_t)v->blockWarps) : 0; - std::printf("RESULT variant-test case=sp43-w32 race_off=%d resolved=%d sparse_blocks=%d block_warps=%d rewritten=%d nonces_arg=%d unit_fn=%d grid_blocks=%u block=%d why=\"%s\"\n", - raceOffSparse ? 1 : 0, v ? 1 : 0, v ? v->sparseBlocks : 0, v ? v->blockWarps : 0, rewritten ? 1 : 0, hasArg ? 1 : 0, hasUnit ? 1 : 0, gridSparse, v ? 32 * v->blockWarps : 0, why.c_str()); - // (3) the race's order the bench builds for the pinned variant (the 02:52Z fault: "variants 1 base only") - Tuning noTuning; - std::vector order = raceOrder(all, "sp43-w32", noTuning, "on"); - std::vector orderBase = raceOrder(all, "", noTuning, "off"); - bool orderOk = order.size() == 2 && order[0].name == "base" && order[1].name == "sp43-w32" && order[1].sparseBlocks == 43 && orderBase.size() == 1; - std::printf("RESULT variant-test case=race-order pinned=sp43-w32 variants=%zu names=%s%s base_only_variants=%zu\n", order.size(), order.size() > 0 ? order[0].name.c_str() : "", order.size() > 1 ? (std::string(",") + order[1].name).c_str() : "", orderBase.size()); - bool pass = raceOffBase && !raceOffSparse && v && v->sparseBlocks == 43 && v->blockWarps == 32 && rewritten && hasArg && hasUnit && gridBase == nonces / 32 && gridSparse == 43 && orderOk; - // the known-failed shape: a base run and a sparse run with the same grid is the 7 October fault - std::printf("RESULT variant-test %s: base grid %u blocks of 32 against sparse grid %u blocks of %d, race %s/%s\n", pass ? "PASS" : "FAIL", gridBase, gridSparse, v ? 32 * v->blockWarps : 0, raceOffBase ? "off" : "on", raceOffSparse ? "off" : "on"); - return pass ? 0 : 1; -} diff --git a/proto-cuda/nvrtc/worker.cpp b/proto-cuda/nvrtc/worker.cpp index 72ec2baa2..0439e4270 100644 --- a/proto-cuda/nvrtc/worker.cpp +++ b/proto-cuda/nvrtc/worker.cpp @@ -53,7 +53,6 @@ #include #include #include -#include #include #include #include @@ -178,8 +177,6 @@ static bool loadDriver(Drv& d, std::string& err, std::string& libName) { LOAD_SYM(d, occupancy, "cuOccupancyMaxActiveBlocksPerMultiprocessor"); LOAD_SYM(d, getErrorString, "cuGetErrorString"); LOAD_SYM(d, getErrorName, "cuGetErrorName"); - d.texObjectCreate = (decltype(d.texObjectCreate))libSym(lib, "cuTexObjectCreate"); // optional, --microbench only - d.texObjectDestroy = (decltype(d.texObjectDestroy))libSym(lib, "cuTexObjectDestroy"); if (!missing.empty()) { err = "the driver library lacks " + missing + " (driver too old; CUDA 11 or newer is needed)"; return false; } return true; #endif @@ -198,7 +195,6 @@ static std::vector nvrtcCandidates() { FindClose(h); } v.push_back("nvrtc64_120_0.dll"); - v.push_back("libnvrtc.so.12"); v.push_back("/usr/local/cuda-12.8/lib64/libnvrtc.so.12"); v.push_back("/usr/local/cuda/lib64/libnvrtc.so.12"); // the box's card-free --list-race check v.push_back("nvrtc64_130_0.dll"); if (const char* cp = std::getenv("CUDA_PATH")) { v.push_back(std::string(cp) + "\\bin\\nvrtc64_120_0.dll"); v.push_back(std::string(cp) + "\\bin\\nvrtc64_130_0.dll"); } return v; @@ -427,7 +423,6 @@ struct Pair { int regs = 0, blocksPerSM = 0; int blockWarps = 1; // threads per block = 32 x this (the winning variant's, else the worker's default) std::string variant = "base"; // the bound kernel in service: a variant name (see allVariants) - int sparseBlocks = 0; // the SM-sparse variants (sp): the persistent grid in blocks, 0 = the plain grid std::string raceLine; // the race's one-line report, emitted by the main thread with "prepared" double raceMs = 0; // read-width experiment (5 October 2026): the pack's load class and, for variant 5, the persistent-warp scratch @@ -486,17 +481,6 @@ static void releasePair(Ctx& c, Pair* p) { struct IgneumInitWordsArg { uint32_t w[8]; }; -// The launch shape of a non-persistent dispatch: grid blocks for `nonces` lanes at `block` threads per block, or the -// SM-sparse grid of `sparseBlocks` persistent blocks (Counter ASIC 4.0 research). Pure, so the emulation test can -// read it (emu/variant-test.cpp): the base shape is nonces / block, the sparse shape is the block count itself. -static unsigned launchGridBlocks(int sparseBlocks, uint32_t nonces, uint32_t block) { - return sparseBlocks > 0 ? (unsigned)sparseBlocks : (unsigned)(nonces / block); -} - -// Whether a --bench/--memprobe/--microbench run keeps the race off: yes unless --variant names a kernel to serve -// (8 October 2026: the SM-sparse run of 7 October served base on 48 rows because this read true with a --variant). -static bool benchRaceOff(bool bench, const std::string& pinned) { return bench && pinned.empty(); } - // `block` threads per block (32 x warps); `nonces` must be a multiple of it. static bool launchHash(Ctx& c, Pair* p, CUdeviceptr out, uint32_t baseNonce, const uint32_t iw[8], uint32_t nonces, uint32_t block, CUstream s, std::string& err) { uint32_t mask = p->words - 1u; @@ -514,17 +498,8 @@ static bool launchHash(Ctx& c, Pair* p, CUdeviceptr out, uint32_t baseNonce, con DRV_CHECK(c, c.drv.launchKernel(p->fHashBound, warps, 1, 1, 32, 1, 1, 0, s, args, nullptr), "cuLaunchKernel igneum_hash_bound (persistent)"); return true; } - if (p->sparseBlocks > 0) { - // The SM-sparse wrapper: a grid of sparseBlocks blocks, the trailing argument the dispatch's nonce count (after - // the hot table when the pack has one); a dispatch smaller than the grid leaves the surplus blocks idle. - if (nonces % block != 0u) { err = "nonces not a multiple of the block"; return false; } - void* args[7] = { &p->ds, &out, &baseNonce, &mask, &a, &nonces, nullptr }; - if (p->hot) { args[5] = &p->hot; args[6] = &nonces; } - DRV_CHECK(c, c.drv.launchKernel(p->fHashBound, launchGridBlocks(p->sparseBlocks, nonces, block), 1, 1, block, 1, 1, 0, s, args, nullptr), "cuLaunchKernel igneum_hash_bound (SM-sparse)"); - return true; - } void* args[6] = { &p->ds, &out, &baseNonce, &mask, &a, &p->hot }; - DRV_CHECK(c, c.drv.launchKernel(p->fHashBound, launchGridBlocks(0, nonces, block), 1, 1, block, 1, 1, 0, s, args, nullptr), "cuLaunchKernel igneum_hash_bound"); + DRV_CHECK(c, c.drv.launchKernel(p->fHashBound, nonces / block, 1, 1, block, 1, 1, 0, s, args, nullptr), "cuLaunchKernel igneum_hash_bound"); return true; } @@ -539,10 +514,6 @@ struct Variant { int maxrreg = 0; // 0: none; N: --maxrregcount=N (registers per thread, occupancy against spills) int blockWarps = 0; // 0: the worker's --block-warps; N: 32 x N threads per block int minBlocks = 0; // N > 0: __launch_bounds__(32 x blockWarps, N) (the compiler fits N blocks per SM) - int sparseBlocks = 0; // N > 0: the SM-sparse shape (Counter ASIC 4.0 research, 7 October 2026): a persistent grid of N - // blocks, every thread looping over the dispatch's nonces with stride gridDim.x * blockDim.x, so the - // hash runs on about N SMs at 32 warps per block and the other SMs idle; the kernel gains a - // trailing `uint32_t nonces` argument. Opt-in by name only (sp or sp-w), never in a race by default }; // The catalogue. Names are stable: the tuning file and the fleet records use them. "base" is the pack's text as @@ -570,36 +541,11 @@ static std::vector allVariants() { return v; } -// sp and sp-w: the SM-sparse variants, made on demand (not in the catalogue, so a default race never runs them): -// N persistent blocks (1 to 4096) of W warps (default 32, the largest block, one block per SM at the hash's register count). -static bool parseSparseVariant(const std::string& name, Variant& out) { - if (name.rfind("sp", 0) != 0) return false; - size_t i = 2, n = 0; int blocks = 0, warps = 32; - while (i < name.size() && name[i] >= '0' && name[i] <= '9') { blocks = blocks * 10 + (name[i] - '0'); ++i; ++n; } - if (n == 0 || blocks < 1 || blocks > 4096) return false; - if (i < name.size()) { - if (name.compare(i, 2, "-w") != 0) return false; - i += 2; n = 0; warps = 0; - while (i < name.size() && name[i] >= '0' && name[i] <= '9') { warps = warps * 10 + (name[i] - '0'); ++i; ++n; } - if (n == 0 || i != name.size() || warps < 1 || warps > 32) return false; - } - out = Variant(); out.name = name; out.blockWarps = warps; out.sparseBlocks = blocks; - return true; -} - static const Variant* findVariant(const std::vector& all, const std::string& name) { for (const Variant& v : all) if (v.name == name) return &v; - static std::vector made; // the on-demand sparse variants, kept so the pointer stays valid - Variant sp; - if (parseSparseVariant(name, sp)) { - for (const Variant& v : made) if (v.name == name) return &v; - made.reserve(64); - if (made.size() < 64) { made.push_back(sp); return &made.back(); } - } return nullptr; } - // The variant's source: the pack's bound-kernel text with the variant's rewrites. Every rewrite has an exact anchor // in the text igneum-pow emits; a text without the anchor refuses the variant (why), it is never guessed. static bool variantSource(const std::string& base, const Variant& v, int blockWarps, std::string& out, std::string& why) { @@ -630,47 +576,6 @@ static bool variantSource(const std::string& base, const Variant& v, int blockWa if (p == std::string::npos) { why = "no kernel declaration anchor"; return false; } out.replace(p, std::strlen(a), fmt("__global__ void __launch_bounds__(%d, %d) igneum_hash_bound(", 32 * blockWarps, v.minBlocks)); } - if (v.sparseBlocks > 0) { - // The SM-sparse shape: the emitted kernel becomes a per-thread unit function taking its gid, and a persistent - // wrapper of the kernel's name loops the unit over the dispatch's nonces. Anchors: the kernel declaration as - // igneum-pow emits it (no __launch_bounds__ on it: the two rewrites are not combined) and its first line. - if (v.minBlocks > 0) { why = "sp and __launch_bounds__ minBlocks are not combined"; return false; } - // line-ending-agnostic: a pack copied through Windows may carry CRLF; the anchors below are LF, so the text is - // normalised first (the rewritten kernel then has the same bytes from LF and CRLF input) - out.erase(std::remove(out.begin(), out.end(), '\r'), out.end()); - const char* a = "__global__ void igneum_hash_bound("; - size_t p = out.find(a); - if (p == std::string::npos) { why = "no kernel declaration anchor"; return false; } - size_t close = out.find(") {", p); - if (close == std::string::npos) { why = "no parameter list close"; return false; } - std::string params = out.substr(p + std::strlen(a), close - (p + std::strlen(a))); - if (params.find("scratch") != std::string::npos || params.find("units") != std::string::npos) { why = "a persistent (variant-5) pack has its own loop"; return false; } - // the argument names: the last token of each parameter - std::string args; size_t from = 0; - while (from <= params.size()) { - size_t comma = params.find(',', from); - std::string one = params.substr(from, comma == std::string::npos ? std::string::npos : comma - from); - size_t e = one.find_last_not_of(" \t"); - if (e == std::string::npos) { why = "an empty parameter"; return false; } - size_t b = one.find_last_of(" \t*&", e); - size_t start = b == std::string::npos ? 0 : b + 1; - std::string nm = one.substr(start, e - start + 1); // e is the last character's index: the length is e - start + 1 - // (the 03:23Z card run read `d, ou, baseNonc, mas, i`: the length was written e - start and dropped every name's last character) - if (nm.empty()) { why = "a parameter without a name"; return false; } - args += (args.empty() ? "" : ", ") + nm; - if (comma == std::string::npos) break; - from = comma + 1; - } - const char* gidLine = "\n uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x;\n"; - size_t g = out.find(gidLine, close); - if (g == std::string::npos) { why = "no gid line after the declaration"; return false; } - out.replace(g, std::strlen(gidLine), "\n"); - out.replace(p, close - p, "__device__ __forceinline__ void igneum_hash_bound_unit(" + params + ", uint32_t gid"); - out += fmt("\n// SM-sparse wrapper (variant %s): %d persistent blocks of %d threads over the dispatch's nonces\n", v.name.c_str(), v.sparseBlocks, 32 * blockWarps); - out += fmt("__global__ void __launch_bounds__(%d) igneum_hash_bound(", 32 * blockWarps) + params + ", uint32_t nonces) {\n"; - out += " for (uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; gid < nonces; gid += gridDim.x * blockDim.x)\n"; - out += " igneum_hash_bound_unit(" + args + ", gid);\n}\n"; - } return true; } @@ -693,24 +598,6 @@ static std::string jsonStringAfter(const std::string& t, size_t from, const char return e == std::string::npos ? "" : t.substr(q + 1, e - q - 1); } -// The race's order: base first, then the pinned or tuned candidates, then the rest (or the --race list only). Pure, so -// the emulation test reads it (emu/variant-test.cpp). Membership in `order` is by NAME: findVariant answers for the -// on-demand sp names whatever list it is asked about, which is the fault of the 8 October 02:52Z run (the pinned -// sparse variant was looked up in `order`, found by the on-demand path, and never pushed: "variants 1 base only"). -static std::vector raceOrder(const std::vector& all, const std::string& pinned, const Tuning& tu, const std::string& race) { - std::vector order; - auto has = [&](const std::string& n) { for (const Variant& v : order) if (v.name == n) return true; return false; }; - auto push = [&](const std::string& n) { const Variant* v = findVariant(all, n); if (v && !has(n)) order.push_back(*v); }; - push("base"); - if (!pinned.empty()) push(pinned); - else { - for (const std::string& n : tu.candidates) push(n); - if (race != "on" && race != "off") { std::string rest = race; size_t i = 0; while (i <= rest.size()) { size_t j = rest.find(',', i); if (j == std::string::npos) j = rest.size(); if (j > i) push(rest.substr(i, j - i)); i = j + 1; } } - else if (race == "on") for (const Variant& v : all) push(v.name); - } - return order; -} - static Tuning readTuning(const std::string& text, const std::string& device) { Tuning tu; if (text.empty()) return tu; @@ -756,9 +643,7 @@ struct RaceEntry { static bool raceTime(Ctx& c, Pair* p, const PfPack& pk, RaceEntry& e, CUdeviceptr dOut, uint32_t batch, int benchMs, CUstream s, bool selfTest) { std::lock_guard hold(gpuMutex); CUfunction keep = p->fHashBound; - int keepSparse = p->sparseBlocks; p->fHashBound = e.fn; - p->sparseBlocks = e.v.sparseBlocks; std::string err; uint32_t block = 32u * (uint32_t)e.blockWarps; bool ok = true; @@ -788,7 +673,6 @@ static bool raceTime(Ctx& c, Pair* p, const PfPack& pk, RaceEntry& e, CUdevicept } } p->fHashBound = keep; - p->sparseBlocks = keepSparse; return ok; } @@ -800,7 +684,16 @@ static void racePair(Ctx& c, Pair* p, const PfPack& pk, const std::string& bound std::vector all = allVariants(); Tuning tu = readTuning(c.tuning, c.name); std::string pinned = !c.pinned.empty() ? c.pinned : (tu.found && !tu.race ? tu.variant : ""); - std::vector order = raceOrder(all, pinned, tu, c.race); + // the order: base first, then the pinned or tuned candidates, then the rest (or the --race list only) + std::vector order; + auto push = [&](const std::string& n) { const Variant* v = findVariant(all, n); if (v && !findVariant(order, n)) order.push_back(*v); }; + push("base"); + if (!pinned.empty()) push(pinned); + else { + for (const std::string& n : tu.candidates) push(n); + if (c.race != "on" && c.race != "off") { std::string rest = c.race; size_t i = 0; while (i <= rest.size()) { size_t j = rest.find(',', i); if (j == std::string::npos) j = rest.size(); if (j > i) push(rest.substr(i, j - i)); i = j + 1; } } + else if (c.race == "on") for (const Variant& v : all) push(v.name); + } #ifdef IGNEUM_EMU order.resize(1); // the stand-in checks that the handed-over text is the pack's; no rewrites under emulation #endif @@ -888,10 +781,9 @@ static void racePair(Ctx& c, Pair* p, const PfPack& pk, const std::string& bound RaceEntry& w = entries[win]; c.drv.moduleUnload(p->modBound); p->modBound = w.mod; p->fHashBound = w.fn; p->regs = w.regs; p->blocksPerSM = w.blocksPerSM; p->blockWarps = w.blockWarps; p->variant = w.v.name; - p->sparseBlocks = w.v.sparseBlocks; w.mod = nullptr; } else { - p->blockWarps = c.blockWarps; p->variant = "base"; p->sparseBlocks = 0; + p->blockWarps = c.blockWarps; p->variant = "base"; } for (size_t i = 1; i < entries.size(); ++i) if (entries[i].mod) c.drv.moduleUnload(entries[i].mod); p->raceMs = wallMs() - t0; @@ -1108,10 +1000,6 @@ static void prepareRun(Ctx* c, PrepareTask* t) { struct Options { bool serve = false, check = false, raceOnly = false; bool bench = false, memprobe = false; // read-width experiment (5 October 2026) - bool microbench = false; // Counter ASIC 4.0 research (7 October 2026): one kernel per GPU hardware block, sustained - int mbSeconds = 60; // --mb-seconds: the sustained window per probe (the power sampler reads it) - std::string mbOnly; // --mb-only a,b,c: these probes only - bool listRace = false; // --list-race: print the race order --bench would run (card-free), then exit int batches = 5, warps = 0, probeMib = 0; int device = 0, batchLog2 = 22, blockWarps = 1; std::string pack, arch = "auto"; @@ -1131,11 +1019,6 @@ static void usage() { " print the 2^B fingerprint at base nonce 0 (one RESULT line); a variant-5 pack runs --warps persistent warps\n" " --memprobe [--probe-mib N] no pack: dependent random 4, 16 and 64-byte reads, independent reads, a coalesced stream and an\n" " integer chain at 4, 64 and 1024 MiB (the same table as igneum-worker-opencl --memprobe)\n" - " --list-race [--pack ] [--variant ] [--race ...] card-free: print the race order the run would build (variants N, names) and,\n" - " with a pack, whether the named variant's kernel rewrite applies to its bound kernel; then exit 0, or 1 when a named variant is missing\n" - " --microbench [--mb-seconds 60] [--mb-only a,b] no pack: one sustained kernel per GPU hardware block (ALU families, shuffle,\n" - " byte permute, FP32 and FP16 FMA, tensor tiles per precision, L2-resident chases, texture fetches, a DRAM\n" - " chase and a sleeping-SM floor), each for --mb-seconds with UTC start and end stamps for a power sampler\n" " --batches N --bench: timed dispatches (default 5)\n" " --warps N --bench on a variant-5 pack: persistent warps (default: the occupancy capacity, rounded down to a power of two)\n" " --race --pack the variant race alone (3 rounds): one line per variant, the race line, exit 0 or 1\n" @@ -1143,8 +1026,7 @@ static void usage() { " --race-bench-ms N timed window per variant (default 2000)\n" " --race-budget-s N a race stops compiling and timing after this (default 120; base is kept)\n" " --race-rounds N interleaved rounds, best per variant (default 1 in --serve, 3 in --race)\n" - " --variant use this variant without a race (also from the tuning file); sp or sp-w is the\n" - " SM-sparse shape (N persistent blocks of W warps, default 32; Counter ASIC 4.0 research)\n" + " --variant use this variant without a race (also from the tuning file)\n" " --tuning the per-card tuning file (default: IGNEUM_TUNING_FILE from the environment)\n", WORKER_VERSION); } @@ -1157,10 +1039,6 @@ static Options parseArgs(int argc, char** argv) { else if (a == "--check") o.check = true; else if (a == "--bench") o.bench = true; else if (a == "--memprobe") o.memprobe = true; - else if (a == "--microbench") o.microbench = true; - else if (a == "--mb-seconds") o.mbSeconds = std::atoi(next().c_str()); - else if (a == "--mb-only") o.mbOnly = next(); - else if (a == "--list-race") o.listRace = true; else if (a == "--batches") o.batches = std::atoi(next().c_str()); else if (a == "--warps") o.warps = std::atoi(next().c_str()); else if (a == "--probe-mib") o.probeMib = std::atoi(next().c_str()); @@ -1182,13 +1060,12 @@ static Options parseArgs(int argc, char** argv) { } if (o.batchLog2 < 10 || o.batchLog2 > 28) { std::printf("--batch-log2 must be between 10 and 28\n"); std::exit(2); } if (o.blockWarps < 1 || o.blockWarps > 32) { std::printf("--block-warps must be between 1 and 32\n"); std::exit(2); } - if (!o.serve && !o.check && !o.raceOnly && !o.bench && !o.memprobe && !o.microbench && !o.listRace) { usage(); std::exit(2); } - if (o.mbSeconds < 5 || o.mbSeconds > 600) { std::printf("--mb-seconds must be between 5 and 600\n"); std::exit(2); } + if (!o.serve && !o.check && !o.raceOnly && !o.bench && !o.memprobe) { usage(); std::exit(2); } if (o.raceBenchMs < 200 || o.raceBenchMs > 20000) { std::printf("--race-bench-ms must be between 200 and 20000\n"); std::exit(2); } if (o.raceBudgetS < 5 || o.raceBudgetS > 540) { std::printf("--race-budget-s must be between 5 and 540 (the prepare lead is 600 DAA)\n"); std::exit(2); } if (o.raceRounds == 0) o.raceRounds = o.raceOnly ? 3 : 1; if (o.tuningPath.empty()) if (const char* t = std::getenv("IGNEUM_TUNING_FILE")) o.tuningPath = t; - if (o.pack.empty() && !o.memprobe && !o.microbench && !o.listRace) { std::printf("--pack is required (igneum-miner export-pack writes one)\n"); std::exit(2); } + if (o.pack.empty() && !o.memprobe) { std::printf("--pack is required (igneum-miner export-pack writes one)\n"); std::exit(2); } while (o.pack.size() > 1 && (o.pack.back() == '/' || o.pack.back() == '\\')) o.pack.pop_back(); return o; } @@ -1410,7 +1287,7 @@ static uint64_t fnv1a64Bytes(const void* p, size_t n) { // cuStreamSynchronize (the driver API path loads no event symbols; a 2^24 dispatch is 60 to 900 ms on the cards here, // so the launch overhead is under 1 percent). static int runBench(Ctx& c, const Options& o, Pair* p) { - uint32_t nonces = 1u << o.batchLog2, block = 32u * (uint32_t)p->blockWarps; + uint32_t nonces = 1u << o.batchLog2, block = 32u * (uint32_t)c.blockWarps; if (p->persistent) { uint32_t unit = 32u * (uint32_t)p->warps; nonces = (nonces / unit) * unit; if (nonces == 0) nonces = unit; } CUdeviceptr dOut = 0; std::string err; @@ -1433,11 +1310,10 @@ static int runBench(Ctx& c, const Options& o, Pair* p) { c.drv.memFree(dOut); std::string dev = c.name; for (char& ch : dev) if (ch == ' ') ch = '_'; std::printf("warm-up dispatch (base 0): %.2f ms; %d timed dispatches of %u nonces: mean %.2f ms\n", warm, o.batches, nonces, sum / o.batches); - std::printf("RESULT pack=%s class=%s device=%s arch=%s regs=%d blocks_per_sm=%d warps=%d resident=%d arena_mib=%llu hot_mib=%u hot_slots=%u hot_fill_ms=%.2f nonces=%u batches=%d check=%s fingerprint=%016llx mhs=%.3f loads=%u bytes=%u scratch_ops=%u variant=%s sparse_blocks=%d block_warps=%d time=wall\n", + std::printf("RESULT pack=%s class=%s device=%s arch=%s regs=%d blocks_per_sm=%d warps=%d resident=%d arena_mib=%llu hot_mib=%u hot_slots=%u hot_fill_ms=%.2f nonces=%u batches=%d check=%s fingerprint=%016llx mhs=%.3f loads=%u bytes=%u scratch_ops=%u time=wall\n", p->dir.c_str(), p->loadClass.c_str(), dev.c_str(), c.archOpt.c_str(), p->regs, p->blocksPerSM, p->warps, p->residentWarps, (unsigned long long)(p->scratchBytes >> 20), p->hotMb, p->hotSlots, p->hotMs, nonces, o.batches, p->checked ? (p->checkPass ? "PASS" : "FAIL") : "skipped", (unsigned long long)fp, (double)nonces * (double)o.batches / (sum / 1000.0) / 1e6, - p->loadsPerHash, p->bytesPerHash, p->scratchOps * 8u, p->variant.c_str(), p->sparseBlocks, p->blockWarps); - if (!o.pinned.empty() && p->variant != o.pinned) std::printf("RESULT variant_not_installed requested=%s served=%s race=\"%s\"\n", o.pinned.c_str(), p->variant.c_str(), p->raceLine.c_str()); + p->loadsPerHash, p->bytesPerHash, p->scratchOps * 8u); return 0; } @@ -1584,267 +1460,14 @@ static int runMemprobe(Ctx& c, const Options& o) { return 0; } - -// --------------------------------------------------------------------------------------------- -// Counter ASIC 4.0 research, 7 October 2026: --microbench. One kernel per GPU hardware block that gaming and AI -// already paid for, each run sustained for --mb-seconds at full residency so a 1 Hz power sampler reads its watts, with -// the counted operations per second beside it, so the energy per operation on this card is (watts minus the sleeping -// floor) over operations per second. Every probe prints one RESULT line with UTC start and end stamps and a checksum of -// its outputs. Probes compile one at a time, so a form the card or NVRTC refuses (an FP8 tile on a card under sm_89, -// a texture the driver cannot bind) drops that probe alone with its error on the row. Nothing here touches a pack. - -struct MicroProbe { - const char* name; // the row's name - const char* unit; // what one counted operation is - double opsPerStep; // counted operations per lane per step - uint32_t steps; // steps per launch (sized so one launch is 50 ms to 2 s on a 5090, approximate) - int tableMib; // a table of uint32 the kernel reads (0 = none); filled by kb_fill mode 0 (or 1 = floats in [0,1)) - int tableMode; - int tex; // 0 none; 1 a point-sampled u32 texture over the table; 2 a linearly filtered float texture over it - const char* body; // the kernel body: has steps, seed, out, tbl, mask, tex, g (the lane), x0..x3 (mixed seeds) -}; - -static const char* MICRO_PRELUDE = - "#include \n" - "__device__ __forceinline__ uint32_t kb_mix(uint32_t x) { x ^= x >> 16; x *= 0x7feb352du; x ^= x >> 15; x *= 0x846ca68bu; x ^= x >> 16; return x; }\n" - "__device__ __forceinline__ uint32_t kb_rotl(uint32_t x, uint32_t r) { return (x << r) | (x >> (32u - r)); }\n" - "extern \"C\" __global__ void kb_fill(uint32_t* t, uint32_t n, uint32_t mode) { uint32_t i = blockIdx.x * blockDim.x + threadIdx.x; if (i >= n) return;\n" - " uint32_t v = kb_mix(i ^ 0x9E3779B9u); if (mode == 1u) { float f = (float)(v >> 8) * (1.0f / 16777216.0f); t[i] = __float_as_uint(f); } else t[i] = v; }\n" - "#define KB_KERNEL(NAME) extern \"C\" __global__ void NAME(uint32_t steps, uint32_t seed, uint32_t* out, const uint32_t* tbl, uint32_t mask, unsigned long long tex) {\\\n" - " uint32_t g = blockIdx.x * blockDim.x + threadIdx.x; uint32_t x0 = kb_mix(g * 4u ^ seed), x1 = kb_mix((g * 4u + 1u) ^ seed), x2 = kb_mix((g * 4u + 2u) ^ seed), x3 = kb_mix((g * 4u + 3u) ^ seed);\n"; - -// Tensor tiles: the dependency carries the accumulator back into the A fragment so the chain is real; two tiles per step for issue overlap. -#define KB_MMA2(INSTR, NA, NB, NC, ATY) \ - " uint32_t a0 = x0, a1 = x1, a2 = x2, a3 = x3, b0 = x1 ^ 0x5bd1e995u, b1 = x2 * 0x27d4eb2fu; " ATY " c0 = 0, c1 = 0, c2 = 0, c3 = 0, e0 = 0, e1 = 0, e2 = 0, e3 = 0;\n" \ - " for (uint32_t s = 0u; s < steps; ++s) {\n" \ - " asm volatile(\"" INSTR "\" : \"=r\"(c0), \"=r\"(c1), \"=r\"(c2), \"=r\"(c3) : \"r\"(a0), \"r\"(a1), \"r\"(a2), \"r\"(a3), \"r\"(b0), \"r\"(b1), \"r\"(c0), \"r\"(c1), \"r\"(c2), \"r\"(c3));\n" \ - " asm volatile(\"" INSTR "\" : \"=r\"(e0), \"=r\"(e1), \"=r\"(e2), \"=r\"(e3) : \"r\"(a1), \"r\"(a2), \"r\"(a3), \"r\"(a0), \"r\"(b1), \"r\"(b0), \"r\"(e0), \"r\"(e1), \"r\"(e2), \"r\"(e3));\n" \ - " a0 ^= (uint32_t)c0 + s; a1 ^= (uint32_t)e1; a2 += (uint32_t)c2; a3 ^= (uint32_t)e3 * 0x9E3779B1u;\n" \ - " }\n" \ - " out[g] = (uint32_t)c0 ^ (uint32_t)c1 ^ (uint32_t)c2 ^ (uint32_t)c3 ^ (uint32_t)e0 ^ (uint32_t)e1 ^ (uint32_t)e2 ^ (uint32_t)e3;\n}\n" - -static const MicroProbe MICRO_PROBES[] = { - { "sleep", "none (the SM-resident floor: full occupancy, __nanosleep, no issue)", 0, 2000, 0, 0, 0, - " for (uint32_t s = 0u; s < steps; ++s) { __nanosleep(1000); x0 += s; }\n out[g] = x0;\n}\n" }, - { "int_arx", "int32 add, xor or rotate (4 independent chains, 3 ops each per step)", 12, 16384, 0, 0, 0, - " for (uint32_t s = 0u; s < steps; ++s) { x0 = kb_rotl(x0 + x1, 7u) ^ s; x1 = kb_rotl(x1 + x2, 13u) ^ x0; x2 = kb_rotl(x2 + x3, 17u) ^ x1; x3 = kb_rotl(x3 + x0, 23u) ^ x2; }\n out[g] = x0 ^ x1 ^ x2 ^ x3;\n}\n" }, - { "int_mul", "int32 multiply-add (4 independent chains)", 4, 16384, 0, 0, 0, - " for (uint32_t s = 0u; s < steps; ++s) { x0 = x0 * 0x9E3779B1u + s; x1 = x1 * 0x85EBCA77u + x0; x2 = x2 * 0xC2B2AE3Du + x1; x3 = x3 * 0x27D4EB2Fu + x2; }\n out[g] = x0 ^ x1 ^ x2 ^ x3;\n}\n" }, - { "int_mulhi", "int32 high multiply (4 independent chains)", 4, 16384, 0, 0, 0, - " for (uint32_t s = 0u; s < steps; ++s) { x0 = __umulhi(x0, 0x9E3779B1u) + s; x1 = __umulhi(x1, x0 | 1u) ^ x1; x2 = __umulhi(x2, 0xC2B2AE3Du) + x1; x3 = __umulhi(x3, x2 | 1u) ^ x3; }\n out[g] = x0 ^ x1 ^ x2 ^ x3;\n}\n" }, - { "prmt", "byte permute, __byte_perm (4 independent chains)", 4, 16384, 0, 0, 0, - " for (uint32_t s = 0u; s < steps; ++s) { x0 = __byte_perm(x0, x1, 0x2103u ^ (s & 7u)); x1 = __byte_perm(x1, x2, 0x3012u); x2 = __byte_perm(x2, x3, 0x1230u); x3 = __byte_perm(x3, x0, 0x0321u) + s; }\n out[g] = x0 ^ x1 ^ x2 ^ x3;\n}\n" }, - { "lop3", "three-input logic (and, or, xor fused to one LOP3; 4 independent chains)", 4, 16384, 0, 0, 0, - " for (uint32_t s = 0u; s < steps; ++s) { x0 = (x0 & x1) ^ (x2 | s); x1 = (x1 & x2) ^ (x3 | x0); x2 = (x2 & x3) ^ (x0 | x1); x3 = (x3 & x0) ^ (x1 | x2); }\n out[g] = x0 ^ x1 ^ x2 ^ x3;\n}\n" }, - { "shfl", "warp shuffle, __shfl_xor_sync (4 independent chains, one xor each beside it)", 4, 8192, 0, 0, 0, - " for (uint32_t s = 0u; s < steps; ++s) { x0 = __shfl_xor_sync(0xffffffffu, x0, (int)((s & 31u) | 1u)) ^ x1; x1 = __shfl_xor_sync(0xffffffffu, x1, 2) ^ x2; x2 = __shfl_xor_sync(0xffffffffu, x2, 4) ^ x3; x3 = __shfl_xor_sync(0xffffffffu, x3, 8) ^ x0; }\n out[g] = x0 ^ x1 ^ x2 ^ x3;\n}\n" }, - { "fp32_fma", "FP32 fused multiply-add (4 independent chains)", 4, 16384, 0, 0, 0, - " float f0 = __uint_as_float((x0 & 0x007fffffu) | 0x3f800000u), f1 = __uint_as_float((x1 & 0x007fffffu) | 0x3f800000u), f2 = __uint_as_float((x2 & 0x007fffffu) | 0x3f800000u), f3 = __uint_as_float((x3 & 0x007fffffu) | 0x3f800000u);\n" - " for (uint32_t s = 0u; s < steps; ++s) { f0 = fmaf(f0, 0.999f, 0.5f); f1 = fmaf(f1, 1.001f, -0.5f); f2 = fmaf(f2, 0.998f, 0.25f); f3 = fmaf(f3, 1.002f, -0.25f); }\n out[g] = __float_as_uint(f0) ^ __float_as_uint(f1) ^ __float_as_uint(f2) ^ __float_as_uint(f3);\n}\n" }, - { "fp16x2_fma", "FP16 fused multiply-add, two per fma.rn.f16x2 (4 independent chains)", 8, 16384, 0, 0, 0, - " uint32_t h0 = (x0 & 0x03ff03ffu) | 0x3c003c00u, h1 = (x1 & 0x03ff03ffu) | 0x3c003c00u, h2 = (x2 & 0x03ff03ffu) | 0x3c003c00u, h3 = (x3 & 0x03ff03ffu) | 0x3c003c00u;\n" - " const uint32_t m = 0x3bff3bffu, a = 0x38003800u;\n" - " for (uint32_t s = 0u; s < steps; ++s) { asm volatile(\"fma.rn.f16x2 %0, %0, %1, %2;\" : \"+r\"(h0) : \"r\"(m), \"r\"(a)); asm volatile(\"fma.rn.f16x2 %0, %0, %1, %2;\" : \"+r\"(h1) : \"r\"(m), \"r\"(a)); asm volatile(\"fma.rn.f16x2 %0, %0, %1, %2;\" : \"+r\"(h2) : \"r\"(m), \"r\"(a)); asm volatile(\"fma.rn.f16x2 %0, %0, %1, %2;\" : \"+r\"(h3) : \"r\"(m), \"r\"(a)); }\n" - " out[g] = h0 ^ h1 ^ h2 ^ h3;\n}\n" }, - { "mma_u8_m8n8k16", "int8 multiply-add inside mma.sync m8n8k16 u8 (1,024 per tile per warp, 32 per lane; two tiles per step)", 64, 8192, 0, 0, 0, - " uint32_t a0 = x0, b0 = x1, c0 = 0, c1 = 0, a1 = x2, b1 = x3, e0 = 0, e1 = 0;\n" - " for (uint32_t s = 0u; s < steps; ++s) {\n" - " asm volatile(\"mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};\" : \"=r\"(c0), \"=r\"(c1) : \"r\"(a0), \"r\"(b0), \"r\"(c0), \"r\"(c1));\n" - " asm volatile(\"mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};\" : \"=r\"(e0), \"=r\"(e1) : \"r\"(a1), \"r\"(b1), \"r\"(e0), \"r\"(e1));\n" - " a0 ^= c0 + s; b0 = kb_rotl(b0, 7u) ^ c1; a1 ^= e1; b1 = kb_rotl(b1, 5u) ^ e0;\n }\n out[g] = c0 ^ c1 ^ e0 ^ e1;\n}\n" }, - { "mma_s8_m16n8k32", "int8 multiply-add inside mma.sync m16n8k32 s8 (4,096 per tile per warp, 128 per lane; two tiles per step)", 256, 4096, 0, 0, 0, - KB_MMA2("mma.sync.aligned.m16n8k32.row.col.s32.s8.s8.s32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};", 4, 2, 4, "int32_t") }, - { "mma_f16_m16n8k16", "FP16 multiply-add with FP32 accumulate inside mma.sync m16n8k16 (2,048 per tile per warp, 64 per lane; two tiles per step)", 128, 4096, 0, 0, 0, - " uint32_t a0 = (x0 & 0x03ff03ffu) | 0x3c003c00u, a1 = (x1 & 0x03ff03ffu) | 0x3c003c00u, a2 = (x2 & 0x03ff03ffu) | 0x3c003c00u, a3 = (x3 & 0x03ff03ffu) | 0x3c003c00u, b0 = 0x3c003c00u ^ (x1 & 0x00ff00ffu), b1 = 0x3c003c00u ^ (x2 & 0x00ff00ffu);\n" - " float c0 = 0.f, c1 = 0.f, c2 = 0.f, c3 = 0.f, e0 = 0.f, e1 = 0.f, e2 = 0.f, e3 = 0.f;\n" - " for (uint32_t s = 0u; s < steps; ++s) {\n" - " asm volatile(\"mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\" : \"=f\"(c0), \"=f\"(c1), \"=f\"(c2), \"=f\"(c3) : \"r\"(a0), \"r\"(a1), \"r\"(a2), \"r\"(a3), \"r\"(b0), \"r\"(b1), \"f\"(c0), \"f\"(c1), \"f\"(c2), \"f\"(c3));\n" - " asm volatile(\"mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\" : \"=f\"(e0), \"=f\"(e1), \"=f\"(e2), \"=f\"(e3) : \"r\"(a1), \"r\"(a2), \"r\"(a3), \"r\"(a0), \"r\"(b1), \"r\"(b0), \"f\"(e0), \"f\"(e1), \"f\"(e2), \"f\"(e3));\n" - " a0 = ((a0 ^ __float_as_uint(c0)) & 0x03ff03ffu) | 0x3c003c00u; a2 = ((a2 ^ __float_as_uint(e2)) & 0x03ff03ffu) | 0x3c003c00u; c0 *= 0.5f; c1 *= 0.5f; c2 *= 0.5f; c3 *= 0.5f; e0 *= 0.5f; e1 *= 0.5f; e2 *= 0.5f; e3 *= 0.5f;\n }\n" - " out[g] = __float_as_uint(c0) ^ __float_as_uint(c1) ^ __float_as_uint(c2) ^ __float_as_uint(c3) ^ __float_as_uint(e0) ^ __float_as_uint(e1) ^ __float_as_uint(e2) ^ __float_as_uint(e3);\n}\n" }, - { "mma_bf16_m16n8k16", "BF16 multiply-add with FP32 accumulate inside mma.sync m16n8k16 (64 per lane; two tiles per step)", 128, 4096, 0, 0, 0, - " uint32_t a0 = (x0 & 0x007f007fu) | 0x3f803f80u, a1 = (x1 & 0x007f007fu) | 0x3f803f80u, a2 = (x2 & 0x007f007fu) | 0x3f803f80u, a3 = (x3 & 0x007f007fu) | 0x3f803f80u, b0 = 0x3f803f80u ^ (x1 & 0x003f003fu), b1 = 0x3f803f80u ^ (x2 & 0x003f003fu);\n" - " float c0 = 0.f, c1 = 0.f, c2 = 0.f, c3 = 0.f, e0 = 0.f, e1 = 0.f, e2 = 0.f, e3 = 0.f;\n" - " for (uint32_t s = 0u; s < steps; ++s) {\n" - " asm volatile(\"mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\" : \"=f\"(c0), \"=f\"(c1), \"=f\"(c2), \"=f\"(c3) : \"r\"(a0), \"r\"(a1), \"r\"(a2), \"r\"(a3), \"r\"(b0), \"r\"(b1), \"f\"(c0), \"f\"(c1), \"f\"(c2), \"f\"(c3));\n" - " asm volatile(\"mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\" : \"=f\"(e0), \"=f\"(e1), \"=f\"(e2), \"=f\"(e3) : \"r\"(a1), \"r\"(a2), \"r\"(a3), \"r\"(a0), \"r\"(b1), \"r\"(b0), \"f\"(e0), \"f\"(e1), \"f\"(e2), \"f\"(e3));\n" - " a0 = ((a0 ^ __float_as_uint(c0)) & 0x007f007fu) | 0x3f803f80u; a2 = ((a2 ^ __float_as_uint(e2)) & 0x007f007fu) | 0x3f803f80u; c0 *= 0.5f; c1 *= 0.5f; c2 *= 0.5f; c3 *= 0.5f; e0 *= 0.5f; e1 *= 0.5f; e2 *= 0.5f; e3 *= 0.5f;\n }\n" - " out[g] = __float_as_uint(c0) ^ __float_as_uint(c1) ^ __float_as_uint(c2) ^ __float_as_uint(c3) ^ __float_as_uint(e0) ^ __float_as_uint(e1) ^ __float_as_uint(e2) ^ __float_as_uint(e3);\n}\n" }, - { "mma_e4m3_m16n8k32", "FP8 e4m3 multiply-add with FP32 accumulate inside mma.sync m16n8k32 (4,096 per tile per warp, 128 per lane; two tiles per step; sm_89 and later)", 256, 4096, 0, 0, 0, - " uint32_t a0 = x0 & 0x7f7f7f7fu, a1 = x1 & 0x7f7f7f7fu, a2 = x2 & 0x7f7f7f7fu, a3 = x3 & 0x7f7f7f7fu, b0 = (x1 >> 1) & 0x3f3f3f3fu, b1 = (x2 >> 1) & 0x3f3f3f3fu;\n" - " float c0 = 0.f, c1 = 0.f, c2 = 0.f, c3 = 0.f, e0 = 0.f, e1 = 0.f, e2 = 0.f, e3 = 0.f;\n" - " for (uint32_t s = 0u; s < steps; ++s) {\n" - " asm volatile(\"mma.sync.aligned.m16n8k32.row.col.f32.e4m3.e4m3.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\" : \"=f\"(c0), \"=f\"(c1), \"=f\"(c2), \"=f\"(c3) : \"r\"(a0), \"r\"(a1), \"r\"(a2), \"r\"(a3), \"r\"(b0), \"r\"(b1), \"f\"(c0), \"f\"(c1), \"f\"(c2), \"f\"(c3));\n" - " asm volatile(\"mma.sync.aligned.m16n8k32.row.col.f32.e4m3.e4m3.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\" : \"=f\"(e0), \"=f\"(e1), \"=f\"(e2), \"=f\"(e3) : \"r\"(a1), \"r\"(a2), \"r\"(a3), \"r\"(a0), \"r\"(b1), \"r\"(b0), \"f\"(e0), \"f\"(e1), \"f\"(e2), \"f\"(e3));\n" - " a0 = (a0 ^ __float_as_uint(c0)) & 0x7f7f7f7fu; a2 = (a2 ^ __float_as_uint(e2)) & 0x7f7f7f7fu; c0 *= 0.5f; c1 *= 0.5f; c2 *= 0.5f; c3 *= 0.5f; e0 *= 0.5f; e1 *= 0.5f; e2 *= 0.5f; e3 *= 0.5f;\n }\n" - " out[g] = __float_as_uint(c0) ^ __float_as_uint(c1) ^ __float_as_uint(c2) ^ __float_as_uint(c3) ^ __float_as_uint(e0) ^ __float_as_uint(e1) ^ __float_as_uint(e2) ^ __float_as_uint(e3);\n}\n" }, - { "l2_chase_32m", "dependent random 4-byte read in a 32 MiB table (L2-resident on a 5090's 96 MB; one chain per lane)", 1, 2048, 32, 0, 0, - " for (uint32_t s = 0u; s < steps; ++s) x0 = tbl[x0 & mask] ^ (x0 * 0x9E3779B1u + s);\n out[g] = x0;\n}\n" }, - { "l2_chase_64m", "dependent random 4-byte read in a 64 MiB table (inside a 5090's L2, outside a 4070's 36 MB)", 1, 2048, 64, 0, 0, - " for (uint32_t s = 0u; s < steps; ++s) x0 = tbl[x0 & mask] ^ (x0 * 0x9E3779B1u + s);\n out[g] = x0;\n}\n" }, - { "l2_indep4_32m", "random 4-byte read in a 32 MiB table, 4 independent chains per lane (L2 throughput)", 4, 2048, 32, 0, 0, - " for (uint32_t s = 0u; s < steps; ++s) { x0 = tbl[x0 & mask] ^ (x0 * 0x9E3779B1u + s); x1 = tbl[x1 & mask] ^ (x1 * 0x9E3779B1u + s); x2 = tbl[x2 & mask] ^ (x2 * 0x9E3779B1u + s); x3 = tbl[x3 & mask] ^ (x3 * 0x9E3779B1u + s); }\n out[g] = x0 ^ x1 ^ x2 ^ x3;\n}\n" }, - { "dram_chase_1g", "dependent random 4-byte read in a 1 GiB table (the hash's own pattern, the control)", 1, 512, 1024, 0, 0, - " for (uint32_t s = 0u; s < steps; ++s) x0 = tbl[x0 & mask] ^ (x0 * 0x9E3779B1u + s);\n out[g] = x0;\n}\n" }, - { "tex_point_u32_32m", "texture fetch, point sampled, tex.1d.v4.u32.s32 over a 32 MiB u32 table (the sampler's address path; dependent chain)", 1, 2048, 32, 0, 1, - " for (uint32_t s = 0u; s < steps; ++s) { int cx = (int)(x0 & mask); uint32_t t0, t1, t2, t3; asm volatile(\"tex.1d.v4.u32.s32 {%0,%1,%2,%3}, [%4, {%5}];\" : \"=r\"(t0), \"=r\"(t1), \"=r\"(t2), \"=r\"(t3) : \"l\"(tex), \"r\"(cx)); x0 = t0 ^ (x0 * 0x9E3779B1u + s); }\n out[g] = x0;\n}\n" }, - { "tex_linear_f32_256k", "texture fetch, linearly filtered, tex.1d.v4.f32.f32 over a 1 MiB float table (the sampler's interpolation; 4 independent chains)", 4, 4096, 1, 1, 2, - " float p0 = (float)(x0 & 0xffffu), p1 = (float)(x1 & 0xffffu), p2 = (float)(x2 & 0xffffu), p3 = (float)(x3 & 0xffffu);\n" - " for (uint32_t s = 0u; s < steps; ++s) { float t0, t1, t2, t3, u0, u1, u2, u3, v0, v1, v2, v3, w0, w1, w2, w3;\n" - " asm volatile(\"tex.1d.v4.f32.f32 {%0,%1,%2,%3}, [%4, {%5}];\" : \"=f\"(t0), \"=f\"(t1), \"=f\"(t2), \"=f\"(t3) : \"l\"(tex), \"f\"(p0));\n" - " asm volatile(\"tex.1d.v4.f32.f32 {%0,%1,%2,%3}, [%4, {%5}];\" : \"=f\"(u0), \"=f\"(u1), \"=f\"(u2), \"=f\"(u3) : \"l\"(tex), \"f\"(p1));\n" - " asm volatile(\"tex.1d.v4.f32.f32 {%0,%1,%2,%3}, [%4, {%5}];\" : \"=f\"(v0), \"=f\"(v1), \"=f\"(v2), \"=f\"(v3) : \"l\"(tex), \"f\"(p2));\n" - " asm volatile(\"tex.1d.v4.f32.f32 {%0,%1,%2,%3}, [%4, {%5}];\" : \"=f\"(w0), \"=f\"(w1), \"=f\"(w2), \"=f\"(w3) : \"l\"(tex), \"f\"(p3));\n" - " p0 = fmaf(t0, 65535.0f, 0.37f); p1 = fmaf(u0, 65535.0f, 0.61f); p2 = fmaf(v0, 65535.0f, 0.13f); p3 = fmaf(w0, 65535.0f, 0.89f); }\n" - " out[g] = __float_as_uint(p0) ^ __float_as_uint(p1) ^ __float_as_uint(p2) ^ __float_as_uint(p3);\n}\n" }, -}; - -static std::string utcNow() { - std::time_t t = std::time(nullptr); char b[32]; std::strftime(b, sizeof b, "%Y-%m-%dT%H:%M:%SZ", std::gmtime(&t)); return b; -} - -static int runMicrobench(Ctx& c, const Options& o) { - std::printf("microbench on %s (sm_%d%d, %d SMs, driver %d.%d, NVRTC %d.%d): %d s per probe, full residency, wall time around cuStreamSynchronize; a 1 Hz power sampler aligns on start_utc and end_utc\n", - c.name.c_str(), c.major, c.minor, c.sms, c.driverVersion / 1000, (c.driverVersion % 100) / 10, c.rtcMajor, c.rtcMinor, o.mbSeconds); - const size_t local = 256; - CUdeviceptr dOut = 0; - const size_t maxLanes = (size_t)(c.sms > 0 ? c.sms : 1) * 2048u; - if (c.drv.memAlloc(&dOut, maxLanes * 4u) != CUDA_SUCCESS) { std::printf("microbench: cuMemAlloc out\n"); return 2; } - std::vector hOut(maxLanes); - int ran = 0, failed = 0; - for (const MicroProbe& pr : MICRO_PROBES) { - if (!o.mbOnly.empty() && ("," + o.mbOnly + ",").find(std::string(",") + pr.name + ",") == std::string::npos) continue; - std::string src = std::string(MICRO_PRELUDE) + "KB_KERNEL(kb_probe)\n" + pr.body; - Compiled cp; std::string err; - if (!rtcCompile(c, src, (std::string(pr.name) + ".cu").c_str(), "", "", {}, cp, err)) { - std::string e = err; for (char& ch : e) if (ch == '\n') ch = ' '; - std::printf("RESULT microbench probe=%s status=compile_failed error=\"%.300s\"\n", pr.name, e.c_str()); ++failed; std::fflush(stdout); continue; - } - CUmodule mod = nullptr; CUfunction kFill = nullptr, kProbe = nullptr; - if (c.drv.moduleLoadData(&mod, cp.image.data()) != CUDA_SUCCESS || c.drv.moduleGetFunction(&kFill, mod, "kb_fill") != CUDA_SUCCESS || c.drv.moduleGetFunction(&kProbe, mod, "kb_probe") != CUDA_SUCCESS) { - std::printf("RESULT microbench probe=%s status=load_failed\n", pr.name); ++failed; std::fflush(stdout); if (mod) c.drv.moduleUnload(mod); continue; - } - int blocksPerSM = 0; c.drv.occupancy(&blocksPerSM, kProbe, (int)local, 0); - if (blocksPerSM < 1) blocksPerSM = 1; - size_t lanes = (size_t)blocksPerSM * local * (size_t)(c.sms > 0 ? c.sms : 1); - if (lanes > maxLanes) lanes = maxLanes; - int regs = 0; c.drv.funcGetAttribute(®s, CU_FUNC_ATTRIBUTE_NUM_REGS, kProbe); - CUdeviceptr dTbl = 0; uint32_t mask = 0; unsigned long long tex = 0; CUtexObject texObj = 0; bool ok = true; std::string why; - if (pr.tableMib > 0) { - uint64_t bytes = (uint64_t)pr.tableMib << 20; uint32_t words = (uint32_t)(bytes / 4ull); mask = words - 1u; - if (c.drv.memAlloc(&dTbl, (size_t)bytes) != CUDA_SUCCESS) { ok = false; why = "cuMemAlloc table"; } - else { uint32_t n = words, mode = (uint32_t)pr.tableMode; void* a[3] = { &dTbl, &n, &mode }; - if (c.drv.launchKernel(kFill, (unsigned)((words + 255u) / 256u), 1, 1, 256, 1, 1, 0, nullptr, a, nullptr) != CUDA_SUCCESS || c.drv.streamSynchronize(nullptr) != CUDA_SUCCESS) { ok = false; why = "fill"; } } - } - if (ok && pr.tex > 0) { - if (!c.drv.texObjectCreate || !c.drv.texObjectDestroy) { ok = false; why = "the driver has no cuTexObjectCreate"; } - else { - CUDA_RESOURCE_DESC rd; std::memset(&rd, 0, sizeof rd); rd.resType = CU_RESOURCE_TYPE_LINEAR; - rd.res.linear.devPtr = dTbl; rd.res.linear.format = pr.tex == 1 ? CU_AD_FORMAT_UNSIGNED_INT32 : CU_AD_FORMAT_FLOAT; rd.res.linear.numChannels = 1; rd.res.linear.sizeInBytes = (size_t)pr.tableMib << 20; - CUDA_TEXTURE_DESC td; std::memset(&td, 0, sizeof td); - td.addressMode[0] = CU_TR_ADDRESS_MODE_CLAMP; td.addressMode[1] = CU_TR_ADDRESS_MODE_CLAMP; td.addressMode[2] = CU_TR_ADDRESS_MODE_CLAMP; - td.filterMode = pr.tex == 1 ? CU_TR_FILTER_MODE_POINT : CU_TR_FILTER_MODE_LINEAR; td.flags = pr.tex == 1 ? CU_TRSF_READ_AS_INTEGER : 0; - CUresult r = c.drv.texObjectCreate(&texObj, &rd, &td, nullptr); - if (r != CUDA_SUCCESS) { ok = false; why = "cuTexObjectCreate: " + c.err(r); } else tex = (unsigned long long)texObj; - } - } - if (ok) { - uint32_t steps = pr.steps, seed = 0x51ed270bu; void* a[6] = { &steps, &seed, &dOut, &dTbl, &mask, &tex }; - // warm-up (also the checksum's launch) - CUresult r = c.drv.launchKernel(kProbe, (unsigned)(lanes / local), 1, 1, (unsigned)local, 1, 1, 0, nullptr, a, nullptr); - if (r == CUDA_SUCCESS) r = c.drv.streamSynchronize(nullptr); - if (r != CUDA_SUCCESS) { ok = false; why = "warm-up: " + c.err(r); } - else { - c.drv.memcpyDtoH(hOut.data(), dOut, lanes * 4u); - uint64_t fp = fnv1a64Bytes(hOut.data(), lanes * 4u); - std::string start = utcNow(); double t0 = wallMs(); uint64_t launches = 0; double last = 0; - while (true) { - double l0 = wallMs(); - seed += 0x9E3779B9u; - if (c.drv.launchKernel(kProbe, (unsigned)(lanes / local), 1, 1, (unsigned)local, 1, 1, 0, nullptr, a, nullptr) != CUDA_SUCCESS || c.drv.streamSynchronize(nullptr) != CUDA_SUCCESS) { ok = false; why = "launch"; break; } - last = wallMs() - l0; ++launches; - if (wallMs() - t0 >= o.mbSeconds * 1000.0) break; - } - double secs = (wallMs() - t0) / 1000.0; std::string end = utcNow(); - if (ok) { - double stepsPerS = (double)lanes * (double)pr.steps * (double)launches / secs; - std::printf("RESULT microbench probe=%s status=ok unit=\"%s\" ops_per_step=%g lanes=%zu regs=%d blocks_per_sm=%d steps=%u launches=%llu launch_ms=%.1f seconds=%.1f start_utc=%s end_utc=%s G_steps_s=%.3f G_ops_s=%.3f checksum=%016llx\n", - pr.name, pr.unit, pr.opsPerStep, lanes, regs, blocksPerSM, pr.steps, (unsigned long long)launches, last, secs, start.c_str(), end.c_str(), stepsPerS / 1e9, stepsPerS * pr.opsPerStep / 1e9, (unsigned long long)fp); - ++ran; - } - } - } - if (!ok) { std::printf("RESULT microbench probe=%s status=skipped why=\"%s\"\n", pr.name, why.c_str()); ++failed; } - std::fflush(stdout); - if (texObj) c.drv.texObjectDestroy(texObj); - if (dTbl) c.drv.memFree(dTbl); - c.drv.moduleUnload(mod); - } - c.drv.memFree(dOut); - std::printf("microbench: done, %d probes ran, %d skipped or failed\n", ran, failed); - return failed > 0 && ran == 0 ? 2 : 0; -} - int main(int argc, char** argv) { Options o = parseArgs(argc, argv); Ctx c; c.blockWarps = o.blockWarps; c.race = o.race; c.raceBenchMs = o.raceBenchMs; c.raceBudgetS = o.raceBudgetS; c.raceRounds = o.raceRounds; c.batchLog2 = o.batchLog2; c.pinned = o.pinned; c.warps = o.warps; c.batches = o.batches; - // --bench runs the pack's base kernel with no race; with --variant (Counter ASIC 4.0 research, 8 October 2026: - // the SM-sparse job of 7 October ran 48 rows on base because --bench turned the race off and buildPair never raced) it - // runs the pinned race instead: base and the named variant only, no timing, the variant installed whatever its speed, - // so the bench reads that kernel; the race line names it - if (benchRaceOff(o.bench || o.memprobe || o.microbench, o.pinned)) c.race = "off"; + if (o.bench || o.memprobe) c.race = "off"; if (!o.tuningPath.empty()) { bool ok = false; c.tuning = readText(o.tuningPath, ok); if (!ok) c.tuning.clear(); } - if (o.listRace) { - // Counter ASIC 4.0 research (8 October 2026, the third fix of the --bench --variant fault): the race order as the - // bench's own option handling builds it, with no device: "variants 2" with the named variant is the pass line, - // "variants 1 base only" the 02:52Z fault; with a pack the rewrite is applied to the bound kernel and reported - bool raceOff = benchRaceOff(o.bench || o.memprobe || o.microbench, o.pinned); - std::vector all = allVariants(); - Tuning tu = readTuning(c.tuning, ""); - std::string pinned = !o.pinned.empty() ? o.pinned : (tu.found && !tu.race ? tu.variant : ""); - std::vector order = raceOff ? std::vector{ *findVariant(all, "base") } : raceOrder(all, pinned, tu, c.race); - std::string names; - for (const Variant& v : order) names += (names.empty() ? "" : ",") + v.name; - std::printf("RESULT list-race bench=%d race_off=%d pinned=%s variants=%zu names=%s\n", o.bench ? 1 : 0, raceOff ? 1 : 0, pinned.c_str(), order.size(), names.c_str()); - bool ok = pinned.empty() || (order.size() >= 2 && order.back().name == pinned); - if (!o.pack.empty() && !pinned.empty()) { - bool rok = true; std::string text = readText(o.pack + "/kernel_bound.cu", rok), dev, err2, src, why; - const Variant* v = findVariant(all, pinned); - bool rewritten = rok && v && deviceOnly(text, dev, err2) && variantSource(dev, *v, v->blockWarps > 0 ? v->blockWarps : o.blockWarps, src, why); - std::printf("RESULT list-race pack=%s variant=%s sparse_blocks=%d block_warps=%d rewrite=%s bytes=%zu nonces_arg=%d unit_fn=%d why=\"%s\"\n", o.pack.c_str(), pinned.c_str(), v ? v->sparseBlocks : 0, v ? (v->blockWarps > 0 ? v->blockWarps : o.blockWarps) : 0, - rewritten ? "applied" : "none", src.size(), rewritten && src.find(", uint32_t nonces) {") != std::string::npos ? 1 : 0, rewritten && src.find("igneum_hash_bound_unit(") != std::string::npos ? 1 : 0, why.empty() ? err2.c_str() : why.c_str()); - ok = ok && rewritten; - if (rewritten && v && v->sparseBlocks > 0) { - // the wrapper's call against the unit function's parameters, name by name (the 03:23Z fault: every - // argument one character short), then the whole rewritten text through NVRTC when its library is here - size_t u = src.find("void igneum_hash_bound_unit("), uc = u == std::string::npos ? u : src.find(")", u); - size_t cl = src.rfind("igneum_hash_bound_unit("), cc = cl == std::string::npos ? cl : src.find(")", cl); - std::string params = u == std::string::npos ? "" : src.substr(u + 27, uc - (u + 27)), call = cl == std::string::npos ? "" : src.substr(cl + 23, cc - (cl + 23)); - auto names = [](const std::string& list, bool lastToken) { std::vector out; size_t i = 0; while (i <= list.size()) { size_t j = list.find(',', i); if (j == std::string::npos) j = list.size(); std::string one = list.substr(i, j - i); size_t e = one.find_last_not_of(" \t"); if (e != std::string::npos) { size_t b = lastToken ? one.find_last_of(" \t*&", e) : std::string::npos; size_t st = b == std::string::npos ? one.find_first_not_of(" \t") : b + 1; out.push_back(one.substr(st, e - st + 1)); } i = j + 1; } return out; }; - std::vector pn = names(params, true), cn = names(call, false); - bool match = pn.size() == cn.size() && !pn.empty(); - for (size_t i = 0; match && i < pn.size(); ++i) if (pn[i] != cn[i]) match = false; - std::printf("RESULT list-race call=\"igneum_hash_bound_unit(%s)\" params=%zu args=%zu names_match=%d\n", call.c_str(), pn.size(), cn.size(), match ? 1 : 0); - ok = ok && match; - std::string rerr, rlib; - if (loadNvrtc(c.rtc, rerr, rlib)) { - c.archOpt = "sm_120"; c.rtcMajor = 12; c.rtcMinor = 8; - Compiled cb; std::string cerr; - bool pr = true; std::string programH = readText(o.pack + "/program.h", pr), memhardH = readText(o.pack + "/memhard.h", pr); - bool compiled = pr && rtcCompile(c, src, "kernel_bound.cu", programH, memhardH, { "igneum_hash_bound" }, cb, cerr); - for (char& ch : cerr) if (ch == '\n') ch = ' '; - std::printf("RESULT list-race nvrtc=%s arch=sm_120 compiled=%d image_bytes=%zu error=\"%.400s\"\n", rlib.c_str(), compiled ? 1 : 0, cb.image.size(), cerr.c_str()); - ok = ok && compiled; - } else { - std::printf("RESULT list-race nvrtc=none (%s): the rewritten text was not compiled here\n", rerr.c_str()); - } - } - } - return ok ? 0 : 1; - } std::string err, drvLib, rtcLib; if (!loadDriver(c.drv, err, drvLib)) { emit("error 0 " + err); return 2; } if (!loadNvrtc(c.rtc, err, rtcLib)) { emit("error 0 " + err); return 2; } @@ -1853,9 +1476,8 @@ int main(int argc, char** argv) { WORKER_VERSION, o.device, c.name.c_str(), c.major, c.minor, c.sms, c.driverVersion / 1000, (c.driverVersion % 100) / 10, drvLib.c_str(), c.rtcMajor, c.rtcMinor, rtcLib.c_str(), c.archOpt.c_str(), c.why.c_str())); if (!c.tuning.empty()) info(fmt("tuning file %s (%zu bytes): %s", o.tuningPath.c_str(), c.tuning.size(), readTuning(c.tuning, c.name).found ? "has an entry for this card" : "no entry for this card")); if (o.memprobe) { int rc = runMemprobe(c, o); c.drv.primaryCtxRelease(c.dev); return rc; } - if (o.microbench) { int rc = runMicrobench(c, o); c.drv.primaryCtxRelease(c.dev); return rc; } double t0 = wallMs(); - Pair* cur = buildPair(c, o.pack, nullptr, err, !o.check && !benchRaceOff(o.bench, o.pinned)); + Pair* cur = buildPair(c, o.pack, nullptr, err, !o.check && !o.bench); if (!cur) { emit("error 0 " + err); return 1; } if (o.bench) { std::printf("pack %s on %s: %s\n", o.pack.c_str(), c.name.c_str(), pairSummary(cur).c_str()); diff --git a/proto-cuda/packs-ca4/mx8_mm128/kernel.cl b/proto-cuda/packs-ca4/mx8_mm128/kernel.cl deleted file mode 100644 index 01401bc46..000000000 --- a/proto-cuda/packs-ca4/mx8_mm128/kernel.cl +++ /dev/null @@ -1,415 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// OpenCL C twin of the Metal kernel for the same seed (see proto-opencl/README.md, WAVEFRONT.md and program.metal). -// Built from source at runtime by proto-opencl/host.c, which passes these defines: -// IGNEUM_GROUP work-group size of igneum_hash, a multiple of 32 (default 32: one work-group = one 32-lane unit) -// IGNEUM_EXCHANGE 0 = local-memory exchange with a barrier (any device, any wave width; the default) -// 1 = sub_group_shuffle_xor (cl_khr_subgroup_shuffle), only with IGNEUM_GROUP 32 and a sub-group size of exactly 32 -// 2 = intel_sub_group_shuffle_xor (cl_intel_subgroups), same condition -// The verification unit is always 32 lanes. A 64-wide hardware wave (AMD GCN/CDNA, RDNA in wave64) runs two units; -// the exchange masks are 1, 2, 4, 8, 16, so every partner lane lies inside the lane's own aligned run of 32. -#ifndef IGNEUM_GROUP -#define IGNEUM_GROUP 32 -#endif -#ifndef IGNEUM_EXCHANGE -#define IGNEUM_EXCHANGE 0 -#endif -#ifdef __OPENCL_VERSION__ -#define IGNEUM_KERNEL_HASH __kernel __attribute__((reqd_work_group_size(IGNEUM_GROUP, 1, 1))) -#define IGNEUM_LOCAL_WORDS(name, n) __local uint name[n] -#if IGNEUM_EXCHANGE == 1 -#ifdef cl_khr_subgroups -#pragma OPENCL EXTENSION cl_khr_subgroups : enable -#endif -#ifdef cl_khr_subgroup_shuffle -#pragma OPENCL EXTENSION cl_khr_subgroup_shuffle : enable -#endif -#elif IGNEUM_EXCHANGE == 2 -#pragma OPENCL EXTENSION cl_intel_subgroups : enable -#endif -#else -// Not an OpenCL compiler: proto-opencl/emu compiles this file as C++ and supplies the built-ins and these two macros. -#include "emu_opencl.h" -#endif - -#if IGNEUM_EXCHANGE == 1 -#define IGNEUM_SHFL_XOR(dst, a, m) dst = sub_group_shuffle_xor((a), (uint)(m)) -#define IGNEUM_BCAST0(dst, a) dst = sub_group_broadcast((a), 0u) -#define IGNEUM_SHFL_IDX(dst, a, i) dst = sub_group_shuffle((a), (uint)(i)) -#elif IGNEUM_EXCHANGE == 2 -#define IGNEUM_SHFL_XOR(dst, a, m) dst = intel_sub_group_shuffle_xor((a), (uint)(m)) -#define IGNEUM_BCAST0(dst, a) dst = sub_group_broadcast((a), 0u) -#define IGNEUM_SHFL_IDX(dst, a, i) dst = intel_sub_group_shuffle((a), (uint)(i)) -#else -// Local-memory exchange. Two buffers of IGNEUM_GROUP words alternate (xk counts exchanges), so one barrier per -// exchange is enough: a lane can only overwrite buffer b at exchange k+2 after passing barrier k+1, and every lane -// reaches barrier k+1 only after its read of buffer b at exchange k. The partner lid ^ m stays inside the lane's -// aligned run of 32 because m < 32. Control flow is uniform, so every work-item reaches every barrier. -#define IGNEUM_SHFL_XOR(dst, a, m) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid ^ (uint)(m))]; xk += 1u; } -#define IGNEUM_BCAST0(dst, a) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid & ~31u)]; xk += 1u; } -#define IGNEUM_SHFL_IDX(dst, a, i) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid & ~31u) + (uint)(i)]; xk += 1u; } -#endif - -// int8 tile reference (Counter ASIC 4.0 research): 12 index shuffles and 32 byte products per lane; the PTX mma.m8n8k16 u8 layout. -#define IGNEUM_MM8_REF(av, bv, d0, d1) { uint row_ = lane >> 2, col0_ = 2u * (lane & 3u); uint acc0_ = 0u, acc1_ = 0u; for (uint j_ = 0u; j_ < 4u; ++j_) { uint aw_, bw0_, bw1_; IGNEUM_SHFL_IDX(aw_, (av), 4u * row_ + j_); IGNEUM_SHFL_IDX(bw0_, (bv), 4u * col0_ + j_); IGNEUM_SHFL_IDX(bw1_, (bv), 4u * (col0_ + 1u) + j_); for (uint t_ = 0u; t_ < 4u; ++t_) { uint ab_ = (aw_ >> (8u * t_)) & 0xffu; acc0_ += ab_ * ((bw0_ >> (8u * t_)) & 0xffu); acc1_ += ab_ * ((bw1_ >> (8u * t_)) & 0xffu); } } d0 = acc0_; d1 = acc1_; } - -static inline uint splitmix32(uint x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -// n is a literal in 1..31 at every call site. OpenCL rotate() rotates left by n modulo 32. -static inline uint rotl_imm(uint x, uint n) { return rotate(x, n); } -// Right rotation by n modulo 32 as a left rotation by (32 - n) modulo 32; n == 0 gives x. -static inline uint rotr_var(uint x, uint n) { return rotate(x, (0u - n) & 31u); } -static inline uint ds_elem(uint i, uint d0, uint d1) { - uint x = i ^ d0; - x *= 0x9E3779B1u; x ^= x >> 15; - x += d1; - x *= 0x85EBCA77u; x ^= x >> 13; - x *= 0xC2B2AE3Du; x ^= x >> 16; - return x; -} - -// Memory-hard dataset core (MEMHARD.md). Cache: 2^26 words in 2^16 segments of 64 chained ChaCha12 lines. -// Item: 8 rounds of 8 x seed-parameterised mixer + one 64-byte cache read, then 8 x final mixer (class v3, mixer multiplier 8, -// docs/plans/mixer-x4.md: the round key of application j of round r is 0x9E3779B9 * (r * m + j + 1)). All parameters are literals. -#define MH_CACHE_LINE_MASK 0x003fffffu -#define MH_SEGMENT_LINES 64u -#define MH_QR(a, b, c, d, r1, r2, r3, r4) { a += b; d ^= a; d = mh_rotl(d, r1); c += d; b ^= c; b = mh_rotl(b, r2); a += b; d ^= a; d = mh_rotl(d, r3); c += d; b ^= c; b = mh_rotl(b, r4); } -static inline uint mh_rotl(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 at every call site - -// y = ChaCha12 core(x) + x -static inline void mh_chacha_block(const uint* x, uint* y) { - for (uint i = 0u; i < 16u; ++i) y[i] = x[i]; - for (uint r = 0u; r < 6u; ++r) { - MH_QR(y[0], y[4], y[8], y[12], 16u, 12u, 8u, 7u) MH_QR(y[1], y[5], y[9], y[13], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[6], y[10], y[14], 16u, 12u, 8u, 7u) MH_QR(y[3], y[7], y[11], y[15], 16u, 12u, 8u, 7u) - MH_QR(y[0], y[5], y[10], y[15], 16u, 12u, 8u, 7u) MH_QR(y[1], y[6], y[11], y[12], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[7], y[8], y[13], 16u, 12u, 8u, 7u) MH_QR(y[3], y[4], y[9], y[14], 16u, 12u, 8u, 7u) - } - for (uint i = 0u; i < 16u; ++i) y[i] += x[i]; -} - -// One cache segment: 64 chained lines written at cache[seg * 1024]. in_j = prev ^ (sigma || K || seg || j || tag), prev_0 = 0. -static inline void mh_cache_segment(__global uint* cache, uint seg) { - uint prev[16]; uint x[16]; uint y[16]; - for (uint i = 0u; i < 16u; ++i) prev[i] = 0u; - for (uint j = 0u; j < MH_SEGMENT_LINES; ++j) { - x[0] = 0x61707865u ^ prev[0]; x[1] = 0x3320646eu ^ prev[1]; x[2] = 0x79622d32u ^ prev[2]; x[3] = 0x6b206574u ^ prev[3]; - x[4] = 0x3067619fu ^ prev[4]; - x[5] = 0x3c269176u ^ prev[5]; - x[6] = 0x84a03b03u ^ prev[6]; - x[7] = 0xf8c63294u ^ prev[7]; - x[8] = 0xff977c5bu ^ prev[8]; - x[9] = 0xe60def3eu ^ prev[9]; - x[10] = 0x63630141u ^ prev[10]; - x[11] = 0xb8fbcb58u ^ prev[11]; - x[12] = seg ^ prev[12]; x[13] = j ^ prev[13]; x[14] = 0x49676e65u ^ prev[14]; x[15] = 0x756d4d48u ^ prev[15]; - mh_chacha_block(x, y); - __global uint* line = cache + ((seg * MH_SEGMENT_LINES + j) * 16u); - for (uint i = 0u; i < 16u; ++i) { line[i] = y[i]; prev[i] = y[i]; } - } -} - -// M_r: per word (s ^ (RC + rk)) * MUL, then a column round and a diagonal round with the seed-drawn rotations. -static inline void mh_mixer(uint* s, uint rk) { - s[0] = (s[0] ^ (0xbab68293u + rk)) * 0x42146205u; - s[1] = (s[1] ^ (0xcc162340u + rk)) * 0x52cbe0fbu; - s[2] = (s[2] ^ (0x6ce151ccu + rk)) * 0x7ecf4a03u; - s[3] = (s[3] ^ (0xe62b8997u + rk)) * 0x6728907fu; - s[4] = (s[4] ^ (0xc9c80297u + rk)) * 0xd81d9751u; - s[5] = (s[5] ^ (0xf74a1654u + rk)) * 0x132952c3u; - s[6] = (s[6] ^ (0x3d704af5u + rk)) * 0xf60de277u; - s[7] = (s[7] ^ (0x3cf522b7u + rk)) * 0x05358035u; - s[8] = (s[8] ^ (0x2b9cac04u + rk)) * 0xbaf6499du; - s[9] = (s[9] ^ (0xa880ac10u + rk)) * 0xe4db9667u; - s[10] = (s[10] ^ (0x13e5dd1du + rk)) * 0x3e98f45du; - s[11] = (s[11] ^ (0x6fc3e233u + rk)) * 0xd0004eddu; - s[12] = (s[12] ^ (0x2d83eeacu + rk)) * 0x2691630du; - s[13] = (s[13] ^ (0x9006e8bfu + rk)) * 0x9beb3bcfu; - s[14] = (s[14] ^ (0x2c4b5362u + rk)) * 0xab310379u; - s[15] = (s[15] ^ (0x31b49ee2u + rk)) * 0x99cfb423u; - MH_QR(s[0], s[4], s[8], s[12], 20u, 20u, 19u, 4u) MH_QR(s[1], s[5], s[9], s[13], 20u, 20u, 19u, 4u) - MH_QR(s[2], s[6], s[10], s[14], 20u, 20u, 19u, 4u) MH_QR(s[3], s[7], s[11], s[15], 20u, 20u, 19u, 4u) - MH_QR(s[0], s[5], s[10], s[15], 26u, 3u, 3u, 27u) MH_QR(s[1], s[6], s[11], s[12], 26u, 3u, 3u, 27u) - MH_QR(s[2], s[7], s[8], s[13], 26u, 3u, 3u, 27u) MH_QR(s[3], s[4], s[9], s[14], 26u, 3u, 3u, 27u) -} - -// Item t: 16 words. s = (K, t * MUL[i] + RC[i]); 8 rounds of 8 x mixer + cache line s[0] & mask; 8 x final mixer. -static inline void mh_item(__global const uint* cache, uint t, uint* s) { - s[0] = 0x3067619fu; - s[1] = 0x3c269176u; - s[2] = 0x84a03b03u; - s[3] = 0xf8c63294u; - s[4] = 0xff977c5bu; - s[5] = 0xe60def3eu; - s[6] = 0x63630141u; - s[7] = 0xb8fbcb58u; - s[8] = t * 0x42146205u + 0xbab68293u; - s[9] = t * 0x52cbe0fbu + 0xcc162340u; - s[10] = t * 0x7ecf4a03u + 0x6ce151ccu; - s[11] = t * 0x6728907fu + 0xe62b8997u; - s[12] = t * 0xd81d9751u + 0xc9c80297u; - s[13] = t * 0x132952c3u + 0xf74a1654u; - s[14] = t * 0xf60de277u + 0x3d704af5u; - s[15] = t * 0x05358035u + 0x3cf522b7u; - for (uint r = 0u; r < 8u; ++r) { - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (r * 8u + j + 1u)); - __global const uint* line = cache + ((s[0] & MH_CACHE_LINE_MASK) * 16u); - for (uint i = 0u; i < 16u; ++i) s[i] ^= line[i]; - } - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (64u + j + 1u)); -} -// dataset[w] without the dataset: derive item w >> 4 and take word w & 15. -static inline uint mh_word(__global const uint* cache, uint w) { uint s[16]; mh_item(cache, w >> 4u, s); return s[w & 15u]; } - -// Memory-hard dataset (MEMHARD.md). One work-item per cache segment; one work-item per 64-byte dataset item. -// The same constants as memhard.h in this pack (one emitter, three dialects). -__kernel void igneum_cache_fill(__global uint* cache, uint nSegments) { - uint seg = (uint)get_global_id(0); - if (seg < nSegments) mh_cache_segment(cache, seg); -} -__kernel void igneum_build(__global uint* ds, __global const uint* cache, uint nItems) { - uint t = (uint)get_global_id(0); - if (t < nItems) { - uint s[16]; - mh_item(cache, t, s); - __global uint* d = ds + ((ulong)t * 16u); - for (uint i = 0u; i < 16u; ++i) d[i] = s[i]; - } -} - -// One hash per work-item. IGNEUM_GROUP is a multiple of 32; lane = lid & 31 and every exchange stays inside the -// lane's own aligned run of 32 work-items, exactly like simd_shuffle_xor inside a 32-wide Metal SIMD group and -// __shfl_xor_sync inside a CUDA warp. Control flow is uniform (no branches at all). -IGNEUM_KERNEL_HASH void igneum_hash(__global const uint* ds, __global ulong* out, uint baseNonce, uint mask) { - uint gid = (uint)get_global_id(0); - uint lid = (uint)get_local_id(0); - uint nonce = baseNonce + gid; - uint r0, r1, r2, r3, r4, r5, r6, r7; -#if IGNEUM_EXCHANGE == 0 - IGNEUM_LOCAL_WORDS(xch, 2 * IGNEUM_GROUP); - uint xk = 0u; -#else - (void)lid; -#endif - uint lane = lid & 31u; - { uint x = nonce ^ 0x67a9a7beu; x += 0x9e3779b9u; x = splitmix32(x); r0 = x ^ 0x1a155b25u; } // SEEDW[0], 0x9e3779b9u * 1u, SEEDW[1] - { uint x = nonce ^ 0x1a155b25u; x += 0x3c6ef372u; x = splitmix32(x); r1 = x ^ 0xfddfb732u; } // SEEDW[1], 0x9e3779b9u * 2u, SEEDW[2] - { uint x = nonce ^ 0xfddfb732u; x += 0xdaa66d2bu; x = splitmix32(x); r2 = x ^ 0x4b5af2e8u; } // SEEDW[2], 0x9e3779b9u * 3u, SEEDW[3] - { uint x = nonce ^ 0x4b5af2e8u; x += 0x78dde6e4u; x = splitmix32(x); r3 = x ^ 0xc55caf33u; } // SEEDW[3], 0x9e3779b9u * 4u, SEEDW[4] - { uint x = nonce ^ 0xc55caf33u; x += 0x1715609du; x = splitmix32(x); r4 = x ^ 0xa27c13b7u; } // SEEDW[4], 0x9e3779b9u * 5u, SEEDW[5] - { uint x = nonce ^ 0xa27c13b7u; x += 0xb54cda56u; x = splitmix32(x); r5 = x ^ 0x06628a48u; } // SEEDW[5], 0x9e3779b9u * 6u, SEEDW[6] - { uint x = nonce ^ 0x06628a48u; x += 0x5384540fu; x = splitmix32(x); r6 = x ^ 0x03852469u; } // SEEDW[6], 0x9e3779b9u * 7u, SEEDW[7] - { uint x = nonce ^ 0x03852469u; x += 0xf1bbcdc8u; x = splitmix32(x); r7 = x ^ 0x67a9a7beu; } // SEEDW[7], 0x9e3779b9u * 8u, SEEDW[0] - - for (uint it = 0u; it < 8u; ++it) { - uint sel = r0; - r2 = r3 * r4 + r2; // 0 mad - r2 = r1 * r1 + r2; // 1 mad - r2 = r3 * r2 + r2; // 2 mad - r3 = r3 ^ r5; // 3 xor - r7 = r7 ^ ds[r2 & mask]; // 4 load - r5 = r5 ^ ds[r7 & mask]; // 5 load - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 8u); r1 = r1 ^ t_; } // 6 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 8u); r7 = r7 ^ t_; } // 7 shfl - r1 = mul_hi(r1, r5); // 8 mulhi - r6 = rotr_var(r6, r3); // 9 rotr - r3 = r3 | r4; // 10 or - r4 = r4 ^ ds[r3 & mask]; // 11 load - r0 = mul_hi(r0, r4); // 12 mulhi - r5 = r5 + r1 + ((((sel >> 30u) & 1u) != 0u) ? 0xd3177981u : 0xc7934706u); // 13 add - r0 = r0 ^ ds[r4 & mask]; // 14 load - r2 = r2 - r4; // 15 sub - r2 = r2 ^ ds[r0 & mask]; // 16 load - r7 = r7 ^ ds[r2 & mask]; // 17 load - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 4u); r7 = r7 ^ t_; } // 18 shfl - r5 = r5 * r0; // 19 mul - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 2u); r3 = r3 ^ t_; } // 20 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 16u); r2 = r2 ^ t_; } // 21 shfl - r6 = mul_hi(r6, r2); // 22 mulhi - r6 = r6 ^ ds[r1 & mask]; // 23 load - r5 = r5 * r0; // 24 mul - r5 = rotl_imm(r5, 19u); // 25 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 2u); r7 = r7 ^ t_; } // 26 shfl - r0 = r0 ^ r5; // 27 xor - r0 = r0 ^ r4; // 28 xor - r3 = r3 - r0; // 29 sub - r5 = r5 * r1; // 30 mul - r7 = r7 ^ ds[r2 & mask]; // 31 load - r1 = r1 ^ ds[r0 & mask]; // 32 load - r5 = r5 ^ r6; // 33 xor - r5 = r5 ^ ds[r1 & mask]; // 34 load - r0 = mul_hi(r0, r5); // 35 mulhi - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 4u); r5 = r5 ^ t_; } // 36 shfl - r7 = r7 ^ ds[r0 & mask]; // 37 load - r3 = r3 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0x230c005cu : 0x75ba2fadu); // 38 add - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 4u); r1 = r1 ^ t_; } // 39 shfl - r2 = r2 ^ r5; // 40 xor - r3 = r6 * r3 + r3; // 41 mad - r6 = r6 - r7; // 42 sub - r7 = r7 ^ r0; // 43 xor - r1 = r1 ^ ds[r7 & mask]; // 44 load - r2 = r2 * r3; // 45 mul - r1 = mul_hi(r1, r5); // 46 mulhi - r4 = r4 - r3; // 47 sub - r2 = rotr_var(r2, r6); // 48 rotr - r3 = r3 ^ ds[r5 & mask]; // 49 load - r1 = r1 + r5 + ((((sel >> 7u) & 1u) != 0u) ? 0x1907970cu : 0x81b8bc2cu); // 50 add - r0 = r0 * r2; // 51 mul - r0 = r0 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x699fd448u : 0x4f92b968u); // 52 add - r1 = r1 + r0 + ((((sel >> 12u) & 1u) != 0u) ? 0x77b1520du : 0x2bb965afu); // 53 add - r7 = rotl_imm(r7, 14u); // 54 rotl - r3 = r3 + r7 + ((((sel >> 1u) & 1u) != 0u) ? 0xa54c55a0u : 0x7b0fe07au); // 55 add - r6 = r6 ^ ds[r7 & mask]; // 56 load - r1 = rotr_var(r1, r5); // 57 rotr - r5 = r5 ^ ds[r4 & mask]; // 58 load - r6 = r6 ^ ds[r2 & mask]; // 59 load - r3 = r5 * r0 + r3; // 60 mad - r5 = r5 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0xad7493e7u : 0xaf9dd72du); // 61 add - r4 = r4 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x1e07c3d9u : 0x89841d87u); // 62 add - r5 = rotl_imm(r5, 19u); // 63 rotl - // int8 tile block (Counter ASIC 4.0 research, experimental): 128 mma.m8n8k16 u8 tiles per iteration, both outputs consumed - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r2 += d0_; r3 += d1_; } // t0 mm8 a=r6 b=r5 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r4 += d0_; r5 += d1_; } // t1 mm8 a=r1 b=r1 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r0 += d0_; r7 += d1_; } // t2 mm8 a=r2 b=r5 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r3 += d1_; } // t3 mm8 a=r1 b=r5 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r2 += d0_; r6 += d1_; } // t4 mm8 a=r2 b=r0 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r7 += d0_; r3 += d1_; } // t5 mm8 a=r2 b=r6 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t6 mm8 a=r1 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r0 += d0_; r4 += d1_; } // t7 mm8 a=r4 b=r6 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r1 += d0_; r5 += d1_; } // t8 mm8 a=r1 b=r1 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r4 += d0_; r6 += d1_; } // t9 mm8 a=r3 b=r7 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r0 += d0_; r4 += d1_; } // t10 mm8 a=r3 b=r0 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r0 += d0_; r6 += d1_; } // t11 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r6 += d0_; r0 += d1_; } // t12 mm8 a=r3 b=r1 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r5 += d1_; } // t13 mm8 a=r1 b=r5 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r3 += d0_; r4 += d1_; } // t14 mm8 a=r3 b=r3 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r5 += d0_; r4 += d1_; } // t15 mm8 a=r1 b=r0 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r0 += d0_; r4 += d1_; } // t16 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r7 += d1_; } // t17 mm8 a=r0 b=r5 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r2 += d0_; r0 += d1_; } // t18 mm8 a=r1 b=r1 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r2 += d0_; r4 += d1_; } // t19 mm8 a=r3 b=r3 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r6 += d0_; r1 += d1_; } // t20 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r6 += d0_; r0 += d1_; } // t21 mm8 a=r0 b=r3 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r4 += d1_; } // t22 mm8 a=r1 b=r7 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r0 += d0_; r3 += d1_; } // t23 mm8 a=r1 b=r2 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r0 += d0_; r2 += d1_; } // t24 mm8 a=r3 b=r4 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r3 += d0_; r7 += d1_; } // t25 mm8 a=r5 b=r0 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t26 mm8 a=r1 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r0 += d0_; r5 += d1_; } // t27 mm8 a=r3 b=r2 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r5 += d1_; } // t28 mm8 a=r2 b=r4 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r1 += d0_; r5 += d1_; } // t29 mm8 a=r3 b=r4 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r6 += d0_; r3 += d1_; } // t30 mm8 a=r6 b=r5 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t31 mm8 a=r1 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r3 += d0_; r0 += d1_; } // t32 mm8 a=r7 b=r6 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r4 += d0_; r6 += d1_; } // t33 mm8 a=r7 b=r1 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r5 += d0_; r1 += d1_; } // t34 mm8 a=r1 b=r0 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t35 mm8 a=r2 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r7 += d0_; r1 += d1_; } // t36 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r0 += d0_; r2 += d1_; } // t37 mm8 a=r1 b=r7 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r2 += d0_; r4 += d1_; } // t38 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r2 += d0_; r5 += d1_; } // t39 mm8 a=r6 b=r2 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r4 += d0_; r1 += d1_; } // t40 mm8 a=r6 b=r1 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r6 += d0_; r2 += d1_; } // t41 mm8 a=r6 b=r0 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r5 += d0_; r1 += d1_; } // t42 mm8 a=r7 b=r4 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r6 += d0_; r0 += d1_; } // t43 mm8 a=r6 b=r1 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r4 += d0_; r1 += d1_; } // t44 mm8 a=r0 b=r6 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r6 += d0_; r5 += d1_; } // t45 mm8 a=r6 b=r5 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r1 += d0_; r0 += d1_; } // t46 mm8 a=r6 b=r6 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r3 += d0_; r1 += d1_; } // t47 mm8 a=r7 b=r3 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t48 mm8 a=r7 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r1 += d0_; r7 += d1_; } // t49 mm8 a=r4 b=r5 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r4 += d0_; r5 += d1_; } // t50 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r0 += d0_; r3 += d1_; } // t51 mm8 a=r3 b=r4 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r7 += d0_; r4 += d1_; } // t52 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r1 += d0_; r3 += d1_; } // t53 mm8 a=r7 b=r7 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t54 mm8 a=r7 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r3 += d0_; r7 += d1_; } // t55 mm8 a=r1 b=r7 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t56 mm8 a=r4 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r6 += d0_; r7 += d1_; } // t57 mm8 a=r4 b=r3 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r6 += d0_; r0 += d1_; } // t58 mm8 a=r2 b=r3 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r7 += d1_; } // t59 mm8 a=r6 b=r7 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t60 mm8 a=r6 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r2 += d0_; r6 += d1_; } // t61 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r6 += d1_; } // t62 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t63 mm8 a=r1 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r5 += d0_; r3 += d1_; } // t64 mm8 a=r0 b=r5 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r2 += d0_; r4 += d1_; } // t65 mm8 a=r0 b=r1 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r1 += d0_; r3 += d1_; } // t66 mm8 a=r3 b=r5 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r3 += d0_; r4 += d1_; } // t67 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r2 += d1_; } // t68 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r4 += d1_; } // t69 mm8 a=r6 b=r7 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r1 += d0_; r5 += d1_; } // t70 mm8 a=r1 b=r7 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r1 += d0_; r0 += d1_; } // t71 mm8 a=r3 b=r0 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r3 += d0_; r6 += d1_; } // t72 mm8 a=r4 b=r0 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r0 += d0_; r2 += d1_; } // t73 mm8 a=r4 b=r1 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r3 += d0_; r4 += d1_; } // t74 mm8 a=r5 b=r1 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r7 += d0_; r3 += d1_; } // t75 mm8 a=r4 b=r4 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r1 += d0_; r7 += d1_; } // t76 mm8 a=r5 b=r6 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r3 += d0_; r2 += d1_; } // t77 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r0 += d0_; r5 += d1_; } // t78 mm8 a=r2 b=r5 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t79 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r3 += d0_; r4 += d1_; } // t80 mm8 a=r6 b=r7 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r1 += d0_; r6 += d1_; } // t81 mm8 a=r4 b=r3 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r1 += d0_; r7 += d1_; } // t82 mm8 a=r6 b=r0 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r1 += d0_; r6 += d1_; } // t83 mm8 a=r2 b=r2 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t84 mm8 a=r0 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t85 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r2 += d0_; r3 += d1_; } // t86 mm8 a=r3 b=r4 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r3 += d0_; r7 += d1_; } // t87 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r5 += d0_; r3 += d1_; } // t88 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r2 += d0_; r1 += d1_; } // t89 mm8 a=r4 b=r2 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r3 += d0_; r6 += d1_; } // t90 mm8 a=r6 b=r5 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r5 += d0_; r4 += d1_; } // t91 mm8 a=r6 b=r5 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r2 += d0_; r3 += d1_; } // t92 mm8 a=r2 b=r6 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r5 += d0_; r7 += d1_; } // t93 mm8 a=r2 b=r2 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r7 += d0_; r3 += d1_; } // t94 mm8 a=r2 b=r4 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r1 += d0_; r6 += d1_; } // t95 mm8 a=r5 b=r4 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r3 += d0_; r1 += d1_; } // t96 mm8 a=r0 b=r6 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r2 += d1_; } // t97 mm8 a=r0 b=r5 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t98 mm8 a=r2 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r0 += d0_; r2 += d1_; } // t99 mm8 a=r1 b=r4 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r2 += d0_; r5 += d1_; } // t100 mm8 a=r2 b=r4 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r7 += d1_; } // t101 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t102 mm8 a=r5 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t103 mm8 a=r4 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r3 += d0_; r7 += d1_; } // t104 mm8 a=r2 b=r0 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r0 += d0_; r5 += d1_; } // t105 mm8 a=r6 b=r1 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r0 += d0_; r5 += d1_; } // t106 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r4 += d0_; r2 += d1_; } // t107 mm8 a=r2 b=r1 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t108 mm8 a=r4 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r5 += d0_; r3 += d1_; } // t109 mm8 a=r2 b=r7 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r0 += d0_; r2 += d1_; } // t110 mm8 a=r1 b=r0 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r4 += d0_; r6 += d1_; } // t111 mm8 a=r6 b=r5 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r4 += d0_; r2 += d1_; } // t112 mm8 a=r5 b=r7 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t113 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t114 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r1 += d0_; r4 += d1_; } // t115 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r5 += d0_; r0 += d1_; } // t116 mm8 a=r7 b=r2 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r7 += d0_; r6 += d1_; } // t117 mm8 a=r7 b=r6 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r2 += d0_; r1 += d1_; } // t118 mm8 a=r0 b=r1 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r5 += d0_; r3 += d1_; } // t119 mm8 a=r4 b=r3 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r7 += d0_; r2 += d1_; } // t120 mm8 a=r5 b=r3 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r1 += d1_; } // t121 mm8 a=r2 b=r7 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r3 += d0_; r0 += d1_; } // t122 mm8 a=r0 b=r1 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r3 += d0_; r5 += d1_; } // t123 mm8 a=r5 b=r2 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t124 mm8 a=r5 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r7 += d0_; r2 += d1_; } // t125 mm8 a=r7 b=r2 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r3 += d0_; r2 += d1_; } // t126 mm8 a=r6 b=r3 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r1 += d1_; } // t127 mm8 a=r6 b=r7 c=r5 c2=r1 - } - uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((ulong)hi << 32) | (ulong)lo; -} - -#if IGNEUM_EXCHANGE != 0 -// Reports the sub-group size this device uses for a work-group of IGNEUM_GROUP items. host.c runs it only when the -// per-kernel query (clGetKernelSubGroupInfoKHR on igneum_hash) is unavailable; that query is preferred because a -// compiler may pick a different wave width per kernel (RDNA: wave32 or wave64). See WAVEFRONT.md. -IGNEUM_KERNEL_HASH void igneum_probe_subgroup(__global uint* out) { - if (get_local_id(0) == 0u) { out[0] = get_sub_group_size(); out[1] = get_num_sub_groups(); } -} -#endif diff --git a/proto-cuda/packs-ca4/mx8_mm128/kernel.cu b/proto-cuda/packs-ca4/mx8_mm128/kernel.cu deleted file mode 100644 index 30b9513a2..000000000 --- a/proto-cuda/packs-ca4/mx8_mm128/kernel.cu +++ /dev/null @@ -1,293 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// Bit-exact twin of the Metal kernel for the same seed (see proto-cuda/CHECKLIST.md and program.metal). -// Compiled ahead of time by nvcc together with proto-cuda/host.cu. No NVRTC. -#include -#include -#include "program.h" -#include "memhard.h" - -__device__ __forceinline__ uint32_t splitmix32(uint32_t x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -// n is a literal in 1..31 at every call site, so both shift amounts are in 1..31. -__device__ __forceinline__ uint32_t rotl_imm(uint32_t x, uint32_t n) { return (x << n) | (x >> (32u - n)); } -// n is masked to 0..31; the second shift amount is masked too, so n == 0 gives x. -__device__ __forceinline__ uint32_t rotr_var(uint32_t x, uint32_t n) { n &= 31u; return (x >> n) | (x << ((32u - n) & 31u)); } -__device__ __forceinline__ uint32_t ds_elem(uint32_t i, uint32_t d0, uint32_t d1) { - uint32_t x = i ^ d0; - x *= 0x9E3779B1u; x ^= x >> 15; - x += d1; - x *= 0x85EBCA77u; x ^= x >> 13; - x *= 0xC2B2AE3Du; x ^= x >> 16; - return x; -} - -// Memory-hard dataset (MEMHARD.md). One thread per cache segment; one thread per 64-byte dataset item. -// The core functions (mh_cache_segment, mh_item) are in memhard.h and are also compiled for the host. -__global__ void igneum_cache_fill(uint32_t* cache, uint32_t nSegments) { - uint32_t seg = blockIdx.x * blockDim.x + threadIdx.x; - if (seg < nSegments) mh_cache_segment(cache, seg); -} -__global__ void igneum_build(uint32_t* ds, const uint32_t* cache, uint32_t nItems) { - uint32_t t = blockIdx.x * blockDim.x + threadIdx.x; - if (t < nItems) { - uint32_t s[16]; - mh_item(cache, t, s); - uint32_t* d = ds + (size_t)t * 16u; - for (uint32_t i = 0u; i < 16u; ++i) d[i] = s[i]; - } -} - -// One hash per thread. blockDim.x is a multiple of 32; lane = threadIdx.x & 31 and every -// __shfl_xor_sync stays inside the lane's own warp, exactly like simd_shuffle_xor inside a -// 32-wide Metal SIMD group. Control flow is uniform, so the full 0xffffffff member mask is valid. -__global__ void igneum_hash(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask) { - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t nonce = baseNonce + gid; - uint32_t r0, r1, r2, r3, r4, r5, r6, r7; - { uint32_t x = nonce ^ 0x67a9a7beu; x += 0x9e3779b9u; x = splitmix32(x); r0 = x ^ 0x1a155b25u; } // SEEDW[0], 0x9e3779b9u * 1u, SEEDW[1] - { uint32_t x = nonce ^ 0x1a155b25u; x += 0x3c6ef372u; x = splitmix32(x); r1 = x ^ 0xfddfb732u; } // SEEDW[1], 0x9e3779b9u * 2u, SEEDW[2] - { uint32_t x = nonce ^ 0xfddfb732u; x += 0xdaa66d2bu; x = splitmix32(x); r2 = x ^ 0x4b5af2e8u; } // SEEDW[2], 0x9e3779b9u * 3u, SEEDW[3] - { uint32_t x = nonce ^ 0x4b5af2e8u; x += 0x78dde6e4u; x = splitmix32(x); r3 = x ^ 0xc55caf33u; } // SEEDW[3], 0x9e3779b9u * 4u, SEEDW[4] - { uint32_t x = nonce ^ 0xc55caf33u; x += 0x1715609du; x = splitmix32(x); r4 = x ^ 0xa27c13b7u; } // SEEDW[4], 0x9e3779b9u * 5u, SEEDW[5] - { uint32_t x = nonce ^ 0xa27c13b7u; x += 0xb54cda56u; x = splitmix32(x); r5 = x ^ 0x06628a48u; } // SEEDW[5], 0x9e3779b9u * 6u, SEEDW[6] - { uint32_t x = nonce ^ 0x06628a48u; x += 0x5384540fu; x = splitmix32(x); r6 = x ^ 0x03852469u; } // SEEDW[6], 0x9e3779b9u * 7u, SEEDW[7] - { uint32_t x = nonce ^ 0x03852469u; x += 0xf1bbcdc8u; x = splitmix32(x); r7 = x ^ 0x67a9a7beu; } // SEEDW[7], 0x9e3779b9u * 8u, SEEDW[0] - - for (uint32_t it = 0u; it < 8u; ++it) { - uint32_t sel = r0; - r2 = r3 * r4 + r2; // 0 mad - r2 = r1 * r1 + r2; // 1 mad - r2 = r3 * r2 + r2; // 2 mad - r3 = r3 ^ r5; // 3 xor - r7 = r7 ^ ds[r2 & mask]; // 4 load - r5 = r5 ^ ds[r7 & mask]; // 5 load - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r4, 8); // 6 shfl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r3, 8); // 7 shfl - r1 = __umulhi(r1, r5); // 8 mulhi - r6 = rotr_var(r6, r3); // 9 rotr - r3 = r3 | r4; // 10 or - r4 = r4 ^ ds[r3 & mask]; // 11 load - r0 = __umulhi(r0, r4); // 12 mulhi - r5 = r5 + r1 + ((((sel >> 30u) & 1u) != 0u) ? 0xd3177981u : 0xc7934706u); // 13 add - r0 = r0 ^ ds[r4 & mask]; // 14 load - r2 = r2 - r4; // 15 sub - r2 = r2 ^ ds[r0 & mask]; // 16 load - r7 = r7 ^ ds[r2 & mask]; // 17 load - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r3, 4); // 18 shfl - r5 = r5 * r0; // 19 mul - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r4, 2); // 20 shfl - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r4, 16); // 21 shfl - r6 = __umulhi(r6, r2); // 22 mulhi - r6 = r6 ^ ds[r1 & mask]; // 23 load - r5 = r5 * r0; // 24 mul - r5 = rotl_imm(r5, 19u); // 25 rotl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r6, 2); // 26 shfl - r0 = r0 ^ r5; // 27 xor - r0 = r0 ^ r4; // 28 xor - r3 = r3 - r0; // 29 sub - r5 = r5 * r1; // 30 mul - r7 = r7 ^ ds[r2 & mask]; // 31 load - r1 = r1 ^ ds[r0 & mask]; // 32 load - r5 = r5 ^ r6; // 33 xor - r5 = r5 ^ ds[r1 & mask]; // 34 load - r0 = __umulhi(r0, r5); // 35 mulhi - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r2, 4); // 36 shfl - r7 = r7 ^ ds[r0 & mask]; // 37 load - r3 = r3 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0x230c005cu : 0x75ba2fadu); // 38 add - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r5, 4); // 39 shfl - r2 = r2 ^ r5; // 40 xor - r3 = r6 * r3 + r3; // 41 mad - r6 = r6 - r7; // 42 sub - r7 = r7 ^ r0; // 43 xor - r1 = r1 ^ ds[r7 & mask]; // 44 load - r2 = r2 * r3; // 45 mul - r1 = __umulhi(r1, r5); // 46 mulhi - r4 = r4 - r3; // 47 sub - r2 = rotr_var(r2, r6); // 48 rotr - r3 = r3 ^ ds[r5 & mask]; // 49 load - r1 = r1 + r5 + ((((sel >> 7u) & 1u) != 0u) ? 0x1907970cu : 0x81b8bc2cu); // 50 add - r0 = r0 * r2; // 51 mul - r0 = r0 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x699fd448u : 0x4f92b968u); // 52 add - r1 = r1 + r0 + ((((sel >> 12u) & 1u) != 0u) ? 0x77b1520du : 0x2bb965afu); // 53 add - r7 = rotl_imm(r7, 14u); // 54 rotl - r3 = r3 + r7 + ((((sel >> 1u) & 1u) != 0u) ? 0xa54c55a0u : 0x7b0fe07au); // 55 add - r6 = r6 ^ ds[r7 & mask]; // 56 load - r1 = rotr_var(r1, r5); // 57 rotr - r5 = r5 ^ ds[r4 & mask]; // 58 load - r6 = r6 ^ ds[r2 & mask]; // 59 load - r3 = r5 * r0 + r3; // 60 mad - r5 = r5 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0xad7493e7u : 0xaf9dd72du); // 61 add - r4 = r4 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x1e07c3d9u : 0x89841d87u); // 62 add - r5 = rotl_imm(r5, 19u); // 63 rotl - // int8 tile block (Counter ASIC 4.0 research, experimental): 128 mma.m8n8k16 u8 tiles per iteration, both outputs consumed - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t0 mm8 a=r6 b=r5 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t1 mm8 a=r1 b=r1 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t2 mm8 a=r2 b=r5 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t3 mm8 a=r1 b=r5 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t4 mm8 a=r2 b=r0 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t5 mm8 a=r2 b=r6 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t6 mm8 a=r1 b=r4 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t7 mm8 a=r4 b=r6 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t8 mm8 a=r1 b=r1 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t9 mm8 a=r3 b=r7 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t10 mm8 a=r3 b=r0 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t11 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t12 mm8 a=r3 b=r1 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t13 mm8 a=r1 b=r5 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t14 mm8 a=r3 b=r3 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t15 mm8 a=r1 b=r0 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t16 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t17 mm8 a=r0 b=r5 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t18 mm8 a=r1 b=r1 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t19 mm8 a=r3 b=r3 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t20 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t21 mm8 a=r0 b=r3 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t22 mm8 a=r1 b=r7 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t23 mm8 a=r1 b=r2 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t24 mm8 a=r3 b=r4 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t25 mm8 a=r5 b=r0 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t26 mm8 a=r1 b=r0 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t27 mm8 a=r3 b=r2 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t28 mm8 a=r2 b=r4 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t29 mm8 a=r3 b=r4 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t30 mm8 a=r6 b=r5 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t31 mm8 a=r1 b=r7 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t32 mm8 a=r7 b=r6 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t33 mm8 a=r7 b=r1 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t34 mm8 a=r1 b=r0 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t35 mm8 a=r2 b=r4 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t36 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t37 mm8 a=r1 b=r7 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t38 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t39 mm8 a=r6 b=r2 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t40 mm8 a=r6 b=r1 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t41 mm8 a=r6 b=r0 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t42 mm8 a=r7 b=r4 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t43 mm8 a=r6 b=r1 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t44 mm8 a=r0 b=r6 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t45 mm8 a=r6 b=r5 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t46 mm8 a=r6 b=r6 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t47 mm8 a=r7 b=r3 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t48 mm8 a=r7 b=r7 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t49 mm8 a=r4 b=r5 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t50 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t51 mm8 a=r3 b=r4 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t52 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t53 mm8 a=r7 b=r7 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t54 mm8 a=r7 b=r6 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t55 mm8 a=r1 b=r7 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t56 mm8 a=r4 b=r0 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t57 mm8 a=r4 b=r3 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t58 mm8 a=r2 b=r3 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t59 mm8 a=r6 b=r7 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t60 mm8 a=r6 b=r0 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t61 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t62 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t63 mm8 a=r1 b=r6 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t64 mm8 a=r0 b=r5 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t65 mm8 a=r0 b=r1 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t66 mm8 a=r3 b=r5 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t67 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t68 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t69 mm8 a=r6 b=r7 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t70 mm8 a=r1 b=r7 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t71 mm8 a=r3 b=r0 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t72 mm8 a=r4 b=r0 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t73 mm8 a=r4 b=r1 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t74 mm8 a=r5 b=r1 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t75 mm8 a=r4 b=r4 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t76 mm8 a=r5 b=r6 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t77 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t78 mm8 a=r2 b=r5 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t79 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t80 mm8 a=r6 b=r7 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t81 mm8 a=r4 b=r3 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t82 mm8 a=r6 b=r0 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t83 mm8 a=r2 b=r2 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t84 mm8 a=r0 b=r6 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t85 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t86 mm8 a=r3 b=r4 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t87 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t88 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t89 mm8 a=r4 b=r2 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t90 mm8 a=r6 b=r5 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t91 mm8 a=r6 b=r5 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t92 mm8 a=r2 b=r6 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t93 mm8 a=r2 b=r2 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t94 mm8 a=r2 b=r4 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t95 mm8 a=r5 b=r4 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t96 mm8 a=r0 b=r6 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t97 mm8 a=r0 b=r5 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t98 mm8 a=r2 b=r7 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t99 mm8 a=r1 b=r4 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t100 mm8 a=r2 b=r4 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t101 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t102 mm8 a=r5 b=r7 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t103 mm8 a=r4 b=r7 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t104 mm8 a=r2 b=r0 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t105 mm8 a=r6 b=r1 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t106 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t107 mm8 a=r2 b=r1 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t108 mm8 a=r4 b=r6 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t109 mm8 a=r2 b=r7 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t110 mm8 a=r1 b=r0 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t111 mm8 a=r6 b=r5 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t112 mm8 a=r5 b=r7 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t113 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t114 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t115 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t116 mm8 a=r7 b=r2 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t117 mm8 a=r7 b=r6 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t118 mm8 a=r0 b=r1 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t119 mm8 a=r4 b=r3 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t120 mm8 a=r5 b=r3 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t121 mm8 a=r2 b=r7 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t122 mm8 a=r0 b=r1 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t123 mm8 a=r5 b=r2 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t124 mm8 a=r5 b=r1 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t125 mm8 a=r7 b=r2 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t126 mm8 a=r6 b=r3 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t127 mm8 a=r6 b=r7 c=r5 c2=r1 - } - uint32_t lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint32_t hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((uint64_t)hi << 32) | (uint64_t)lo; -} - -// Host-side launch wrappers. Declared in program.h, called from host.cu. -cudaError_t igneum_launch_cache_fill(uint32_t* cache, uint32_t nSegments) { - if (nSegments == 0u) return cudaErrorInvalidValue; - uint32_t block = 256u; - uint32_t grid = (nSegments + block - 1u) / block; - igneum_cache_fill<<>>(cache, nSegments); - return cudaGetLastError(); -} - -cudaError_t igneum_launch_build(uint32_t* ds, const uint32_t* cache, uint32_t nItems) { - if (nItems == 0u) return cudaErrorInvalidValue; - uint32_t block = 256u; - uint32_t grid = (nItems + block - 1u) / block; - igneum_build<<>>(ds, cache, nItems); - return cudaGetLastError(); -} - -cudaError_t igneum_launch_hash(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask, - uint32_t nonces, uint32_t blockWarps) { - if (blockWarps == 0u || blockWarps > 32u) return cudaErrorInvalidValue; - uint32_t block = 32u * blockWarps; - if (nonces == 0u || (nonces % block) != 0u) return cudaErrorInvalidValue; - igneum_hash<<>>(ds, out, baseNonce, mask); - return cudaGetLastError(); -} - -cudaError_t igneum_hash_info(int* numRegs, int* blocksPerSM, uint32_t blockWarps) { - cudaFuncAttributes attr; - cudaError_t e = cudaFuncGetAttributes(&attr, igneum_hash); - if (e != cudaSuccess) return e; - *numRegs = attr.numRegs; - return cudaOccupancyMaxActiveBlocksPerMultiprocessor(blocksPerSM, igneum_hash, (int)(32u * blockWarps), 0); -} diff --git a/proto-cuda/packs-ca4/mx8_mm128/kernel_bound.cl b/proto-cuda/packs-ca4/mx8_mm128/kernel_bound.cl deleted file mode 100644 index 9a99fb311..000000000 --- a/proto-cuda/packs-ca4/mx8_mm128/kernel_bound.cl +++ /dev/null @@ -1,639 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// OpenCL C twin of the Metal kernel for the same seed (see proto-opencl/README.md, WAVEFRONT.md and program.metal). -// Built from source at runtime by proto-opencl/host.c, which passes these defines: -// IGNEUM_GROUP work-group size of igneum_hash, a multiple of 32 (default 32: one work-group = one 32-lane unit) -// IGNEUM_EXCHANGE 0 = local-memory exchange with a barrier (any device, any wave width; the default) -// 1 = sub_group_shuffle_xor (cl_khr_subgroup_shuffle), only with IGNEUM_GROUP 32 and a sub-group size of exactly 32 -// 2 = intel_sub_group_shuffle_xor (cl_intel_subgroups), same condition -// The verification unit is always 32 lanes. A 64-wide hardware wave (AMD GCN/CDNA, RDNA in wave64) runs two units; -// the exchange masks are 1, 2, 4, 8, 16, so every partner lane lies inside the lane's own aligned run of 32. -#ifndef IGNEUM_GROUP -#define IGNEUM_GROUP 32 -#endif -#ifndef IGNEUM_EXCHANGE -#define IGNEUM_EXCHANGE 0 -#endif -#ifdef __OPENCL_VERSION__ -#define IGNEUM_KERNEL_HASH __kernel __attribute__((reqd_work_group_size(IGNEUM_GROUP, 1, 1))) -#define IGNEUM_LOCAL_WORDS(name, n) __local uint name[n] -#if IGNEUM_EXCHANGE == 1 -#ifdef cl_khr_subgroups -#pragma OPENCL EXTENSION cl_khr_subgroups : enable -#endif -#ifdef cl_khr_subgroup_shuffle -#pragma OPENCL EXTENSION cl_khr_subgroup_shuffle : enable -#endif -#elif IGNEUM_EXCHANGE == 2 -#pragma OPENCL EXTENSION cl_intel_subgroups : enable -#endif -#else -// Not an OpenCL compiler: proto-opencl/emu compiles this file as C++ and supplies the built-ins and these two macros. -#include "emu_opencl.h" -#endif - -#if IGNEUM_EXCHANGE == 1 -#define IGNEUM_SHFL_XOR(dst, a, m) dst = sub_group_shuffle_xor((a), (uint)(m)) -#define IGNEUM_BCAST0(dst, a) dst = sub_group_broadcast((a), 0u) -#define IGNEUM_SHFL_IDX(dst, a, i) dst = sub_group_shuffle((a), (uint)(i)) -#elif IGNEUM_EXCHANGE == 2 -#define IGNEUM_SHFL_XOR(dst, a, m) dst = intel_sub_group_shuffle_xor((a), (uint)(m)) -#define IGNEUM_BCAST0(dst, a) dst = sub_group_broadcast((a), 0u) -#define IGNEUM_SHFL_IDX(dst, a, i) dst = intel_sub_group_shuffle((a), (uint)(i)) -#else -// Local-memory exchange. Two buffers of IGNEUM_GROUP words alternate (xk counts exchanges), so one barrier per -// exchange is enough: a lane can only overwrite buffer b at exchange k+2 after passing barrier k+1, and every lane -// reaches barrier k+1 only after its read of buffer b at exchange k. The partner lid ^ m stays inside the lane's -// aligned run of 32 because m < 32. Control flow is uniform, so every work-item reaches every barrier. -#define IGNEUM_SHFL_XOR(dst, a, m) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid ^ (uint)(m))]; xk += 1u; } -#define IGNEUM_BCAST0(dst, a) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid & ~31u)]; xk += 1u; } -#define IGNEUM_SHFL_IDX(dst, a, i) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid & ~31u) + (uint)(i)]; xk += 1u; } -#endif - -// int8 tile reference (Counter ASIC 4.0 research): 12 index shuffles and 32 byte products per lane; the PTX mma.m8n8k16 u8 layout. -#define IGNEUM_MM8_REF(av, bv, d0, d1) { uint row_ = lane >> 2, col0_ = 2u * (lane & 3u); uint acc0_ = 0u, acc1_ = 0u; for (uint j_ = 0u; j_ < 4u; ++j_) { uint aw_, bw0_, bw1_; IGNEUM_SHFL_IDX(aw_, (av), 4u * row_ + j_); IGNEUM_SHFL_IDX(bw0_, (bv), 4u * col0_ + j_); IGNEUM_SHFL_IDX(bw1_, (bv), 4u * (col0_ + 1u) + j_); for (uint t_ = 0u; t_ < 4u; ++t_) { uint ab_ = (aw_ >> (8u * t_)) & 0xffu; acc0_ += ab_ * ((bw0_ >> (8u * t_)) & 0xffu); acc1_ += ab_ * ((bw1_ >> (8u * t_)) & 0xffu); } } d0 = acc0_; d1 = acc1_; } - -static inline uint splitmix32(uint x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -// n is a literal in 1..31 at every call site. OpenCL rotate() rotates left by n modulo 32. -static inline uint rotl_imm(uint x, uint n) { return rotate(x, n); } -// Right rotation by n modulo 32 as a left rotation by (32 - n) modulo 32; n == 0 gives x. -static inline uint rotr_var(uint x, uint n) { return rotate(x, (0u - n) & 31u); } -static inline uint ds_elem(uint i, uint d0, uint d1) { - uint x = i ^ d0; - x *= 0x9E3779B1u; x ^= x >> 15; - x += d1; - x *= 0x85EBCA77u; x ^= x >> 13; - x *= 0xC2B2AE3Du; x ^= x >> 16; - return x; -} - -// Memory-hard dataset core (MEMHARD.md). Cache: 2^26 words in 2^16 segments of 64 chained ChaCha12 lines. -// Item: 8 rounds of 8 x seed-parameterised mixer + one 64-byte cache read, then 8 x final mixer (class v3, mixer multiplier 8, -// docs/plans/mixer-x4.md: the round key of application j of round r is 0x9E3779B9 * (r * m + j + 1)). All parameters are literals. -#define MH_CACHE_LINE_MASK 0x003fffffu -#define MH_SEGMENT_LINES 64u -#define MH_QR(a, b, c, d, r1, r2, r3, r4) { a += b; d ^= a; d = mh_rotl(d, r1); c += d; b ^= c; b = mh_rotl(b, r2); a += b; d ^= a; d = mh_rotl(d, r3); c += d; b ^= c; b = mh_rotl(b, r4); } -static inline uint mh_rotl(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 at every call site - -// y = ChaCha12 core(x) + x -static inline void mh_chacha_block(const uint* x, uint* y) { - for (uint i = 0u; i < 16u; ++i) y[i] = x[i]; - for (uint r = 0u; r < 6u; ++r) { - MH_QR(y[0], y[4], y[8], y[12], 16u, 12u, 8u, 7u) MH_QR(y[1], y[5], y[9], y[13], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[6], y[10], y[14], 16u, 12u, 8u, 7u) MH_QR(y[3], y[7], y[11], y[15], 16u, 12u, 8u, 7u) - MH_QR(y[0], y[5], y[10], y[15], 16u, 12u, 8u, 7u) MH_QR(y[1], y[6], y[11], y[12], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[7], y[8], y[13], 16u, 12u, 8u, 7u) MH_QR(y[3], y[4], y[9], y[14], 16u, 12u, 8u, 7u) - } - for (uint i = 0u; i < 16u; ++i) y[i] += x[i]; -} - -// One cache segment: 64 chained lines written at cache[seg * 1024]. in_j = prev ^ (sigma || K || seg || j || tag), prev_0 = 0. -static inline void mh_cache_segment(__global uint* cache, uint seg) { - uint prev[16]; uint x[16]; uint y[16]; - for (uint i = 0u; i < 16u; ++i) prev[i] = 0u; - for (uint j = 0u; j < MH_SEGMENT_LINES; ++j) { - x[0] = 0x61707865u ^ prev[0]; x[1] = 0x3320646eu ^ prev[1]; x[2] = 0x79622d32u ^ prev[2]; x[3] = 0x6b206574u ^ prev[3]; - x[4] = 0x3067619fu ^ prev[4]; - x[5] = 0x3c269176u ^ prev[5]; - x[6] = 0x84a03b03u ^ prev[6]; - x[7] = 0xf8c63294u ^ prev[7]; - x[8] = 0xff977c5bu ^ prev[8]; - x[9] = 0xe60def3eu ^ prev[9]; - x[10] = 0x63630141u ^ prev[10]; - x[11] = 0xb8fbcb58u ^ prev[11]; - x[12] = seg ^ prev[12]; x[13] = j ^ prev[13]; x[14] = 0x49676e65u ^ prev[14]; x[15] = 0x756d4d48u ^ prev[15]; - mh_chacha_block(x, y); - __global uint* line = cache + ((seg * MH_SEGMENT_LINES + j) * 16u); - for (uint i = 0u; i < 16u; ++i) { line[i] = y[i]; prev[i] = y[i]; } - } -} - -// M_r: per word (s ^ (RC + rk)) * MUL, then a column round and a diagonal round with the seed-drawn rotations. -static inline void mh_mixer(uint* s, uint rk) { - s[0] = (s[0] ^ (0xbab68293u + rk)) * 0x42146205u; - s[1] = (s[1] ^ (0xcc162340u + rk)) * 0x52cbe0fbu; - s[2] = (s[2] ^ (0x6ce151ccu + rk)) * 0x7ecf4a03u; - s[3] = (s[3] ^ (0xe62b8997u + rk)) * 0x6728907fu; - s[4] = (s[4] ^ (0xc9c80297u + rk)) * 0xd81d9751u; - s[5] = (s[5] ^ (0xf74a1654u + rk)) * 0x132952c3u; - s[6] = (s[6] ^ (0x3d704af5u + rk)) * 0xf60de277u; - s[7] = (s[7] ^ (0x3cf522b7u + rk)) * 0x05358035u; - s[8] = (s[8] ^ (0x2b9cac04u + rk)) * 0xbaf6499du; - s[9] = (s[9] ^ (0xa880ac10u + rk)) * 0xe4db9667u; - s[10] = (s[10] ^ (0x13e5dd1du + rk)) * 0x3e98f45du; - s[11] = (s[11] ^ (0x6fc3e233u + rk)) * 0xd0004eddu; - s[12] = (s[12] ^ (0x2d83eeacu + rk)) * 0x2691630du; - s[13] = (s[13] ^ (0x9006e8bfu + rk)) * 0x9beb3bcfu; - s[14] = (s[14] ^ (0x2c4b5362u + rk)) * 0xab310379u; - s[15] = (s[15] ^ (0x31b49ee2u + rk)) * 0x99cfb423u; - MH_QR(s[0], s[4], s[8], s[12], 20u, 20u, 19u, 4u) MH_QR(s[1], s[5], s[9], s[13], 20u, 20u, 19u, 4u) - MH_QR(s[2], s[6], s[10], s[14], 20u, 20u, 19u, 4u) MH_QR(s[3], s[7], s[11], s[15], 20u, 20u, 19u, 4u) - MH_QR(s[0], s[5], s[10], s[15], 26u, 3u, 3u, 27u) MH_QR(s[1], s[6], s[11], s[12], 26u, 3u, 3u, 27u) - MH_QR(s[2], s[7], s[8], s[13], 26u, 3u, 3u, 27u) MH_QR(s[3], s[4], s[9], s[14], 26u, 3u, 3u, 27u) -} - -// Item t: 16 words. s = (K, t * MUL[i] + RC[i]); 8 rounds of 8 x mixer + cache line s[0] & mask; 8 x final mixer. -static inline void mh_item(__global const uint* cache, uint t, uint* s) { - s[0] = 0x3067619fu; - s[1] = 0x3c269176u; - s[2] = 0x84a03b03u; - s[3] = 0xf8c63294u; - s[4] = 0xff977c5bu; - s[5] = 0xe60def3eu; - s[6] = 0x63630141u; - s[7] = 0xb8fbcb58u; - s[8] = t * 0x42146205u + 0xbab68293u; - s[9] = t * 0x52cbe0fbu + 0xcc162340u; - s[10] = t * 0x7ecf4a03u + 0x6ce151ccu; - s[11] = t * 0x6728907fu + 0xe62b8997u; - s[12] = t * 0xd81d9751u + 0xc9c80297u; - s[13] = t * 0x132952c3u + 0xf74a1654u; - s[14] = t * 0xf60de277u + 0x3d704af5u; - s[15] = t * 0x05358035u + 0x3cf522b7u; - for (uint r = 0u; r < 8u; ++r) { - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (r * 8u + j + 1u)); - __global const uint* line = cache + ((s[0] & MH_CACHE_LINE_MASK) * 16u); - for (uint i = 0u; i < 16u; ++i) s[i] ^= line[i]; - } - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (64u + j + 1u)); -} -// dataset[w] without the dataset: derive item w >> 4 and take word w & 15. -static inline uint mh_word(__global const uint* cache, uint w) { uint s[16]; mh_item(cache, w >> 4u, s); return s[w & 15u]; } - -// Memory-hard dataset (MEMHARD.md). One work-item per cache segment; one work-item per 64-byte dataset item. -// The same constants as memhard.h in this pack (one emitter, three dialects). -__kernel void igneum_cache_fill(__global uint* cache, uint nSegments) { - uint seg = (uint)get_global_id(0); - if (seg < nSegments) mh_cache_segment(cache, seg); -} -__kernel void igneum_build(__global uint* ds, __global const uint* cache, uint nItems) { - uint t = (uint)get_global_id(0); - if (t < nItems) { - uint s[16]; - mh_item(cache, t, s); - __global uint* d = ds + ((ulong)t * 16u); - for (uint i = 0u; i < 16u; ++i) d[i] = s[i]; - } -} - -// One hash per work-item. IGNEUM_GROUP is a multiple of 32; lane = lid & 31 and every exchange stays inside the -// lane's own aligned run of 32 work-items, exactly like simd_shuffle_xor inside a 32-wide Metal SIMD group and -// __shfl_xor_sync inside a CUDA warp. Control flow is uniform (no branches at all). -IGNEUM_KERNEL_HASH void igneum_hash(__global const uint* ds, __global ulong* out, uint baseNonce, uint mask) { - uint gid = (uint)get_global_id(0); - uint lid = (uint)get_local_id(0); - uint nonce = baseNonce + gid; - uint r0, r1, r2, r3, r4, r5, r6, r7; -#if IGNEUM_EXCHANGE == 0 - IGNEUM_LOCAL_WORDS(xch, 2 * IGNEUM_GROUP); - uint xk = 0u; -#else - (void)lid; -#endif - uint lane = lid & 31u; - { uint x = nonce ^ 0x67a9a7beu; x += 0x9e3779b9u; x = splitmix32(x); r0 = x ^ 0x1a155b25u; } // SEEDW[0], 0x9e3779b9u * 1u, SEEDW[1] - { uint x = nonce ^ 0x1a155b25u; x += 0x3c6ef372u; x = splitmix32(x); r1 = x ^ 0xfddfb732u; } // SEEDW[1], 0x9e3779b9u * 2u, SEEDW[2] - { uint x = nonce ^ 0xfddfb732u; x += 0xdaa66d2bu; x = splitmix32(x); r2 = x ^ 0x4b5af2e8u; } // SEEDW[2], 0x9e3779b9u * 3u, SEEDW[3] - { uint x = nonce ^ 0x4b5af2e8u; x += 0x78dde6e4u; x = splitmix32(x); r3 = x ^ 0xc55caf33u; } // SEEDW[3], 0x9e3779b9u * 4u, SEEDW[4] - { uint x = nonce ^ 0xc55caf33u; x += 0x1715609du; x = splitmix32(x); r4 = x ^ 0xa27c13b7u; } // SEEDW[4], 0x9e3779b9u * 5u, SEEDW[5] - { uint x = nonce ^ 0xa27c13b7u; x += 0xb54cda56u; x = splitmix32(x); r5 = x ^ 0x06628a48u; } // SEEDW[5], 0x9e3779b9u * 6u, SEEDW[6] - { uint x = nonce ^ 0x06628a48u; x += 0x5384540fu; x = splitmix32(x); r6 = x ^ 0x03852469u; } // SEEDW[6], 0x9e3779b9u * 7u, SEEDW[7] - { uint x = nonce ^ 0x03852469u; x += 0xf1bbcdc8u; x = splitmix32(x); r7 = x ^ 0x67a9a7beu; } // SEEDW[7], 0x9e3779b9u * 8u, SEEDW[0] - - for (uint it = 0u; it < 8u; ++it) { - uint sel = r0; - r2 = r3 * r4 + r2; // 0 mad - r2 = r1 * r1 + r2; // 1 mad - r2 = r3 * r2 + r2; // 2 mad - r3 = r3 ^ r5; // 3 xor - r7 = r7 ^ ds[r2 & mask]; // 4 load - r5 = r5 ^ ds[r7 & mask]; // 5 load - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 8u); r1 = r1 ^ t_; } // 6 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 8u); r7 = r7 ^ t_; } // 7 shfl - r1 = mul_hi(r1, r5); // 8 mulhi - r6 = rotr_var(r6, r3); // 9 rotr - r3 = r3 | r4; // 10 or - r4 = r4 ^ ds[r3 & mask]; // 11 load - r0 = mul_hi(r0, r4); // 12 mulhi - r5 = r5 + r1 + ((((sel >> 30u) & 1u) != 0u) ? 0xd3177981u : 0xc7934706u); // 13 add - r0 = r0 ^ ds[r4 & mask]; // 14 load - r2 = r2 - r4; // 15 sub - r2 = r2 ^ ds[r0 & mask]; // 16 load - r7 = r7 ^ ds[r2 & mask]; // 17 load - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 4u); r7 = r7 ^ t_; } // 18 shfl - r5 = r5 * r0; // 19 mul - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 2u); r3 = r3 ^ t_; } // 20 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 16u); r2 = r2 ^ t_; } // 21 shfl - r6 = mul_hi(r6, r2); // 22 mulhi - r6 = r6 ^ ds[r1 & mask]; // 23 load - r5 = r5 * r0; // 24 mul - r5 = rotl_imm(r5, 19u); // 25 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 2u); r7 = r7 ^ t_; } // 26 shfl - r0 = r0 ^ r5; // 27 xor - r0 = r0 ^ r4; // 28 xor - r3 = r3 - r0; // 29 sub - r5 = r5 * r1; // 30 mul - r7 = r7 ^ ds[r2 & mask]; // 31 load - r1 = r1 ^ ds[r0 & mask]; // 32 load - r5 = r5 ^ r6; // 33 xor - r5 = r5 ^ ds[r1 & mask]; // 34 load - r0 = mul_hi(r0, r5); // 35 mulhi - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 4u); r5 = r5 ^ t_; } // 36 shfl - r7 = r7 ^ ds[r0 & mask]; // 37 load - r3 = r3 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0x230c005cu : 0x75ba2fadu); // 38 add - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 4u); r1 = r1 ^ t_; } // 39 shfl - r2 = r2 ^ r5; // 40 xor - r3 = r6 * r3 + r3; // 41 mad - r6 = r6 - r7; // 42 sub - r7 = r7 ^ r0; // 43 xor - r1 = r1 ^ ds[r7 & mask]; // 44 load - r2 = r2 * r3; // 45 mul - r1 = mul_hi(r1, r5); // 46 mulhi - r4 = r4 - r3; // 47 sub - r2 = rotr_var(r2, r6); // 48 rotr - r3 = r3 ^ ds[r5 & mask]; // 49 load - r1 = r1 + r5 + ((((sel >> 7u) & 1u) != 0u) ? 0x1907970cu : 0x81b8bc2cu); // 50 add - r0 = r0 * r2; // 51 mul - r0 = r0 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x699fd448u : 0x4f92b968u); // 52 add - r1 = r1 + r0 + ((((sel >> 12u) & 1u) != 0u) ? 0x77b1520du : 0x2bb965afu); // 53 add - r7 = rotl_imm(r7, 14u); // 54 rotl - r3 = r3 + r7 + ((((sel >> 1u) & 1u) != 0u) ? 0xa54c55a0u : 0x7b0fe07au); // 55 add - r6 = r6 ^ ds[r7 & mask]; // 56 load - r1 = rotr_var(r1, r5); // 57 rotr - r5 = r5 ^ ds[r4 & mask]; // 58 load - r6 = r6 ^ ds[r2 & mask]; // 59 load - r3 = r5 * r0 + r3; // 60 mad - r5 = r5 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0xad7493e7u : 0xaf9dd72du); // 61 add - r4 = r4 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x1e07c3d9u : 0x89841d87u); // 62 add - r5 = rotl_imm(r5, 19u); // 63 rotl - // int8 tile block (Counter ASIC 4.0 research, experimental): 128 mma.m8n8k16 u8 tiles per iteration, both outputs consumed - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r2 += d0_; r3 += d1_; } // t0 mm8 a=r6 b=r5 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r4 += d0_; r5 += d1_; } // t1 mm8 a=r1 b=r1 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r0 += d0_; r7 += d1_; } // t2 mm8 a=r2 b=r5 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r3 += d1_; } // t3 mm8 a=r1 b=r5 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r2 += d0_; r6 += d1_; } // t4 mm8 a=r2 b=r0 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r7 += d0_; r3 += d1_; } // t5 mm8 a=r2 b=r6 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t6 mm8 a=r1 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r0 += d0_; r4 += d1_; } // t7 mm8 a=r4 b=r6 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r1 += d0_; r5 += d1_; } // t8 mm8 a=r1 b=r1 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r4 += d0_; r6 += d1_; } // t9 mm8 a=r3 b=r7 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r0 += d0_; r4 += d1_; } // t10 mm8 a=r3 b=r0 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r0 += d0_; r6 += d1_; } // t11 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r6 += d0_; r0 += d1_; } // t12 mm8 a=r3 b=r1 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r5 += d1_; } // t13 mm8 a=r1 b=r5 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r3 += d0_; r4 += d1_; } // t14 mm8 a=r3 b=r3 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r5 += d0_; r4 += d1_; } // t15 mm8 a=r1 b=r0 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r0 += d0_; r4 += d1_; } // t16 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r7 += d1_; } // t17 mm8 a=r0 b=r5 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r2 += d0_; r0 += d1_; } // t18 mm8 a=r1 b=r1 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r2 += d0_; r4 += d1_; } // t19 mm8 a=r3 b=r3 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r6 += d0_; r1 += d1_; } // t20 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r6 += d0_; r0 += d1_; } // t21 mm8 a=r0 b=r3 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r4 += d1_; } // t22 mm8 a=r1 b=r7 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r0 += d0_; r3 += d1_; } // t23 mm8 a=r1 b=r2 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r0 += d0_; r2 += d1_; } // t24 mm8 a=r3 b=r4 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r3 += d0_; r7 += d1_; } // t25 mm8 a=r5 b=r0 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t26 mm8 a=r1 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r0 += d0_; r5 += d1_; } // t27 mm8 a=r3 b=r2 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r5 += d1_; } // t28 mm8 a=r2 b=r4 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r1 += d0_; r5 += d1_; } // t29 mm8 a=r3 b=r4 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r6 += d0_; r3 += d1_; } // t30 mm8 a=r6 b=r5 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t31 mm8 a=r1 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r3 += d0_; r0 += d1_; } // t32 mm8 a=r7 b=r6 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r4 += d0_; r6 += d1_; } // t33 mm8 a=r7 b=r1 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r5 += d0_; r1 += d1_; } // t34 mm8 a=r1 b=r0 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t35 mm8 a=r2 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r7 += d0_; r1 += d1_; } // t36 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r0 += d0_; r2 += d1_; } // t37 mm8 a=r1 b=r7 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r2 += d0_; r4 += d1_; } // t38 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r2 += d0_; r5 += d1_; } // t39 mm8 a=r6 b=r2 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r4 += d0_; r1 += d1_; } // t40 mm8 a=r6 b=r1 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r6 += d0_; r2 += d1_; } // t41 mm8 a=r6 b=r0 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r5 += d0_; r1 += d1_; } // t42 mm8 a=r7 b=r4 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r6 += d0_; r0 += d1_; } // t43 mm8 a=r6 b=r1 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r4 += d0_; r1 += d1_; } // t44 mm8 a=r0 b=r6 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r6 += d0_; r5 += d1_; } // t45 mm8 a=r6 b=r5 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r1 += d0_; r0 += d1_; } // t46 mm8 a=r6 b=r6 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r3 += d0_; r1 += d1_; } // t47 mm8 a=r7 b=r3 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t48 mm8 a=r7 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r1 += d0_; r7 += d1_; } // t49 mm8 a=r4 b=r5 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r4 += d0_; r5 += d1_; } // t50 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r0 += d0_; r3 += d1_; } // t51 mm8 a=r3 b=r4 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r7 += d0_; r4 += d1_; } // t52 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r1 += d0_; r3 += d1_; } // t53 mm8 a=r7 b=r7 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t54 mm8 a=r7 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r3 += d0_; r7 += d1_; } // t55 mm8 a=r1 b=r7 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t56 mm8 a=r4 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r6 += d0_; r7 += d1_; } // t57 mm8 a=r4 b=r3 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r6 += d0_; r0 += d1_; } // t58 mm8 a=r2 b=r3 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r7 += d1_; } // t59 mm8 a=r6 b=r7 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t60 mm8 a=r6 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r2 += d0_; r6 += d1_; } // t61 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r6 += d1_; } // t62 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t63 mm8 a=r1 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r5 += d0_; r3 += d1_; } // t64 mm8 a=r0 b=r5 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r2 += d0_; r4 += d1_; } // t65 mm8 a=r0 b=r1 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r1 += d0_; r3 += d1_; } // t66 mm8 a=r3 b=r5 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r3 += d0_; r4 += d1_; } // t67 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r2 += d1_; } // t68 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r4 += d1_; } // t69 mm8 a=r6 b=r7 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r1 += d0_; r5 += d1_; } // t70 mm8 a=r1 b=r7 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r1 += d0_; r0 += d1_; } // t71 mm8 a=r3 b=r0 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r3 += d0_; r6 += d1_; } // t72 mm8 a=r4 b=r0 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r0 += d0_; r2 += d1_; } // t73 mm8 a=r4 b=r1 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r3 += d0_; r4 += d1_; } // t74 mm8 a=r5 b=r1 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r7 += d0_; r3 += d1_; } // t75 mm8 a=r4 b=r4 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r1 += d0_; r7 += d1_; } // t76 mm8 a=r5 b=r6 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r3 += d0_; r2 += d1_; } // t77 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r0 += d0_; r5 += d1_; } // t78 mm8 a=r2 b=r5 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t79 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r3 += d0_; r4 += d1_; } // t80 mm8 a=r6 b=r7 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r1 += d0_; r6 += d1_; } // t81 mm8 a=r4 b=r3 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r1 += d0_; r7 += d1_; } // t82 mm8 a=r6 b=r0 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r1 += d0_; r6 += d1_; } // t83 mm8 a=r2 b=r2 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t84 mm8 a=r0 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t85 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r2 += d0_; r3 += d1_; } // t86 mm8 a=r3 b=r4 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r3 += d0_; r7 += d1_; } // t87 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r5 += d0_; r3 += d1_; } // t88 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r2 += d0_; r1 += d1_; } // t89 mm8 a=r4 b=r2 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r3 += d0_; r6 += d1_; } // t90 mm8 a=r6 b=r5 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r5 += d0_; r4 += d1_; } // t91 mm8 a=r6 b=r5 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r2 += d0_; r3 += d1_; } // t92 mm8 a=r2 b=r6 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r5 += d0_; r7 += d1_; } // t93 mm8 a=r2 b=r2 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r7 += d0_; r3 += d1_; } // t94 mm8 a=r2 b=r4 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r1 += d0_; r6 += d1_; } // t95 mm8 a=r5 b=r4 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r3 += d0_; r1 += d1_; } // t96 mm8 a=r0 b=r6 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r2 += d1_; } // t97 mm8 a=r0 b=r5 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t98 mm8 a=r2 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r0 += d0_; r2 += d1_; } // t99 mm8 a=r1 b=r4 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r2 += d0_; r5 += d1_; } // t100 mm8 a=r2 b=r4 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r7 += d1_; } // t101 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t102 mm8 a=r5 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t103 mm8 a=r4 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r3 += d0_; r7 += d1_; } // t104 mm8 a=r2 b=r0 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r0 += d0_; r5 += d1_; } // t105 mm8 a=r6 b=r1 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r0 += d0_; r5 += d1_; } // t106 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r4 += d0_; r2 += d1_; } // t107 mm8 a=r2 b=r1 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t108 mm8 a=r4 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r5 += d0_; r3 += d1_; } // t109 mm8 a=r2 b=r7 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r0 += d0_; r2 += d1_; } // t110 mm8 a=r1 b=r0 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r4 += d0_; r6 += d1_; } // t111 mm8 a=r6 b=r5 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r4 += d0_; r2 += d1_; } // t112 mm8 a=r5 b=r7 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t113 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t114 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r1 += d0_; r4 += d1_; } // t115 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r5 += d0_; r0 += d1_; } // t116 mm8 a=r7 b=r2 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r7 += d0_; r6 += d1_; } // t117 mm8 a=r7 b=r6 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r2 += d0_; r1 += d1_; } // t118 mm8 a=r0 b=r1 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r5 += d0_; r3 += d1_; } // t119 mm8 a=r4 b=r3 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r7 += d0_; r2 += d1_; } // t120 mm8 a=r5 b=r3 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r1 += d1_; } // t121 mm8 a=r2 b=r7 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r3 += d0_; r0 += d1_; } // t122 mm8 a=r0 b=r1 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r3 += d0_; r5 += d1_; } // t123 mm8 a=r5 b=r2 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t124 mm8 a=r5 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r7 += d0_; r2 += d1_; } // t125 mm8 a=r7 b=r2 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r3 += d0_; r2 += d1_; } // t126 mm8 a=r6 b=r3 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r1 += d1_; } // t127 mm8 a=r6 b=r7 c=r5 c2=r1 - } - uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((ulong)hi << 32) | (ulong)lo; -} - -#if IGNEUM_EXCHANGE != 0 -// Reports the sub-group size this device uses for a work-group of IGNEUM_GROUP items. host.c runs it only when the -// per-kernel query (clGetKernelSubGroupInfoKHR on igneum_hash) is unavailable; that query is preferred because a -// compiler may pick a different wave width per kernel (RDNA: wave32 or wave64). See WAVEFRONT.md. -IGNEUM_KERNEL_HASH void igneum_probe_subgroup(__global uint* out) { - if (get_local_id(0) == 0u) { out[0] = get_sub_group_size(); out[1] = get_num_sub_groups(); } -} -#endif - -// Header-bound variant (bind.rs): the init words come from initw, not SEEDW. Same body as igneum_hash. -IGNEUM_KERNEL_HASH void igneum_hash_bound(__global const uint* ds, __global ulong* out, uint baseNonce, uint mask, __global const uint* initw) { - uint gid = (uint)get_global_id(0); - uint lid = (uint)get_local_id(0); - uint nonce = baseNonce + gid; - uint r0, r1, r2, r3, r4, r5, r6, r7; - uint iw0 = initw[0], iw1 = initw[1], iw2 = initw[2], iw3 = initw[3], iw4 = initw[4], iw5 = initw[5], iw6 = initw[6], iw7 = initw[7]; -#if IGNEUM_EXCHANGE == 0 - IGNEUM_LOCAL_WORDS(xch, 2 * IGNEUM_GROUP); - uint xk = 0u; -#else - (void)lid; -#endif - uint lane = lid & 31u; - { uint x = nonce ^ iw0; x += 0x9e3779b9u * 1u; x = splitmix32(x); r0 = x ^ iw1; } - { uint x = nonce ^ iw1; x += 0x9e3779b9u * 2u; x = splitmix32(x); r1 = x ^ iw2; } - { uint x = nonce ^ iw2; x += 0x9e3779b9u * 3u; x = splitmix32(x); r2 = x ^ iw3; } - { uint x = nonce ^ iw3; x += 0x9e3779b9u * 4u; x = splitmix32(x); r3 = x ^ iw4; } - { uint x = nonce ^ iw4; x += 0x9e3779b9u * 5u; x = splitmix32(x); r4 = x ^ iw5; } - { uint x = nonce ^ iw5; x += 0x9e3779b9u * 6u; x = splitmix32(x); r5 = x ^ iw6; } - { uint x = nonce ^ iw6; x += 0x9e3779b9u * 7u; x = splitmix32(x); r6 = x ^ iw7; } - { uint x = nonce ^ iw7; x += 0x9e3779b9u * 8u; x = splitmix32(x); r7 = x ^ iw0; } - - for (uint it = 0u; it < 8u; ++it) { - uint sel = r0; - r2 = r3 * r4 + r2; // 0 mad - r2 = r1 * r1 + r2; // 1 mad - r2 = r3 * r2 + r2; // 2 mad - r3 = r3 ^ r5; // 3 xor - r7 = r7 ^ ds[r2 & mask]; // 4 load - r5 = r5 ^ ds[r7 & mask]; // 5 load - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 8u); r1 = r1 ^ t_; } // 6 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 8u); r7 = r7 ^ t_; } // 7 shfl - r1 = mul_hi(r1, r5); // 8 mulhi - r6 = rotr_var(r6, r3); // 9 rotr - r3 = r3 | r4; // 10 or - r4 = r4 ^ ds[r3 & mask]; // 11 load - r0 = mul_hi(r0, r4); // 12 mulhi - r5 = r5 + r1 + ((((sel >> 30u) & 1u) != 0u) ? 0xd3177981u : 0xc7934706u); // 13 add - r0 = r0 ^ ds[r4 & mask]; // 14 load - r2 = r2 - r4; // 15 sub - r2 = r2 ^ ds[r0 & mask]; // 16 load - r7 = r7 ^ ds[r2 & mask]; // 17 load - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 4u); r7 = r7 ^ t_; } // 18 shfl - r5 = r5 * r0; // 19 mul - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 2u); r3 = r3 ^ t_; } // 20 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 16u); r2 = r2 ^ t_; } // 21 shfl - r6 = mul_hi(r6, r2); // 22 mulhi - r6 = r6 ^ ds[r1 & mask]; // 23 load - r5 = r5 * r0; // 24 mul - r5 = rotl_imm(r5, 19u); // 25 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 2u); r7 = r7 ^ t_; } // 26 shfl - r0 = r0 ^ r5; // 27 xor - r0 = r0 ^ r4; // 28 xor - r3 = r3 - r0; // 29 sub - r5 = r5 * r1; // 30 mul - r7 = r7 ^ ds[r2 & mask]; // 31 load - r1 = r1 ^ ds[r0 & mask]; // 32 load - r5 = r5 ^ r6; // 33 xor - r5 = r5 ^ ds[r1 & mask]; // 34 load - r0 = mul_hi(r0, r5); // 35 mulhi - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 4u); r5 = r5 ^ t_; } // 36 shfl - r7 = r7 ^ ds[r0 & mask]; // 37 load - r3 = r3 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0x230c005cu : 0x75ba2fadu); // 38 add - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 4u); r1 = r1 ^ t_; } // 39 shfl - r2 = r2 ^ r5; // 40 xor - r3 = r6 * r3 + r3; // 41 mad - r6 = r6 - r7; // 42 sub - r7 = r7 ^ r0; // 43 xor - r1 = r1 ^ ds[r7 & mask]; // 44 load - r2 = r2 * r3; // 45 mul - r1 = mul_hi(r1, r5); // 46 mulhi - r4 = r4 - r3; // 47 sub - r2 = rotr_var(r2, r6); // 48 rotr - r3 = r3 ^ ds[r5 & mask]; // 49 load - r1 = r1 + r5 + ((((sel >> 7u) & 1u) != 0u) ? 0x1907970cu : 0x81b8bc2cu); // 50 add - r0 = r0 * r2; // 51 mul - r0 = r0 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x699fd448u : 0x4f92b968u); // 52 add - r1 = r1 + r0 + ((((sel >> 12u) & 1u) != 0u) ? 0x77b1520du : 0x2bb965afu); // 53 add - r7 = rotl_imm(r7, 14u); // 54 rotl - r3 = r3 + r7 + ((((sel >> 1u) & 1u) != 0u) ? 0xa54c55a0u : 0x7b0fe07au); // 55 add - r6 = r6 ^ ds[r7 & mask]; // 56 load - r1 = rotr_var(r1, r5); // 57 rotr - r5 = r5 ^ ds[r4 & mask]; // 58 load - r6 = r6 ^ ds[r2 & mask]; // 59 load - r3 = r5 * r0 + r3; // 60 mad - r5 = r5 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0xad7493e7u : 0xaf9dd72du); // 61 add - r4 = r4 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x1e07c3d9u : 0x89841d87u); // 62 add - r5 = rotl_imm(r5, 19u); // 63 rotl - // int8 tile block (Counter ASIC 4.0 research, experimental): 128 mma.m8n8k16 u8 tiles per iteration, both outputs consumed - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r2 += d0_; r3 += d1_; } // t0 mm8 a=r6 b=r5 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r4 += d0_; r5 += d1_; } // t1 mm8 a=r1 b=r1 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r0 += d0_; r7 += d1_; } // t2 mm8 a=r2 b=r5 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r3 += d1_; } // t3 mm8 a=r1 b=r5 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r2 += d0_; r6 += d1_; } // t4 mm8 a=r2 b=r0 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r7 += d0_; r3 += d1_; } // t5 mm8 a=r2 b=r6 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t6 mm8 a=r1 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r0 += d0_; r4 += d1_; } // t7 mm8 a=r4 b=r6 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r1 += d0_; r5 += d1_; } // t8 mm8 a=r1 b=r1 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r4 += d0_; r6 += d1_; } // t9 mm8 a=r3 b=r7 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r0 += d0_; r4 += d1_; } // t10 mm8 a=r3 b=r0 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r0 += d0_; r6 += d1_; } // t11 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r6 += d0_; r0 += d1_; } // t12 mm8 a=r3 b=r1 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r5 += d1_; } // t13 mm8 a=r1 b=r5 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r3 += d0_; r4 += d1_; } // t14 mm8 a=r3 b=r3 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r5 += d0_; r4 += d1_; } // t15 mm8 a=r1 b=r0 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r0 += d0_; r4 += d1_; } // t16 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r7 += d1_; } // t17 mm8 a=r0 b=r5 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r2 += d0_; r0 += d1_; } // t18 mm8 a=r1 b=r1 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r2 += d0_; r4 += d1_; } // t19 mm8 a=r3 b=r3 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r6 += d0_; r1 += d1_; } // t20 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r6 += d0_; r0 += d1_; } // t21 mm8 a=r0 b=r3 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r4 += d1_; } // t22 mm8 a=r1 b=r7 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r0 += d0_; r3 += d1_; } // t23 mm8 a=r1 b=r2 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r0 += d0_; r2 += d1_; } // t24 mm8 a=r3 b=r4 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r3 += d0_; r7 += d1_; } // t25 mm8 a=r5 b=r0 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t26 mm8 a=r1 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r0 += d0_; r5 += d1_; } // t27 mm8 a=r3 b=r2 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r5 += d1_; } // t28 mm8 a=r2 b=r4 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r1 += d0_; r5 += d1_; } // t29 mm8 a=r3 b=r4 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r6 += d0_; r3 += d1_; } // t30 mm8 a=r6 b=r5 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t31 mm8 a=r1 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r3 += d0_; r0 += d1_; } // t32 mm8 a=r7 b=r6 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r4 += d0_; r6 += d1_; } // t33 mm8 a=r7 b=r1 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r5 += d0_; r1 += d1_; } // t34 mm8 a=r1 b=r0 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t35 mm8 a=r2 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r7 += d0_; r1 += d1_; } // t36 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r0 += d0_; r2 += d1_; } // t37 mm8 a=r1 b=r7 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r2 += d0_; r4 += d1_; } // t38 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r2 += d0_; r5 += d1_; } // t39 mm8 a=r6 b=r2 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r4 += d0_; r1 += d1_; } // t40 mm8 a=r6 b=r1 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r6 += d0_; r2 += d1_; } // t41 mm8 a=r6 b=r0 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r5 += d0_; r1 += d1_; } // t42 mm8 a=r7 b=r4 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r6 += d0_; r0 += d1_; } // t43 mm8 a=r6 b=r1 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r4 += d0_; r1 += d1_; } // t44 mm8 a=r0 b=r6 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r6 += d0_; r5 += d1_; } // t45 mm8 a=r6 b=r5 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r1 += d0_; r0 += d1_; } // t46 mm8 a=r6 b=r6 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r3 += d0_; r1 += d1_; } // t47 mm8 a=r7 b=r3 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t48 mm8 a=r7 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r1 += d0_; r7 += d1_; } // t49 mm8 a=r4 b=r5 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r4 += d0_; r5 += d1_; } // t50 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r0 += d0_; r3 += d1_; } // t51 mm8 a=r3 b=r4 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r7 += d0_; r4 += d1_; } // t52 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r1 += d0_; r3 += d1_; } // t53 mm8 a=r7 b=r7 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t54 mm8 a=r7 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r3 += d0_; r7 += d1_; } // t55 mm8 a=r1 b=r7 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t56 mm8 a=r4 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r6 += d0_; r7 += d1_; } // t57 mm8 a=r4 b=r3 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r6 += d0_; r0 += d1_; } // t58 mm8 a=r2 b=r3 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r7 += d1_; } // t59 mm8 a=r6 b=r7 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t60 mm8 a=r6 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r2 += d0_; r6 += d1_; } // t61 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r6 += d1_; } // t62 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t63 mm8 a=r1 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r5 += d0_; r3 += d1_; } // t64 mm8 a=r0 b=r5 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r2 += d0_; r4 += d1_; } // t65 mm8 a=r0 b=r1 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r1 += d0_; r3 += d1_; } // t66 mm8 a=r3 b=r5 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r3 += d0_; r4 += d1_; } // t67 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r2 += d1_; } // t68 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r4 += d1_; } // t69 mm8 a=r6 b=r7 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r1 += d0_; r5 += d1_; } // t70 mm8 a=r1 b=r7 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r1 += d0_; r0 += d1_; } // t71 mm8 a=r3 b=r0 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r3 += d0_; r6 += d1_; } // t72 mm8 a=r4 b=r0 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r0 += d0_; r2 += d1_; } // t73 mm8 a=r4 b=r1 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r3 += d0_; r4 += d1_; } // t74 mm8 a=r5 b=r1 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r7 += d0_; r3 += d1_; } // t75 mm8 a=r4 b=r4 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r1 += d0_; r7 += d1_; } // t76 mm8 a=r5 b=r6 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r3 += d0_; r2 += d1_; } // t77 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r0 += d0_; r5 += d1_; } // t78 mm8 a=r2 b=r5 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t79 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r3 += d0_; r4 += d1_; } // t80 mm8 a=r6 b=r7 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r1 += d0_; r6 += d1_; } // t81 mm8 a=r4 b=r3 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r1 += d0_; r7 += d1_; } // t82 mm8 a=r6 b=r0 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r1 += d0_; r6 += d1_; } // t83 mm8 a=r2 b=r2 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t84 mm8 a=r0 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t85 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r2 += d0_; r3 += d1_; } // t86 mm8 a=r3 b=r4 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r3 += d0_; r7 += d1_; } // t87 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r5 += d0_; r3 += d1_; } // t88 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r2 += d0_; r1 += d1_; } // t89 mm8 a=r4 b=r2 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r3 += d0_; r6 += d1_; } // t90 mm8 a=r6 b=r5 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r5 += d0_; r4 += d1_; } // t91 mm8 a=r6 b=r5 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r2 += d0_; r3 += d1_; } // t92 mm8 a=r2 b=r6 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r5 += d0_; r7 += d1_; } // t93 mm8 a=r2 b=r2 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r7 += d0_; r3 += d1_; } // t94 mm8 a=r2 b=r4 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r1 += d0_; r6 += d1_; } // t95 mm8 a=r5 b=r4 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r3 += d0_; r1 += d1_; } // t96 mm8 a=r0 b=r6 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r2 += d1_; } // t97 mm8 a=r0 b=r5 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t98 mm8 a=r2 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r0 += d0_; r2 += d1_; } // t99 mm8 a=r1 b=r4 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r2 += d0_; r5 += d1_; } // t100 mm8 a=r2 b=r4 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r7 += d1_; } // t101 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t102 mm8 a=r5 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t103 mm8 a=r4 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r3 += d0_; r7 += d1_; } // t104 mm8 a=r2 b=r0 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r0 += d0_; r5 += d1_; } // t105 mm8 a=r6 b=r1 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r0 += d0_; r5 += d1_; } // t106 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r4 += d0_; r2 += d1_; } // t107 mm8 a=r2 b=r1 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t108 mm8 a=r4 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r5 += d0_; r3 += d1_; } // t109 mm8 a=r2 b=r7 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r0 += d0_; r2 += d1_; } // t110 mm8 a=r1 b=r0 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r4 += d0_; r6 += d1_; } // t111 mm8 a=r6 b=r5 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r4 += d0_; r2 += d1_; } // t112 mm8 a=r5 b=r7 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t113 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t114 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r1 += d0_; r4 += d1_; } // t115 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r5 += d0_; r0 += d1_; } // t116 mm8 a=r7 b=r2 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r7 += d0_; r6 += d1_; } // t117 mm8 a=r7 b=r6 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r2 += d0_; r1 += d1_; } // t118 mm8 a=r0 b=r1 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r5 += d0_; r3 += d1_; } // t119 mm8 a=r4 b=r3 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r7 += d0_; r2 += d1_; } // t120 mm8 a=r5 b=r3 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r1 += d1_; } // t121 mm8 a=r2 b=r7 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r3 += d0_; r0 += d1_; } // t122 mm8 a=r0 b=r1 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r3 += d0_; r5 += d1_; } // t123 mm8 a=r5 b=r2 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t124 mm8 a=r5 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r7 += d0_; r2 += d1_; } // t125 mm8 a=r7 b=r2 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r3 += d0_; r2 += d1_; } // t126 mm8 a=r6 b=r3 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r1 += d1_; } // t127 mm8 a=r6 b=r7 c=r5 c2=r1 - } - uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((ulong)hi << 32) | (ulong)lo; -} diff --git a/proto-cuda/packs-ca4/mx8_mm128/kernel_bound.cu b/proto-cuda/packs-ca4/mx8_mm128/kernel_bound.cu deleted file mode 100644 index 3a4255bc2..000000000 --- a/proto-cuda/packs-ca4/mx8_mm128/kernel_bound.cu +++ /dev/null @@ -1,252 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// Header-bound twin of igneum_hash in kernel.cu: the init words come from a kernel argument, not SEEDW. -// Host declarations (also in program_bound.h if present): -// struct IgneumInitWords { uint32_t w[8]; }; -// cudaError_t igneum_launch_hash_bound(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask, -// IgneumInitWords iw, uint32_t nonces, uint32_t blockWarps); -// cudaError_t igneum_hash_bound_info(int* numRegs, int* blocksPerSM, uint32_t blockWarps); -#include -#include -#include "program.h" - -struct IgneumInitWords { uint32_t w[8]; }; - -__device__ __forceinline__ uint32_t splitmix32(uint32_t x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -__device__ __forceinline__ uint32_t rotl_imm(uint32_t x, uint32_t n) { return (x << n) | (x >> (32u - n)); } -__device__ __forceinline__ uint32_t rotr_var(uint32_t x, uint32_t n) { n &= 31u; return (x >> n) | (x << ((32u - n) & 31u)); } - -__global__ void igneum_hash_bound(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask, IgneumInitWords iw) { - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t nonce = baseNonce + gid; - uint32_t r0, r1, r2, r3, r4, r5, r6, r7; - { uint32_t x = nonce ^ iw.w[0]; x += 0x9e3779b9u * 1u; x = splitmix32(x); r0 = x ^ iw.w[1]; } - { uint32_t x = nonce ^ iw.w[1]; x += 0x9e3779b9u * 2u; x = splitmix32(x); r1 = x ^ iw.w[2]; } - { uint32_t x = nonce ^ iw.w[2]; x += 0x9e3779b9u * 3u; x = splitmix32(x); r2 = x ^ iw.w[3]; } - { uint32_t x = nonce ^ iw.w[3]; x += 0x9e3779b9u * 4u; x = splitmix32(x); r3 = x ^ iw.w[4]; } - { uint32_t x = nonce ^ iw.w[4]; x += 0x9e3779b9u * 5u; x = splitmix32(x); r4 = x ^ iw.w[5]; } - { uint32_t x = nonce ^ iw.w[5]; x += 0x9e3779b9u * 6u; x = splitmix32(x); r5 = x ^ iw.w[6]; } - { uint32_t x = nonce ^ iw.w[6]; x += 0x9e3779b9u * 7u; x = splitmix32(x); r6 = x ^ iw.w[7]; } - { uint32_t x = nonce ^ iw.w[7]; x += 0x9e3779b9u * 8u; x = splitmix32(x); r7 = x ^ iw.w[0]; } - - for (uint32_t it = 0u; it < 8u; ++it) { - uint32_t sel = r0; - r2 = r3 * r4 + r2; // 0 mad - r2 = r1 * r1 + r2; // 1 mad - r2 = r3 * r2 + r2; // 2 mad - r3 = r3 ^ r5; // 3 xor - r7 = r7 ^ ds[r2 & mask]; // 4 load - r5 = r5 ^ ds[r7 & mask]; // 5 load - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r4, 8); // 6 shfl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r3, 8); // 7 shfl - r1 = __umulhi(r1, r5); // 8 mulhi - r6 = rotr_var(r6, r3); // 9 rotr - r3 = r3 | r4; // 10 or - r4 = r4 ^ ds[r3 & mask]; // 11 load - r0 = __umulhi(r0, r4); // 12 mulhi - r5 = r5 + r1 + ((((sel >> 30u) & 1u) != 0u) ? 0xd3177981u : 0xc7934706u); // 13 add - r0 = r0 ^ ds[r4 & mask]; // 14 load - r2 = r2 - r4; // 15 sub - r2 = r2 ^ ds[r0 & mask]; // 16 load - r7 = r7 ^ ds[r2 & mask]; // 17 load - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r3, 4); // 18 shfl - r5 = r5 * r0; // 19 mul - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r4, 2); // 20 shfl - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r4, 16); // 21 shfl - r6 = __umulhi(r6, r2); // 22 mulhi - r6 = r6 ^ ds[r1 & mask]; // 23 load - r5 = r5 * r0; // 24 mul - r5 = rotl_imm(r5, 19u); // 25 rotl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r6, 2); // 26 shfl - r0 = r0 ^ r5; // 27 xor - r0 = r0 ^ r4; // 28 xor - r3 = r3 - r0; // 29 sub - r5 = r5 * r1; // 30 mul - r7 = r7 ^ ds[r2 & mask]; // 31 load - r1 = r1 ^ ds[r0 & mask]; // 32 load - r5 = r5 ^ r6; // 33 xor - r5 = r5 ^ ds[r1 & mask]; // 34 load - r0 = __umulhi(r0, r5); // 35 mulhi - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r2, 4); // 36 shfl - r7 = r7 ^ ds[r0 & mask]; // 37 load - r3 = r3 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0x230c005cu : 0x75ba2fadu); // 38 add - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r5, 4); // 39 shfl - r2 = r2 ^ r5; // 40 xor - r3 = r6 * r3 + r3; // 41 mad - r6 = r6 - r7; // 42 sub - r7 = r7 ^ r0; // 43 xor - r1 = r1 ^ ds[r7 & mask]; // 44 load - r2 = r2 * r3; // 45 mul - r1 = __umulhi(r1, r5); // 46 mulhi - r4 = r4 - r3; // 47 sub - r2 = rotr_var(r2, r6); // 48 rotr - r3 = r3 ^ ds[r5 & mask]; // 49 load - r1 = r1 + r5 + ((((sel >> 7u) & 1u) != 0u) ? 0x1907970cu : 0x81b8bc2cu); // 50 add - r0 = r0 * r2; // 51 mul - r0 = r0 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x699fd448u : 0x4f92b968u); // 52 add - r1 = r1 + r0 + ((((sel >> 12u) & 1u) != 0u) ? 0x77b1520du : 0x2bb965afu); // 53 add - r7 = rotl_imm(r7, 14u); // 54 rotl - r3 = r3 + r7 + ((((sel >> 1u) & 1u) != 0u) ? 0xa54c55a0u : 0x7b0fe07au); // 55 add - r6 = r6 ^ ds[r7 & mask]; // 56 load - r1 = rotr_var(r1, r5); // 57 rotr - r5 = r5 ^ ds[r4 & mask]; // 58 load - r6 = r6 ^ ds[r2 & mask]; // 59 load - r3 = r5 * r0 + r3; // 60 mad - r5 = r5 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0xad7493e7u : 0xaf9dd72du); // 61 add - r4 = r4 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x1e07c3d9u : 0x89841d87u); // 62 add - r5 = rotl_imm(r5, 19u); // 63 rotl - // int8 tile block (Counter ASIC 4.0 research, experimental): 128 mma.m8n8k16 u8 tiles per iteration, both outputs consumed - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t0 mm8 a=r6 b=r5 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t1 mm8 a=r1 b=r1 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t2 mm8 a=r2 b=r5 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t3 mm8 a=r1 b=r5 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t4 mm8 a=r2 b=r0 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t5 mm8 a=r2 b=r6 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t6 mm8 a=r1 b=r4 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t7 mm8 a=r4 b=r6 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t8 mm8 a=r1 b=r1 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t9 mm8 a=r3 b=r7 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t10 mm8 a=r3 b=r0 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t11 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t12 mm8 a=r3 b=r1 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t13 mm8 a=r1 b=r5 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t14 mm8 a=r3 b=r3 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t15 mm8 a=r1 b=r0 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t16 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t17 mm8 a=r0 b=r5 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t18 mm8 a=r1 b=r1 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t19 mm8 a=r3 b=r3 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t20 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t21 mm8 a=r0 b=r3 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t22 mm8 a=r1 b=r7 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t23 mm8 a=r1 b=r2 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t24 mm8 a=r3 b=r4 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t25 mm8 a=r5 b=r0 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t26 mm8 a=r1 b=r0 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t27 mm8 a=r3 b=r2 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t28 mm8 a=r2 b=r4 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t29 mm8 a=r3 b=r4 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t30 mm8 a=r6 b=r5 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t31 mm8 a=r1 b=r7 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t32 mm8 a=r7 b=r6 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t33 mm8 a=r7 b=r1 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t34 mm8 a=r1 b=r0 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t35 mm8 a=r2 b=r4 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t36 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t37 mm8 a=r1 b=r7 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t38 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t39 mm8 a=r6 b=r2 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t40 mm8 a=r6 b=r1 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t41 mm8 a=r6 b=r0 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t42 mm8 a=r7 b=r4 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t43 mm8 a=r6 b=r1 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t44 mm8 a=r0 b=r6 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t45 mm8 a=r6 b=r5 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t46 mm8 a=r6 b=r6 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t47 mm8 a=r7 b=r3 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t48 mm8 a=r7 b=r7 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t49 mm8 a=r4 b=r5 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t50 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t51 mm8 a=r3 b=r4 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t52 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t53 mm8 a=r7 b=r7 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t54 mm8 a=r7 b=r6 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t55 mm8 a=r1 b=r7 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t56 mm8 a=r4 b=r0 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t57 mm8 a=r4 b=r3 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t58 mm8 a=r2 b=r3 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t59 mm8 a=r6 b=r7 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t60 mm8 a=r6 b=r0 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t61 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t62 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t63 mm8 a=r1 b=r6 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t64 mm8 a=r0 b=r5 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t65 mm8 a=r0 b=r1 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t66 mm8 a=r3 b=r5 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t67 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t68 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t69 mm8 a=r6 b=r7 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t70 mm8 a=r1 b=r7 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t71 mm8 a=r3 b=r0 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t72 mm8 a=r4 b=r0 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t73 mm8 a=r4 b=r1 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t74 mm8 a=r5 b=r1 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t75 mm8 a=r4 b=r4 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t76 mm8 a=r5 b=r6 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t77 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t78 mm8 a=r2 b=r5 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t79 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t80 mm8 a=r6 b=r7 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t81 mm8 a=r4 b=r3 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t82 mm8 a=r6 b=r0 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t83 mm8 a=r2 b=r2 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t84 mm8 a=r0 b=r6 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t85 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t86 mm8 a=r3 b=r4 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t87 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t88 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t89 mm8 a=r4 b=r2 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t90 mm8 a=r6 b=r5 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t91 mm8 a=r6 b=r5 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t92 mm8 a=r2 b=r6 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t93 mm8 a=r2 b=r2 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t94 mm8 a=r2 b=r4 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t95 mm8 a=r5 b=r4 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t96 mm8 a=r0 b=r6 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t97 mm8 a=r0 b=r5 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t98 mm8 a=r2 b=r7 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t99 mm8 a=r1 b=r4 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t100 mm8 a=r2 b=r4 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t101 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t102 mm8 a=r5 b=r7 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t103 mm8 a=r4 b=r7 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t104 mm8 a=r2 b=r0 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t105 mm8 a=r6 b=r1 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t106 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t107 mm8 a=r2 b=r1 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t108 mm8 a=r4 b=r6 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t109 mm8 a=r2 b=r7 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t110 mm8 a=r1 b=r0 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t111 mm8 a=r6 b=r5 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t112 mm8 a=r5 b=r7 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t113 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t114 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t115 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t116 mm8 a=r7 b=r2 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t117 mm8 a=r7 b=r6 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t118 mm8 a=r0 b=r1 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t119 mm8 a=r4 b=r3 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t120 mm8 a=r5 b=r3 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t121 mm8 a=r2 b=r7 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t122 mm8 a=r0 b=r1 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t123 mm8 a=r5 b=r2 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t124 mm8 a=r5 b=r1 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t125 mm8 a=r7 b=r2 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t126 mm8 a=r6 b=r3 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t127 mm8 a=r6 b=r7 c=r5 c2=r1 - } - uint32_t lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint32_t hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((uint64_t)hi << 32) | (uint64_t)lo; -} - -cudaError_t igneum_launch_hash_bound(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask, - IgneumInitWords iw, uint32_t nonces, uint32_t blockWarps) { - if (blockWarps == 0u || blockWarps > 32u) return cudaErrorInvalidValue; - uint32_t block = 32u * blockWarps; - if (nonces == 0u || (nonces % block) != 0u) return cudaErrorInvalidValue; - igneum_hash_bound<<>>(ds, out, baseNonce, mask, iw); - return cudaGetLastError(); -} - -cudaError_t igneum_hash_bound_info(int* numRegs, int* blocksPerSM, uint32_t blockWarps) { - cudaFuncAttributes attr; - cudaError_t e = cudaFuncGetAttributes(&attr, igneum_hash_bound); - if (e != cudaSuccess) return e; - *numRegs = attr.numRegs; - return cudaOccupancyMaxActiveBlocksPerMultiprocessor(blocksPerSM, igneum_hash_bound, (int)(32u * blockWarps), 0); -} diff --git a/proto-cuda/packs-ca4/mx8_mm128/memhard.h b/proto-cuda/packs-ca4/mx8_mm128/memhard.h deleted file mode 100644 index f7f34c732..000000000 --- a/proto-cuda/packs-ca4/mx8_mm128/memhard.h +++ /dev/null @@ -1,109 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// Memory-hard dataset core, the same text that the Mac's Metal kernels and CPU verifier were checked against. -// Included by kernel.cu (device), host.cu (host reference) and proto-opencl/host.c (C99 host reference). -// See proto-metal/MEMHARD.md for the construction. kernel.cl carries the same text in OpenCL C. -#pragma once -#ifdef __cplusplus -#include -#else -#include -#endif -#if defined(__CUDACC__) -#define IGNEUM_HD __host__ __device__ __forceinline__ -#elif defined(_MSC_VER) && !defined(__cplusplus) -#define IGNEUM_HD static __inline -#else -#define IGNEUM_HD static inline -#endif -// Memory-hard dataset core (MEMHARD.md). Cache: 2^26 words in 2^16 segments of 64 chained ChaCha12 lines. -// Item: 8 rounds of 8 x seed-parameterised mixer + one 64-byte cache read, then 8 x final mixer (class v3, mixer multiplier 8, -// docs/plans/mixer-x4.md: the round key of application j of round r is 0x9E3779B9 * (r * m + j + 1)). All parameters are literals. -#define MH_CACHE_LINE_MASK 0x003fffffu -#define MH_SEGMENT_LINES 64u -#define MH_QR(a, b, c, d, r1, r2, r3, r4) { a += b; d ^= a; d = mh_rotl(d, r1); c += d; b ^= c; b = mh_rotl(b, r2); a += b; d ^= a; d = mh_rotl(d, r3); c += d; b ^= c; b = mh_rotl(b, r4); } -IGNEUM_HD uint32_t mh_rotl(uint32_t x, uint32_t n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 at every call site - -// y = ChaCha12 core(x) + x -IGNEUM_HD void mh_chacha_block(const uint32_t* x, uint32_t* y) { - for (uint32_t i = 0u; i < 16u; ++i) y[i] = x[i]; - for (uint32_t r = 0u; r < 6u; ++r) { - MH_QR(y[0], y[4], y[8], y[12], 16u, 12u, 8u, 7u) MH_QR(y[1], y[5], y[9], y[13], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[6], y[10], y[14], 16u, 12u, 8u, 7u) MH_QR(y[3], y[7], y[11], y[15], 16u, 12u, 8u, 7u) - MH_QR(y[0], y[5], y[10], y[15], 16u, 12u, 8u, 7u) MH_QR(y[1], y[6], y[11], y[12], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[7], y[8], y[13], 16u, 12u, 8u, 7u) MH_QR(y[3], y[4], y[9], y[14], 16u, 12u, 8u, 7u) - } - for (uint32_t i = 0u; i < 16u; ++i) y[i] += x[i]; -} - -// One cache segment: 64 chained lines written at cache[seg * 1024]. in_j = prev ^ (sigma || K || seg || j || tag), prev_0 = 0. -IGNEUM_HD void mh_cache_segment(uint32_t* cache, uint32_t seg) { - uint32_t prev[16]; uint32_t x[16]; uint32_t y[16]; - for (uint32_t i = 0u; i < 16u; ++i) prev[i] = 0u; - for (uint32_t j = 0u; j < MH_SEGMENT_LINES; ++j) { - x[0] = 0x61707865u ^ prev[0]; x[1] = 0x3320646eu ^ prev[1]; x[2] = 0x79622d32u ^ prev[2]; x[3] = 0x6b206574u ^ prev[3]; - x[4] = 0x3067619fu ^ prev[4]; - x[5] = 0x3c269176u ^ prev[5]; - x[6] = 0x84a03b03u ^ prev[6]; - x[7] = 0xf8c63294u ^ prev[7]; - x[8] = 0xff977c5bu ^ prev[8]; - x[9] = 0xe60def3eu ^ prev[9]; - x[10] = 0x63630141u ^ prev[10]; - x[11] = 0xb8fbcb58u ^ prev[11]; - x[12] = seg ^ prev[12]; x[13] = j ^ prev[13]; x[14] = 0x49676e65u ^ prev[14]; x[15] = 0x756d4d48u ^ prev[15]; - mh_chacha_block(x, y); - uint32_t* line = cache + ((seg * MH_SEGMENT_LINES + j) * 16u); - for (uint32_t i = 0u; i < 16u; ++i) { line[i] = y[i]; prev[i] = y[i]; } - } -} - -// M_r: per word (s ^ (RC + rk)) * MUL, then a column round and a diagonal round with the seed-drawn rotations. -IGNEUM_HD void mh_mixer(uint32_t* s, uint32_t rk) { - s[0] = (s[0] ^ (0xbab68293u + rk)) * 0x42146205u; - s[1] = (s[1] ^ (0xcc162340u + rk)) * 0x52cbe0fbu; - s[2] = (s[2] ^ (0x6ce151ccu + rk)) * 0x7ecf4a03u; - s[3] = (s[3] ^ (0xe62b8997u + rk)) * 0x6728907fu; - s[4] = (s[4] ^ (0xc9c80297u + rk)) * 0xd81d9751u; - s[5] = (s[5] ^ (0xf74a1654u + rk)) * 0x132952c3u; - s[6] = (s[6] ^ (0x3d704af5u + rk)) * 0xf60de277u; - s[7] = (s[7] ^ (0x3cf522b7u + rk)) * 0x05358035u; - s[8] = (s[8] ^ (0x2b9cac04u + rk)) * 0xbaf6499du; - s[9] = (s[9] ^ (0xa880ac10u + rk)) * 0xe4db9667u; - s[10] = (s[10] ^ (0x13e5dd1du + rk)) * 0x3e98f45du; - s[11] = (s[11] ^ (0x6fc3e233u + rk)) * 0xd0004eddu; - s[12] = (s[12] ^ (0x2d83eeacu + rk)) * 0x2691630du; - s[13] = (s[13] ^ (0x9006e8bfu + rk)) * 0x9beb3bcfu; - s[14] = (s[14] ^ (0x2c4b5362u + rk)) * 0xab310379u; - s[15] = (s[15] ^ (0x31b49ee2u + rk)) * 0x99cfb423u; - MH_QR(s[0], s[4], s[8], s[12], 20u, 20u, 19u, 4u) MH_QR(s[1], s[5], s[9], s[13], 20u, 20u, 19u, 4u) - MH_QR(s[2], s[6], s[10], s[14], 20u, 20u, 19u, 4u) MH_QR(s[3], s[7], s[11], s[15], 20u, 20u, 19u, 4u) - MH_QR(s[0], s[5], s[10], s[15], 26u, 3u, 3u, 27u) MH_QR(s[1], s[6], s[11], s[12], 26u, 3u, 3u, 27u) - MH_QR(s[2], s[7], s[8], s[13], 26u, 3u, 3u, 27u) MH_QR(s[3], s[4], s[9], s[14], 26u, 3u, 3u, 27u) -} - -// Item t: 16 words. s = (K, t * MUL[i] + RC[i]); 8 rounds of 8 x mixer + cache line s[0] & mask; 8 x final mixer. -IGNEUM_HD void mh_item(const uint32_t* cache, uint32_t t, uint32_t* s) { - s[0] = 0x3067619fu; - s[1] = 0x3c269176u; - s[2] = 0x84a03b03u; - s[3] = 0xf8c63294u; - s[4] = 0xff977c5bu; - s[5] = 0xe60def3eu; - s[6] = 0x63630141u; - s[7] = 0xb8fbcb58u; - s[8] = t * 0x42146205u + 0xbab68293u; - s[9] = t * 0x52cbe0fbu + 0xcc162340u; - s[10] = t * 0x7ecf4a03u + 0x6ce151ccu; - s[11] = t * 0x6728907fu + 0xe62b8997u; - s[12] = t * 0xd81d9751u + 0xc9c80297u; - s[13] = t * 0x132952c3u + 0xf74a1654u; - s[14] = t * 0xf60de277u + 0x3d704af5u; - s[15] = t * 0x05358035u + 0x3cf522b7u; - for (uint32_t r = 0u; r < 8u; ++r) { - for (uint32_t j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (r * 8u + j + 1u)); - const uint32_t* line = cache + ((s[0] & MH_CACHE_LINE_MASK) * 16u); - for (uint32_t i = 0u; i < 16u; ++i) s[i] ^= line[i]; - } - for (uint32_t j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (64u + j + 1u)); -} -// dataset[w] without the dataset: derive item w >> 4 and take word w & 15. -IGNEUM_HD uint32_t mh_word(const uint32_t* cache, uint32_t w) { uint32_t s[16]; mh_item(cache, w >> 4u, s); return s[w & 15u]; } diff --git a/proto-cuda/packs-ca4/mx8_mm128/memhard.metal b/proto-cuda/packs-ca4/mx8_mm128/memhard.metal deleted file mode 100644 index 01b263d6d..000000000 --- a/proto-cuda/packs-ca4/mx8_mm128/memhard.metal +++ /dev/null @@ -1,107 +0,0 @@ -#include -using namespace metal; -// Memory-hard dataset core (MEMHARD.md). Cache: 2^26 words in 2^16 segments of 64 chained ChaCha12 lines. -// Item: 8 rounds of 8 x seed-parameterised mixer + one 64-byte cache read, then 8 x final mixer (class v3, mixer multiplier 8, -// docs/plans/mixer-x4.md: the round key of application j of round r is 0x9E3779B9 * (r * m + j + 1)). All parameters are literals. -#define MH_CACHE_LINE_MASK 0x003fffffu -#define MH_SEGMENT_LINES 64u -#define MH_QR(a, b, c, d, r1, r2, r3, r4) { a += b; d ^= a; d = mh_rotl(d, r1); c += d; b ^= c; b = mh_rotl(b, r2); a += b; d ^= a; d = mh_rotl(d, r3); c += d; b ^= c; b = mh_rotl(b, r4); } -inline uint mh_rotl(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 at every call site - -// y = ChaCha12 core(x) + x -inline void mh_chacha_block(const thread uint* x, thread uint* y) { - for (uint i = 0u; i < 16u; ++i) y[i] = x[i]; - for (uint r = 0u; r < 6u; ++r) { - MH_QR(y[0], y[4], y[8], y[12], 16u, 12u, 8u, 7u) MH_QR(y[1], y[5], y[9], y[13], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[6], y[10], y[14], 16u, 12u, 8u, 7u) MH_QR(y[3], y[7], y[11], y[15], 16u, 12u, 8u, 7u) - MH_QR(y[0], y[5], y[10], y[15], 16u, 12u, 8u, 7u) MH_QR(y[1], y[6], y[11], y[12], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[7], y[8], y[13], 16u, 12u, 8u, 7u) MH_QR(y[3], y[4], y[9], y[14], 16u, 12u, 8u, 7u) - } - for (uint i = 0u; i < 16u; ++i) y[i] += x[i]; -} - -// One cache segment: 64 chained lines written at cache[seg * 1024]. in_j = prev ^ (sigma || K || seg || j || tag), prev_0 = 0. -inline void mh_cache_segment(device uint* cache, uint seg) { - uint prev[16]; uint x[16]; uint y[16]; - for (uint i = 0u; i < 16u; ++i) prev[i] = 0u; - for (uint j = 0u; j < MH_SEGMENT_LINES; ++j) { - x[0] = 0x61707865u ^ prev[0]; x[1] = 0x3320646eu ^ prev[1]; x[2] = 0x79622d32u ^ prev[2]; x[3] = 0x6b206574u ^ prev[3]; - x[4] = 0x3067619fu ^ prev[4]; - x[5] = 0x3c269176u ^ prev[5]; - x[6] = 0x84a03b03u ^ prev[6]; - x[7] = 0xf8c63294u ^ prev[7]; - x[8] = 0xff977c5bu ^ prev[8]; - x[9] = 0xe60def3eu ^ prev[9]; - x[10] = 0x63630141u ^ prev[10]; - x[11] = 0xb8fbcb58u ^ prev[11]; - x[12] = seg ^ prev[12]; x[13] = j ^ prev[13]; x[14] = 0x49676e65u ^ prev[14]; x[15] = 0x756d4d48u ^ prev[15]; - mh_chacha_block(x, y); - device uint* line = cache + ((seg * MH_SEGMENT_LINES + j) * 16u); - for (uint i = 0u; i < 16u; ++i) { line[i] = y[i]; prev[i] = y[i]; } - } -} - -// M_r: per word (s ^ (RC + rk)) * MUL, then a column round and a diagonal round with the seed-drawn rotations. -inline void mh_mixer(thread uint* s, uint rk) { - s[0] = (s[0] ^ (0xbab68293u + rk)) * 0x42146205u; - s[1] = (s[1] ^ (0xcc162340u + rk)) * 0x52cbe0fbu; - s[2] = (s[2] ^ (0x6ce151ccu + rk)) * 0x7ecf4a03u; - s[3] = (s[3] ^ (0xe62b8997u + rk)) * 0x6728907fu; - s[4] = (s[4] ^ (0xc9c80297u + rk)) * 0xd81d9751u; - s[5] = (s[5] ^ (0xf74a1654u + rk)) * 0x132952c3u; - s[6] = (s[6] ^ (0x3d704af5u + rk)) * 0xf60de277u; - s[7] = (s[7] ^ (0x3cf522b7u + rk)) * 0x05358035u; - s[8] = (s[8] ^ (0x2b9cac04u + rk)) * 0xbaf6499du; - s[9] = (s[9] ^ (0xa880ac10u + rk)) * 0xe4db9667u; - s[10] = (s[10] ^ (0x13e5dd1du + rk)) * 0x3e98f45du; - s[11] = (s[11] ^ (0x6fc3e233u + rk)) * 0xd0004eddu; - s[12] = (s[12] ^ (0x2d83eeacu + rk)) * 0x2691630du; - s[13] = (s[13] ^ (0x9006e8bfu + rk)) * 0x9beb3bcfu; - s[14] = (s[14] ^ (0x2c4b5362u + rk)) * 0xab310379u; - s[15] = (s[15] ^ (0x31b49ee2u + rk)) * 0x99cfb423u; - MH_QR(s[0], s[4], s[8], s[12], 20u, 20u, 19u, 4u) MH_QR(s[1], s[5], s[9], s[13], 20u, 20u, 19u, 4u) - MH_QR(s[2], s[6], s[10], s[14], 20u, 20u, 19u, 4u) MH_QR(s[3], s[7], s[11], s[15], 20u, 20u, 19u, 4u) - MH_QR(s[0], s[5], s[10], s[15], 26u, 3u, 3u, 27u) MH_QR(s[1], s[6], s[11], s[12], 26u, 3u, 3u, 27u) - MH_QR(s[2], s[7], s[8], s[13], 26u, 3u, 3u, 27u) MH_QR(s[3], s[4], s[9], s[14], 26u, 3u, 3u, 27u) -} - -// Item t: 16 words. s = (K, t * MUL[i] + RC[i]); 8 rounds of 8 x mixer + cache line s[0] & mask; 8 x final mixer. -inline void mh_item(device const uint* cache, uint t, thread uint* s) { - s[0] = 0x3067619fu; - s[1] = 0x3c269176u; - s[2] = 0x84a03b03u; - s[3] = 0xf8c63294u; - s[4] = 0xff977c5bu; - s[5] = 0xe60def3eu; - s[6] = 0x63630141u; - s[7] = 0xb8fbcb58u; - s[8] = t * 0x42146205u + 0xbab68293u; - s[9] = t * 0x52cbe0fbu + 0xcc162340u; - s[10] = t * 0x7ecf4a03u + 0x6ce151ccu; - s[11] = t * 0x6728907fu + 0xe62b8997u; - s[12] = t * 0xd81d9751u + 0xc9c80297u; - s[13] = t * 0x132952c3u + 0xf74a1654u; - s[14] = t * 0xf60de277u + 0x3d704af5u; - s[15] = t * 0x05358035u + 0x3cf522b7u; - for (uint r = 0u; r < 8u; ++r) { - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (r * 8u + j + 1u)); - device const uint* line = cache + ((s[0] & MH_CACHE_LINE_MASK) * 16u); - for (uint i = 0u; i < 16u; ++i) s[i] ^= line[i]; - } - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (64u + j + 1u)); -} -// dataset[w] without the dataset: derive item w >> 4 and take word w & 15. -inline uint mh_word(device const uint* cache, uint w) { uint s[16]; mh_item(cache, w >> 4u, s); return s[w & 15u]; } - -// One thread per segment (2^16 threads). -kernel void igneum_cache_fill(device uint* cache [[buffer(0)]], uint gid [[thread_position_in_grid]]) { - mh_cache_segment(cache, gid); -} -// One thread per 64-byte item (dataset words / 16 threads). -kernel void igneum_build(device const uint* cache [[buffer(0)]], device uint* dataset [[buffer(1)]], - uint gid [[thread_position_in_grid]]) { - uint s[16]; - mh_item(cache, gid, s); - device uint* d = dataset + gid * 16u; - for (uint i = 0u; i < 16u; ++i) d[i] = s[i]; -} diff --git a/proto-cuda/packs-ca4/mx8_mm128/program.h b/proto-cuda/packs-ca4/mx8_mm128/program.h deleted file mode 100644 index ed2a84814..000000000 --- a/proto-cuda/packs-ca4/mx8_mm128/program.h +++ /dev/null @@ -1,73 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// Program metadata for host.cu plus the launch wrappers defined in kernel.cu. -// Also included by proto-opencl/host.c (C99), which defines IGNEUM_NO_CUDA first and reads only the macros. -#pragma once -#ifdef __cplusplus -#include -#else -#include -#endif -#ifndef IGNEUM_NO_CUDA -#include -#endif - -#define IGNEUM_SEED_STRING "igneum-genesis" -#define IGNEUM_SEED_BYTES_HEX "69676e65756d2d67656e65736973" -#define IGNEUM_GENERATOR 2 -#define IGNEUM_PROGRAM_ATTEMPT 0 -#define IGNEUM_PROGRAM_ID 0xd6fc3093180e44f1ull -#define IGNEUM_DAY_STRING "2026-10-03" -#define IGNEUM_DAY_BYTES_HEX "6461792f323032362d31302d3033" -#define IGNEUM_DAY0 0x3067619fu -#define IGNEUM_DAY1 0x3c269176u -#define IGNEUM_DATASET_LOG2 28 -#define IGNEUM_MASK 0x0fffffffu -#define IGNEUM_LANES 32 -#define IGNEUM_ITERATIONS 8 -#define IGNEUM_INSTR_COUNT 64 -#define IGNEUM_LOADS_PER_HASH 128 -#define IGNEUM_WIDE_LOADS_PER_HASH 0 -#define IGNEUM_OP_MIX "load=16 add=8 shfl=8 xor=6 mad=5 mul=5 mulhi=5 sub=4 rotl=3 rotr=3 or=1" -// Class v3 construction (Counter ASIC 2.0, 5 October 2026, docs/plans/mixer-x4.md): version 2 loads; the dataset item -// derivation applies the mixer IGNEUM_MIXER_MULT times per round (memhard.h), and the cache follows the growth rule. -#define IGNEUM_LOAD_CLASS "mx8+mm128" -#define IGNEUM_CLASS_MIXER_MULT 8 -#define IGNEUM_CACHE_GROWTH 1 // 1: cache words = 2^(26 + doublings(day)), doublings = floor(log2(1 + day / 1460)) -#define IGNEUM_LOAD_SLOTS 16 -#define IGNEUM_LOAD_MIX { 100, 0, 0 } -#define IGNEUM_LOAD_WIDTH_COUNTS { 16, 0, 0 } // loads of 4, 16, 64 bytes per program -#define IGNEUM_BYTES_PER_HASH 512 -#define IGNEUM_FOLD_ROT 11 -#define IGNEUM_FOLD_MUL 0x9e3779b1u -// Latency-shadow block (Counter ASIC 3.0 item 8, docs/analysis/latency-shadow-2026-10-06.md): NOT the lottery hash. A block of -// IGNEUM_SHADOW_INSTRS ALU instructions (the ten non-load families) runs IGNEUM_SHADOW_REPS times at the end of every -// iteration; the 16 loads, the acceptance rule and the base program are the class's without the shadow. -#define IGNEUM_SHADOW_INSTRS 0 -#define IGNEUM_SHADOW_REPS 0 -#define IGNEUM_SHADOW_INSTRS_PER_HASH 0 -#define IGNEUM_SHADOW_OP_MIX "" -// Counter ASIC 4.0 research (experimental): IGNEUM_MM8_TILES int8 mma.m8n8k16 u8 tiles per iteration after the shadow block. -#define IGNEUM_MM8_TILES 128 -#define IGNEUM_MM8_TILES_PER_HASH 1024 -// 0 = closed-form dataset (ds_elem), 1 = memory-hard cache construction (MEMHARD.md, memhard.h) -#define IGNEUM_DATASET_MODE 1 - -#define IGNEUM_SEEDW_INIT { 0x67a9a7beu, 0x1a155b25u, 0xfddfb732u, 0x4b5af2e8u, 0xc55caf33u, 0xa27c13b7u, 0x06628a48u, 0x03852469u } -#define IGNEUM_KEY_INIT { 0x3067619fu, 0x3c269176u, 0x84a03b03u, 0xf8c63294u, 0xff977c5bu, 0xe60def3eu, 0x63630141u, 0xb8fbcb58u } -#define IGNEUM_CACHE_LOG2_WORDS 26 -#define IGNEUM_CACHE_SEGMENT_LOG2_LINES 6 -#define IGNEUM_CACHE_SEGMENTS 65536u -#define IGNEUM_ITEM_ROUNDS 8 -#define IGNEUM_MIXER_MULT 8 // mixer applications per round and after the last read (class v3, docs/plans/mixer-x4.md) -#define IGNEUM_MIX_ROT_INIT { 20u, 20u, 19u, 4u, 26u, 3u, 3u, 27u } -#define IGNEUM_MIX_MUL_INIT { 0x42146205u, 0x52cbe0fbu, 0x7ecf4a03u, 0x6728907fu, 0xd81d9751u, 0x132952c3u, 0xf60de277u, 0x05358035u, 0xbaf6499du, 0xe4db9667u, 0x3e98f45du, 0xd0004eddu, 0x2691630du, 0x9beb3bcfu, 0xab310379u, 0x99cfb423u } -#define IGNEUM_MIX_RC_INIT { 0xbab68293u, 0xcc162340u, 0x6ce151ccu, 0xe62b8997u, 0xc9c80297u, 0xf74a1654u, 0x3d704af5u, 0x3cf522b7u, 0x2b9cac04u, 0xa880ac10u, 0x13e5dd1du, 0x6fc3e233u, 0x2d83eeacu, 0x9006e8bfu, 0x2c4b5362u, 0x31b49ee2u } - -#ifndef IGNEUM_NO_CUDA -// Defined in kernel.cu. All launch on the default stream and return cudaGetLastError(). -cudaError_t igneum_launch_cache_fill(uint32_t* cache, uint32_t nSegments); -cudaError_t igneum_launch_build(uint32_t* ds, const uint32_t* cache, uint32_t nItems); -cudaError_t igneum_launch_hash(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask, - uint32_t nonces, uint32_t blockWarps); -cudaError_t igneum_hash_info(int* numRegs, int* blocksPerSM, uint32_t blockWarps); -#endif diff --git a/proto-cuda/packs-ca4/mx8_mm128/program.json b/proto-cuda/packs-ca4/mx8_mm128/program.json deleted file mode 100644 index 729e56af7..000000000 --- a/proto-cuda/packs-ca4/mx8_mm128/program.json +++ /dev/null @@ -1,134 +0,0 @@ -{ - "format": "igneum-program-pack-3", - "generator": 2, - "attempt": 0, - "program_id": "0xd6fc3093180e44f1", - "program_id_derivation": "FNV-1a 64 over 'igneum-program/' || generator_le32 || seed_words as little-endian bytes || attempt_le32", - "dataset_mode": "memory-hard", - "seed": "igneum-genesis", - "seed_bytes": "69676e65756d2d67656e65736973", - "seed_words": ["0x67a9a7be", "0x1a155b25", "0xfddfb732", "0x4b5af2e8", "0xc55caf33", "0xa27c13b7", "0x06628a48", "0x03852469"], - "seed_derivation": "seed_words = FNV-1a 64 over seed_bytes (attempt 0) or seed_bytes || attempt_le32 (attempt k >= 1), basis ^ (salt * 0x9E3779B97F4A7C15) for salt 0..3, then h ^= h>>33; h *= 0xff51afd7ed558ccd; h ^= h>>33; words[2*salt] = low 32, words[2*salt+1] = high 32", - "generator_rule": "version 2: exactly 16 load slots drawn first from instructions 1..63 (partial Fisher-Yates), the other 48 ops from the ten non-load weights (sum 75); a load's source is drawn from the registers other than dst written by an earlier instruction and not read by a load since; the candidate must pass the acceptance rule of spec 01 section 1.4.6 (static: no cyclically stale load source, every register has an injecting write; dynamic: 64 units on the seed-keyed closed-form dataset with no constant register bit, no lane-constant load site, under 164 saturated final values, every output bit within 136 of 1024, distinct addresses above 245760), else the next attempt of the seed is tried", - "lanes": 32, - "registers": 8, - "iterations": 8, - "instruction_count": 64, - "loads_per_hash": 128, - "load_class": "mx8+mm128", - "mixer_mult": 8, - "cache_growth": true, - "mixer": "class v3 (Counter ASIC 2.0, 5 October 2026, docs/plans/mixer-x4.md): every mixer application of the item derivation is 8 applications with round keys (r * 8 + j + 1) * 0x9E3779B9, the 8 dependent cache reads per item unchanged; cache growth rule option C: cache words = 2^(26 + doublings(day)), dataset words = 2^(genesis_log2 + doublings(day)), doublings(day) = floor(log2(1 + day / 1460)) for day = days since genesis", - "load_slots": 16, - "load_mix_percent_4_16_64": [100, 0, 0], - "load_width_counts_4_16_64": [16, 0, 0], - "bytes_per_hash": 512, - "wide_load": "read-width experiment (5 October 2026, docs/plans/read-width.md), NOT the lottery hash: a load of W words (width field, 4 or 16) reads dataset[b .. b + W) with b = (src & mask) & ~(W - 1) and folds every word into dst: x = dst ^ w[0]; for j in 1..W: x = (rotl(x, 11) * 0x9e3779b1) ^ w[j]; dst = x; width 1 is the plain load; the width is drawn per instruction from the class mix with one extra below(100) draw after the nine of version 2, and the program id is FNV-1a 64 over 'igneum-program-rw/' || generator_le32 || seed words || attempt_le32 || mix[3] || load_slots", - "op_mix": {"load": 16, "add": 8, "shfl": 8, "xor": 6, "mad": 5, "mul": 5, "mulhi": 5, "sub": 4, "rotl": 3, "rotr": 3, "or": 1}, - "register_init": "for i in 0..7: x = nonce ^ seed_words[i]; x += 0x9e3779b9 * (i+1) (mod 2^32); x = splitmix32(x); r[i] = x ^ seed_words[(i+1) & 7]", - "splitmix32": "x ^= x>>16; x *= 0x7feb352d; x ^= x>>15; x *= 0x846ca68b; x ^= x>>16", - "iteration": "sel = r0 sampled once at the top of each iteration, then all instructions in order", - "output": "lo = r0 ^ rotl(r1,7) ^ rotl(r2,14) ^ rotl(r3,21); hi = r4 ^ rotl(r5,9) ^ rotl(r6,18) ^ rotl(r7,27); out = (hi << 32) | lo", - "op_semantics": { - "add": "dst = dst + src + (bit `bit` of sel ? imm2 : imm)", - "sub": "dst = dst - src", - "mul": "dst = dst * src (low 32)", - "mulhi": "dst = high 32 bits of dst * src", - "xor": "dst = dst ^ src", - "or": "dst = dst | src", - "rotl": "dst = rotl(dst, rot), rot in 1..31", - "rotr": "dst = rotr(dst, src & 31)", - "mad": "dst = src * src2 + dst", - "shfl": "dst = dst ^ (src of lane (lane ^ mask)), mask in {1,2,4,8,16}, within the 32-lane warp", - "load": "dst = dst ^ dataset[src & dataset.mask]", - "wload": "base = (src of lane 0 & dataset.mask) & ~31; dst = dst ^ dataset[base + lane] (warp-coalesced 128-byte load, lever b, only when --wide-frac > 0)" - }, - "dataset": { - "log2_words": 28, - "bytes": 1073741824, - "mask": "0x0fffffff", - "day": "2026-10-03", - "day_bytes": "6461792f323032362d31302d3033", - "day_words_from": "seed_words_from_bytes(day_bytes)", - "d0": "0x3067619f", - "d1": "0x3c269176", - "mode": "memory-hard", - "spec": "proto-metal/MEMHARD.md", - "key": ["0x3067619f", "0x3c269176", "0x84a03b03", "0xf8c63294", "0xff977c5b", "0xe60def3e", "0x63630141", "0xb8fbcb58"], - "key_derivation": "the 8 words of seed_words_from_bytes(day_bytes); d0, d1 are key[0], key[1]", - "cache": {"log2_words": 26, "bytes": 268435456, "line_words": 16, "segment_lines": 64, "segments": 65536, "block": "ChaCha12 core + feed-forward, rotations 16 12 8 7", "sigma": ["0x61707865", "0x3320646e", "0x79622d32", "0x6b206574"], "tag": ["0x49676e65", "0x756d4d48"], "chain": "in_j = prev_line ^ (sigma[0..3] || key[0..7] || seg || j || tag[0..1]); line_j = block(in_j); prev_0 = 0"}, - "mixer": {"draw": "SplitMix64 seeded with key[0] | key[1] << 32: rot[0..7] = 1 + next() % 31, mul[0..15] = low32(next()) | 1, rc[0..15] = low32(next())", "rot": [20, 20, 19, 4, 26, 3, 3, 27], "mul": ["0x42146205", "0x52cbe0fb", "0x7ecf4a03", "0x6728907f", "0xd81d9751", "0x132952c3", "0xf60de277", "0x05358035", "0xbaf6499d", "0xe4db9667", "0x3e98f45d", "0xd0004edd", "0x2691630d", "0x9beb3bcf", "0xab310379", "0x99cfb423"], "rc": ["0xbab68293", "0xcc162340", "0x6ce151cc", "0xe62b8997", "0xc9c80297", "0xf74a1654", "0x3d704af5", "0x3cf522b7", "0x2b9cac04", "0xa880ac10", "0x13e5dd1d", "0x6fc3e233", "0x2d83eeac", "0x9006e8bf", "0x2c4b5362", "0x31b49ee2"], "round": "for i in 0..15: s[i] = (s[i] ^ (rc[i] + (r+1) * 0x9E3779B9)) * mul[i]; then quarter rounds on columns (0,4,8,12) (1,5,9,13) (2,6,10,14) (3,7,11,15) with rot[0..3] and diagonals (0,5,10,15) (1,6,11,12) (2,7,8,13) (3,4,9,14) with rot[4..7]", "quarter_round": "a += b; d ^= a; d = rotl(d, r1); c += d; b ^= c; b = rotl(b, r2); a += b; d ^= a; d = rotl(d, r3); c += d; b ^= c; b = rotl(b, r4)"}, - "mixer_mult": 8, - "item": "s[0..7] = key; s[8+i] = t * mul[i] + rc[i] for i in 0..7; for r in 0..7: for j in 0..7: s = M(s, rk = (r * 8 + j + 1) * 0x9E3779B9); line = s[0] & 0x003fffff; s[i] ^= cache[line * 16 + i]; then for j in 0..7: s = M(s, rk = (64 + j + 1) * 0x9E3779B9); item(t) = s", - "word": "dataset[w] = item(w >> 4)[w & 15]" - }, - "shadow": {"instrs": 0, "reps": 0, "instrs_per_hash": 0, "op_mix": {}, "rule": "Counter ASIC 3.0 item 8 (docs/analysis/latency-shadow-2026-10-06.md): after the 64 base instructions the program stream draws instrs more ALU instructions (the non-load table, nine draws each, the source as on an ALU slot); the block runs reps times at the end of every iteration with the iteration's sel; the acceptance rule interprets the base program only", "program_id_suffix": "'shadow/' || instrs_le16 || reps_le16", "instructions": [ - ]}, - "mm8_tiles": {"tiles": 128, "tiles_per_hash": 1024, "rule": "Counter ASIC 4.0 research (docs/analysis/counter-asic-4-research.md 15.2): after the shadow draws the stream draws tiles descriptors a, b, c (below 8 each) and c2 (below 7, skipping c); each tile is the PTX mma.m8n8k16 u8 product of the 32 lanes' r[a] (8 x 16 bytes) and r[b] (16 x 8), lane l adds C[l >> 2][2 (l & 3)] into r[c] and C[l >> 2][2 (l & 3) + 1] into r[c2] modulo 2^32; the block runs once per iteration after the shadow", "program_id_suffix": "'mm8/' || tiles_le16", "descriptors": [[6,5,2,3],[1,1,4,5],[2,5,0,7],[1,5,2,3],[2,0,2,6],[2,6,7,3],[1,4,1,3],[4,6,0,4],[1,1,1,5],[3,7,4,6],[3,0,0,4],[2,3,0,6],[3,1,6,0],[1,5,2,5],[3,3,3,4],[1,0,5,4],[7,2,0,4],[0,5,1,7],[1,1,2,0],[3,3,2,4],[3,0,6,1],[0,3,6,0],[1,7,6,4],[1,2,0,3],[3,4,0,2],[5,0,3,7],[1,0,7,3],[3,2,0,5],[2,4,6,5],[3,4,1,5],[6,5,6,3],[1,7,4,3],[7,6,3,0],[7,1,4,6],[1,0,5,1],[2,4,1,3],[1,7,7,1],[1,7,0,2],[3,6,2,4],[6,2,2,5],[6,1,4,1],[6,0,6,2],[7,4,5,1],[6,1,6,0],[0,6,4,1],[6,5,6,5],[6,6,1,0],[7,3,3,1],[7,7,6,3],[4,5,1,7],[1,7,4,5],[3,4,0,3],[2,5,7,4],[7,7,1,3],[7,6,6,5],[1,7,3,7],[4,0,7,3],[4,3,6,7],[2,3,6,0],[6,7,5,7],[6,0,2,4],[2,1,2,6],[4,6,1,6],[1,6,0,3],[0,5,5,3],[0,1,2,4],[3,5,1,3],[6,2,3,4],[7,1,1,2],[6,7,5,4],[1,7,1,5],[3,0,1,0],[4,0,3,6],[4,1,0,2],[5,1,3,4],[4,4,7,3],[5,6,1,7],[0,6,3,2],[2,5,0,5],[7,4,4,1],[6,7,3,4],[4,3,1,6],[6,0,1,7],[2,2,1,6],[0,6,0,3],[1,3,4,1],[3,4,2,3],[4,4,3,7],[6,4,5,3],[4,2,2,1],[6,5,3,6],[6,5,5,4],[2,6,2,3],[2,2,5,7],[2,4,7,3],[5,4,1,6],[0,6,3,1],[0,5,1,2],[2,7,3,2],[1,4,0,2],[2,4,2,5],[2,4,6,7],[5,7,6,3],[4,7,4,3],[2,0,3,7],[6,1,0,5],[4,1,0,5],[2,1,4,2],[4,6,1,2],[2,7,5,3],[1,0,0,2],[6,5,4,6],[5,7,4,2],[6,3,4,1],[2,1,7,5],[1,4,1,4],[7,2,5,0],[7,6,7,6],[0,1,2,1],[4,3,5,3],[5,3,7,2],[2,7,2,1],[0,1,3,0],[5,2,3,5],[5,1,2,5],[7,2,7,2],[6,3,3,2],[6,7,5,1]]}, - "instructions": [ - {"i": 0, "op": "mad", "dst": 2, "src": 3, "src2": 4, "imm": "0xbf7b174d", "imm2": "0x337b762e", "rot": 17, "bit": 2, "mask": 2, "width": 1}, - {"i": 1, "op": "mad", "dst": 2, "src": 1, "src2": 1, "imm": "0xdd04a5da", "imm2": "0x42da7657", "rot": 15, "bit": 30, "mask": 16, "width": 1}, - {"i": 2, "op": "mad", "dst": 2, "src": 3, "src2": 2, "imm": "0x734003fa", "imm2": "0x5bb67700", "rot": 3, "bit": 20, "mask": 1, "width": 1}, - {"i": 3, "op": "xor", "dst": 3, "src": 5, "src2": 5, "imm": "0xc55a1b1c", "imm2": "0xa19720f3", "rot": 7, "bit": 8, "mask": 1, "width": 1}, - {"i": 4, "op": "load", "dst": 7, "src": 2, "src2": 5, "imm": "0xad572dd7", "imm2": "0x9ceb3ea7", "rot": 18, "bit": 30, "mask": 2, "width": 1}, - {"i": 5, "op": "load", "dst": 5, "src": 7, "src2": 3, "imm": "0x769a53be", "imm2": "0x80f9067e", "rot": 12, "bit": 22, "mask": 1, "width": 1}, - {"i": 6, "op": "shfl", "dst": 1, "src": 4, "src2": 0, "imm": "0xd3613d88", "imm2": "0x262fb219", "rot": 10, "bit": 30, "mask": 8, "width": 1}, - {"i": 7, "op": "shfl", "dst": 7, "src": 3, "src2": 4, "imm": "0xce38e42f", "imm2": "0xb868b818", "rot": 11, "bit": 8, "mask": 8, "width": 1}, - {"i": 8, "op": "mulhi", "dst": 1, "src": 5, "src2": 2, "imm": "0xa5eebca5", "imm2": "0x5703a72b", "rot": 13, "bit": 13, "mask": 16, "width": 1}, - {"i": 9, "op": "rotr", "dst": 6, "src": 3, "src2": 4, "imm": "0x17a5a9c7", "imm2": "0xdcfb93a1", "rot": 20, "bit": 27, "mask": 2, "width": 1}, - {"i": 10, "op": "or", "dst": 3, "src": 4, "src2": 1, "imm": "0xccb7d785", "imm2": "0xc335364c", "rot": 14, "bit": 12, "mask": 4, "width": 1}, - {"i": 11, "op": "load", "dst": 4, "src": 3, "src2": 2, "imm": "0x88cb9af3", "imm2": "0x4e7dc10d", "rot": 24, "bit": 17, "mask": 4, "width": 1}, - {"i": 12, "op": "mulhi", "dst": 0, "src": 4, "src2": 4, "imm": "0x45374321", "imm2": "0x3cd91989", "rot": 11, "bit": 4, "mask": 2, "width": 1}, - {"i": 13, "op": "add", "dst": 5, "src": 1, "src2": 2, "imm": "0xc7934706", "imm2": "0xd3177981", "rot": 16, "bit": 30, "mask": 2, "width": 1}, - {"i": 14, "op": "load", "dst": 0, "src": 4, "src2": 3, "imm": "0xfd7f56bb", "imm2": "0x65e14f52", "rot": 13, "bit": 22, "mask": 2, "width": 1}, - {"i": 15, "op": "sub", "dst": 2, "src": 4, "src2": 4, "imm": "0x35a80b49", "imm2": "0x060f2d13", "rot": 16, "bit": 20, "mask": 16, "width": 1}, - {"i": 16, "op": "load", "dst": 2, "src": 0, "src2": 2, "imm": "0xae0a32c2", "imm2": "0x4c2a4cfe", "rot": 8, "bit": 31, "mask": 16, "width": 1}, - {"i": 17, "op": "load", "dst": 7, "src": 2, "src2": 6, "imm": "0x82a84cc3", "imm2": "0x21a38d68", "rot": 15, "bit": 21, "mask": 2, "width": 1}, - {"i": 18, "op": "shfl", "dst": 7, "src": 3, "src2": 3, "imm": "0xa3818806", "imm2": "0x8f66b5c8", "rot": 14, "bit": 6, "mask": 4, "width": 1}, - {"i": 19, "op": "mul", "dst": 5, "src": 0, "src2": 1, "imm": "0xa00de107", "imm2": "0x77bfcaa5", "rot": 3, "bit": 10, "mask": 2, "width": 1}, - {"i": 20, "op": "shfl", "dst": 3, "src": 4, "src2": 7, "imm": "0x1d2b8cab", "imm2": "0x80b4f9a2", "rot": 14, "bit": 25, "mask": 2, "width": 1}, - {"i": 21, "op": "shfl", "dst": 2, "src": 4, "src2": 5, "imm": "0x3ac915d2", "imm2": "0x5fba7bc2", "rot": 16, "bit": 1, "mask": 16, "width": 1}, - {"i": 22, "op": "mulhi", "dst": 6, "src": 2, "src2": 0, "imm": "0xdc3ec8fd", "imm2": "0x599e2fa3", "rot": 22, "bit": 3, "mask": 2, "width": 1}, - {"i": 23, "op": "load", "dst": 6, "src": 1, "src2": 5, "imm": "0x2a6b16d5", "imm2": "0xd73e396f", "rot": 28, "bit": 29, "mask": 2, "width": 1}, - {"i": 24, "op": "mul", "dst": 5, "src": 0, "src2": 7, "imm": "0x376d0223", "imm2": "0xe1c2169a", "rot": 4, "bit": 16, "mask": 16, "width": 1}, - {"i": 25, "op": "rotl", "dst": 5, "src": 7, "src2": 3, "imm": "0x78ad8c60", "imm2": "0x6f5b77d5", "rot": 19, "bit": 11, "mask": 16, "width": 1}, - {"i": 26, "op": "shfl", "dst": 7, "src": 6, "src2": 5, "imm": "0x93915b9f", "imm2": "0x1e61fb6b", "rot": 28, "bit": 23, "mask": 2, "width": 1}, - {"i": 27, "op": "xor", "dst": 0, "src": 5, "src2": 7, "imm": "0x6378fe15", "imm2": "0x66c78f42", "rot": 12, "bit": 31, "mask": 8, "width": 1}, - {"i": 28, "op": "xor", "dst": 0, "src": 4, "src2": 7, "imm": "0x20a57fda", "imm2": "0x088c848e", "rot": 16, "bit": 13, "mask": 4, "width": 1}, - {"i": 29, "op": "sub", "dst": 3, "src": 0, "src2": 2, "imm": "0x49d95fd5", "imm2": "0x1a5f946a", "rot": 6, "bit": 12, "mask": 1, "width": 1}, - {"i": 30, "op": "mul", "dst": 5, "src": 1, "src2": 7, "imm": "0x0a816217", "imm2": "0x405c4f73", "rot": 13, "bit": 27, "mask": 4, "width": 1}, - {"i": 31, "op": "load", "dst": 7, "src": 2, "src2": 2, "imm": "0x09ed045e", "imm2": "0xd69c4715", "rot": 5, "bit": 9, "mask": 2, "width": 1}, - {"i": 32, "op": "load", "dst": 1, "src": 0, "src2": 6, "imm": "0xeb79ea49", "imm2": "0xcc587f5a", "rot": 6, "bit": 8, "mask": 16, "width": 1}, - {"i": 33, "op": "xor", "dst": 5, "src": 6, "src2": 1, "imm": "0x3027401e", "imm2": "0x5f20c27e", "rot": 18, "bit": 9, "mask": 2, "width": 1}, - {"i": 34, "op": "load", "dst": 5, "src": 1, "src2": 3, "imm": "0x0e1cab07", "imm2": "0x09356c5b", "rot": 19, "bit": 31, "mask": 1, "width": 1}, - {"i": 35, "op": "mulhi", "dst": 0, "src": 5, "src2": 2, "imm": "0x90e31357", "imm2": "0xabd32484", "rot": 26, "bit": 5, "mask": 8, "width": 1}, - {"i": 36, "op": "shfl", "dst": 5, "src": 2, "src2": 5, "imm": "0xee9a955f", "imm2": "0x31b3faed", "rot": 8, "bit": 24, "mask": 4, "width": 1}, - {"i": 37, "op": "load", "dst": 7, "src": 0, "src2": 7, "imm": "0x3ba2f832", "imm2": "0x1160dcd3", "rot": 4, "bit": 29, "mask": 1, "width": 1}, - {"i": 38, "op": "add", "dst": 3, "src": 1, "src2": 7, "imm": "0x75ba2fad", "imm2": "0x230c005c", "rot": 4, "bit": 27, "mask": 1, "width": 1}, - {"i": 39, "op": "shfl", "dst": 1, "src": 5, "src2": 3, "imm": "0xdbf37e75", "imm2": "0xb5ac1969", "rot": 30, "bit": 13, "mask": 4, "width": 1}, - {"i": 40, "op": "xor", "dst": 2, "src": 5, "src2": 5, "imm": "0x47f136c5", "imm2": "0x06ce9153", "rot": 19, "bit": 10, "mask": 2, "width": 1}, - {"i": 41, "op": "mad", "dst": 3, "src": 6, "src2": 3, "imm": "0xce13eff8", "imm2": "0x04cc1d55", "rot": 3, "bit": 1, "mask": 4, "width": 1}, - {"i": 42, "op": "sub", "dst": 6, "src": 7, "src2": 1, "imm": "0x6a65ab71", "imm2": "0x8fbc1bcd", "rot": 4, "bit": 1, "mask": 8, "width": 1}, - {"i": 43, "op": "xor", "dst": 7, "src": 0, "src2": 7, "imm": "0xdaeb4928", "imm2": "0xc0423027", "rot": 24, "bit": 11, "mask": 8, "width": 1}, - {"i": 44, "op": "load", "dst": 1, "src": 7, "src2": 7, "imm": "0x778f01c9", "imm2": "0x28cedcea", "rot": 12, "bit": 4, "mask": 16, "width": 1}, - {"i": 45, "op": "mul", "dst": 2, "src": 3, "src2": 2, "imm": "0xf4264f1b", "imm2": "0x0f627d56", "rot": 5, "bit": 28, "mask": 8, "width": 1}, - {"i": 46, "op": "mulhi", "dst": 1, "src": 5, "src2": 1, "imm": "0xffb2147a", "imm2": "0xccde9b05", "rot": 13, "bit": 9, "mask": 2, "width": 1}, - {"i": 47, "op": "sub", "dst": 4, "src": 3, "src2": 5, "imm": "0x73b36234", "imm2": "0x3f5d5997", "rot": 7, "bit": 18, "mask": 2, "width": 1}, - {"i": 48, "op": "rotr", "dst": 2, "src": 6, "src2": 3, "imm": "0x3a4d9aa9", "imm2": "0x212bec7b", "rot": 4, "bit": 29, "mask": 16, "width": 1}, - {"i": 49, "op": "load", "dst": 3, "src": 5, "src2": 2, "imm": "0x626f11df", "imm2": "0x56cd5bfd", "rot": 7, "bit": 1, "mask": 1, "width": 1}, - {"i": 50, "op": "add", "dst": 1, "src": 5, "src2": 6, "imm": "0x81b8bc2c", "imm2": "0x1907970c", "rot": 28, "bit": 7, "mask": 4, "width": 1}, - {"i": 51, "op": "mul", "dst": 0, "src": 2, "src2": 2, "imm": "0xa8848b30", "imm2": "0xef6ac348", "rot": 9, "bit": 15, "mask": 8, "width": 1}, - {"i": 52, "op": "add", "dst": 0, "src": 2, "src2": 0, "imm": "0x4f92b968", "imm2": "0x699fd448", "rot": 22, "bit": 6, "mask": 4, "width": 1}, - {"i": 53, "op": "add", "dst": 1, "src": 0, "src2": 2, "imm": "0x2bb965af", "imm2": "0x77b1520d", "rot": 2, "bit": 12, "mask": 8, "width": 1}, - {"i": 54, "op": "rotl", "dst": 7, "src": 1, "src2": 0, "imm": "0x553e678b", "imm2": "0x3cc8eae0", "rot": 14, "bit": 20, "mask": 2, "width": 1}, - {"i": 55, "op": "add", "dst": 3, "src": 7, "src2": 2, "imm": "0x7b0fe07a", "imm2": "0xa54c55a0", "rot": 10, "bit": 1, "mask": 1, "width": 1}, - {"i": 56, "op": "load", "dst": 6, "src": 7, "src2": 4, "imm": "0x01eba9aa", "imm2": "0x2758c0f7", "rot": 14, "bit": 15, "mask": 4, "width": 1}, - {"i": 57, "op": "rotr", "dst": 1, "src": 5, "src2": 2, "imm": "0x1f5267b3", "imm2": "0x236f5a27", "rot": 2, "bit": 31, "mask": 16, "width": 1}, - {"i": 58, "op": "load", "dst": 5, "src": 4, "src2": 3, "imm": "0xa9954a9b", "imm2": "0x6a54d4e8", "rot": 11, "bit": 10, "mask": 16, "width": 1}, - {"i": 59, "op": "load", "dst": 6, "src": 2, "src2": 4, "imm": "0x9923ff88", "imm2": "0x9357254e", "rot": 16, "bit": 1, "mask": 16, "width": 1}, - {"i": 60, "op": "mad", "dst": 3, "src": 5, "src2": 0, "imm": "0xc0cc51a6", "imm2": "0x3fd7701b", "rot": 20, "bit": 1, "mask": 4, "width": 1}, - {"i": 61, "op": "add", "dst": 5, "src": 7, "src2": 7, "imm": "0xaf9dd72d", "imm2": "0xad7493e7", "rot": 7, "bit": 31, "mask": 16, "width": 1}, - {"i": 62, "op": "add", "dst": 4, "src": 6, "src2": 2, "imm": "0x89841d87", "imm2": "0x1e07c3d9", "rot": 6, "bit": 27, "mask": 1, "width": 1}, - {"i": 63, "op": "rotl", "dst": 5, "src": 4, "src2": 2, "imm": "0xae210f8d", "imm2": "0x8e499ba4", "rot": 19, "bit": 9, "mask": 1, "width": 1} - ] -} diff --git a/proto-cuda/packs-ca4/mx8_mm128/program.metal b/proto-cuda/packs-ca4/mx8_mm128/program.metal deleted file mode 100644 index e1588746a..000000000 --- a/proto-cuda/packs-ca4/mx8_mm128/program.metal +++ /dev/null @@ -1,257 +0,0 @@ -#include -using namespace metal; -// int8 tile reference (Counter ASIC 4.0 research): A = the 32 lanes' av as 8 x 16 bytes (lane l holds row l >> 2, bytes 4 (l & 3)..), -// B = bv as 16 x 8 (lane l holds column l >> 2); lane l gets C[l >> 2][2 (l & 3)] and C[l >> 2][2 (l & 3) + 1]; the PTX mma.m8n8k16 u8 layout. -static inline void mm8_ref(uint av, uint bv, uint lane, thread uint& d0, thread uint& d1) { - uint row = lane >> 2, col0 = 2u * (lane & 3u); - uint acc0 = 0u, acc1 = 0u; - for (uint j = 0u; j < 4u; ++j) { - uint aw = simd_shuffle(av, (ushort)(4u * row + j)); - uint bw0 = simd_shuffle(bv, (ushort)(4u * col0 + j)); - uint bw1 = simd_shuffle(bv, (ushort)(4u * (col0 + 1u) + j)); - for (uint t = 0u; t < 4u; ++t) { - uint ab = (aw >> (8u * t)) & 0xffu; - acc0 += ab * ((bw0 >> (8u * t)) & 0xffu); - acc1 += ab * ((bw1 >> (8u * t)) & 0xffu); - } - } - d0 = acc0; d1 = acc1; -} - - -#define MASK 0x0fffffffu -constant uint SEEDW[8] = { 0x67a9a7beu, 0x1a155b25u, 0xfddfb732u, 0x4b5af2e8u, 0xc55caf33u, 0xa27c13b7u, 0x06628a48u, 0x03852469u }; - -inline uint splitmix32(uint x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -inline uint rotl_imm(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 -inline uint rotr_var(uint x, uint n) { n &= 31u; return (x >> n) | (x << ((32u - n) & 31u)); } -inline uint ds_elem(uint i, uint d0, uint d1) { - uint x = i ^ d0; - x *= 0x9E3779B1u; x ^= x >> 15; - x += d1; - x *= 0x85EBCA77u; x ^= x >> 13; - x *= 0xC2B2AE3Du; x ^= x >> 16; - return x; -} - -kernel void igneum_hash(device const uint* dataset [[buffer(0)]], - device ulong* out [[buffer(1)]], - constant uint& baseNonce [[buffer(2)]], - uint gid [[thread_position_in_grid]]) { - uint nonce = baseNonce + gid; - uint r0, r1, r2, r3, r4, r5, r6, r7; - uint lane = gid & 31u; - { uint x = nonce ^ SEEDW[0]; x += 0x9e3779b9u * 1u; x = splitmix32(x); r0 = x ^ SEEDW[1]; } - { uint x = nonce ^ SEEDW[1]; x += 0x9e3779b9u * 2u; x = splitmix32(x); r1 = x ^ SEEDW[2]; } - { uint x = nonce ^ SEEDW[2]; x += 0x9e3779b9u * 3u; x = splitmix32(x); r2 = x ^ SEEDW[3]; } - { uint x = nonce ^ SEEDW[3]; x += 0x9e3779b9u * 4u; x = splitmix32(x); r3 = x ^ SEEDW[4]; } - { uint x = nonce ^ SEEDW[4]; x += 0x9e3779b9u * 5u; x = splitmix32(x); r4 = x ^ SEEDW[5]; } - { uint x = nonce ^ SEEDW[5]; x += 0x9e3779b9u * 6u; x = splitmix32(x); r5 = x ^ SEEDW[6]; } - { uint x = nonce ^ SEEDW[6]; x += 0x9e3779b9u * 7u; x = splitmix32(x); r6 = x ^ SEEDW[7]; } - { uint x = nonce ^ SEEDW[7]; x += 0x9e3779b9u * 8u; x = splitmix32(x); r7 = x ^ SEEDW[0]; } - - for (uint it = 0u; it < 8u; ++it) { - uint sel = r0; - r2 = r3 * r4 + r2; // 0 - r2 = r1 * r1 + r2; // 1 - r2 = r3 * r2 + r2; // 2 - r3 = r3 ^ r5; // 3 - r7 = r7 ^ dataset[r2 & MASK]; // 4 - r5 = r5 ^ dataset[r7 & MASK]; // 5 - r1 = r1 ^ simd_shuffle_xor(r4, (ushort)8); // 6 - r7 = r7 ^ simd_shuffle_xor(r3, (ushort)8); // 7 - r1 = mulhi(r1, r5); // 8 - r6 = rotr_var(r6, r3); // 9 - r3 = r3 | r4; // 10 - r4 = r4 ^ dataset[r3 & MASK]; // 11 - r0 = mulhi(r0, r4); // 12 - r5 = r5 + r1 + select(0xc7934706u, 0xd3177981u, ((sel >> 30u) & 1u) != 0u); // 13 - r0 = r0 ^ dataset[r4 & MASK]; // 14 - r2 = r2 - r4; // 15 - r2 = r2 ^ dataset[r0 & MASK]; // 16 - r7 = r7 ^ dataset[r2 & MASK]; // 17 - r7 = r7 ^ simd_shuffle_xor(r3, (ushort)4); // 18 - r5 = r5 * r0; // 19 - r3 = r3 ^ simd_shuffle_xor(r4, (ushort)2); // 20 - r2 = r2 ^ simd_shuffle_xor(r4, (ushort)16); // 21 - r6 = mulhi(r6, r2); // 22 - r6 = r6 ^ dataset[r1 & MASK]; // 23 - r5 = r5 * r0; // 24 - r5 = rotl_imm(r5, 19u); // 25 - r7 = r7 ^ simd_shuffle_xor(r6, (ushort)2); // 26 - r0 = r0 ^ r5; // 27 - r0 = r0 ^ r4; // 28 - r3 = r3 - r0; // 29 - r5 = r5 * r1; // 30 - r7 = r7 ^ dataset[r2 & MASK]; // 31 - r1 = r1 ^ dataset[r0 & MASK]; // 32 - r5 = r5 ^ r6; // 33 - r5 = r5 ^ dataset[r1 & MASK]; // 34 - r0 = mulhi(r0, r5); // 35 - r5 = r5 ^ simd_shuffle_xor(r2, (ushort)4); // 36 - r7 = r7 ^ dataset[r0 & MASK]; // 37 - r3 = r3 + r1 + select(0x75ba2fadu, 0x230c005cu, ((sel >> 27u) & 1u) != 0u); // 38 - r1 = r1 ^ simd_shuffle_xor(r5, (ushort)4); // 39 - r2 = r2 ^ r5; // 40 - r3 = r6 * r3 + r3; // 41 - r6 = r6 - r7; // 42 - r7 = r7 ^ r0; // 43 - r1 = r1 ^ dataset[r7 & MASK]; // 44 - r2 = r2 * r3; // 45 - r1 = mulhi(r1, r5); // 46 - r4 = r4 - r3; // 47 - r2 = rotr_var(r2, r6); // 48 - r3 = r3 ^ dataset[r5 & MASK]; // 49 - r1 = r1 + r5 + select(0x81b8bc2cu, 0x1907970cu, ((sel >> 7u) & 1u) != 0u); // 50 - r0 = r0 * r2; // 51 - r0 = r0 + r2 + select(0x4f92b968u, 0x699fd448u, ((sel >> 6u) & 1u) != 0u); // 52 - r1 = r1 + r0 + select(0x2bb965afu, 0x77b1520du, ((sel >> 12u) & 1u) != 0u); // 53 - r7 = rotl_imm(r7, 14u); // 54 - r3 = r3 + r7 + select(0x7b0fe07au, 0xa54c55a0u, ((sel >> 1u) & 1u) != 0u); // 55 - r6 = r6 ^ dataset[r7 & MASK]; // 56 - r1 = rotr_var(r1, r5); // 57 - r5 = r5 ^ dataset[r4 & MASK]; // 58 - r6 = r6 ^ dataset[r2 & MASK]; // 59 - r3 = r5 * r0 + r3; // 60 - r5 = r5 + r7 + select(0xaf9dd72du, 0xad7493e7u, ((sel >> 31u) & 1u) != 0u); // 61 - r4 = r4 + r6 + select(0x89841d87u, 0x1e07c3d9u, ((sel >> 27u) & 1u) != 0u); // 62 - r5 = rotl_imm(r5, 19u); // 63 - // int8 tile block (Counter ASIC 4.0 research, experimental): 128 mma.m8n8k16 u8 tiles per iteration, both outputs consumed - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t0 mm8 a=r6 b=r5 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t1 mm8 a=r1 b=r1 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t2 mm8 a=r2 b=r5 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t3 mm8 a=r1 b=r5 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t4 mm8 a=r2 b=r0 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t5 mm8 a=r2 b=r6 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t6 mm8 a=r1 b=r4 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t7 mm8 a=r4 b=r6 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t8 mm8 a=r1 b=r1 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t9 mm8 a=r3 b=r7 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t10 mm8 a=r3 b=r0 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t11 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t12 mm8 a=r3 b=r1 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t13 mm8 a=r1 b=r5 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t14 mm8 a=r3 b=r3 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t15 mm8 a=r1 b=r0 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t16 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t17 mm8 a=r0 b=r5 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t18 mm8 a=r1 b=r1 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t19 mm8 a=r3 b=r3 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t20 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t21 mm8 a=r0 b=r3 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t22 mm8 a=r1 b=r7 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t23 mm8 a=r1 b=r2 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t24 mm8 a=r3 b=r4 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t25 mm8 a=r5 b=r0 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t26 mm8 a=r1 b=r0 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t27 mm8 a=r3 b=r2 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t28 mm8 a=r2 b=r4 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t29 mm8 a=r3 b=r4 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t30 mm8 a=r6 b=r5 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t31 mm8 a=r1 b=r7 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t32 mm8 a=r7 b=r6 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t33 mm8 a=r7 b=r1 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t34 mm8 a=r1 b=r0 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t35 mm8 a=r2 b=r4 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t36 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t37 mm8 a=r1 b=r7 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t38 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t39 mm8 a=r6 b=r2 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t40 mm8 a=r6 b=r1 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t41 mm8 a=r6 b=r0 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t42 mm8 a=r7 b=r4 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t43 mm8 a=r6 b=r1 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t44 mm8 a=r0 b=r6 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t45 mm8 a=r6 b=r5 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t46 mm8 a=r6 b=r6 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t47 mm8 a=r7 b=r3 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t48 mm8 a=r7 b=r7 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t49 mm8 a=r4 b=r5 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t50 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t51 mm8 a=r3 b=r4 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t52 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t53 mm8 a=r7 b=r7 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t54 mm8 a=r7 b=r6 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t55 mm8 a=r1 b=r7 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t56 mm8 a=r4 b=r0 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t57 mm8 a=r4 b=r3 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t58 mm8 a=r2 b=r3 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t59 mm8 a=r6 b=r7 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t60 mm8 a=r6 b=r0 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t61 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t62 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t63 mm8 a=r1 b=r6 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t64 mm8 a=r0 b=r5 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t65 mm8 a=r0 b=r1 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t66 mm8 a=r3 b=r5 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t67 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t68 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t69 mm8 a=r6 b=r7 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t70 mm8 a=r1 b=r7 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t71 mm8 a=r3 b=r0 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t72 mm8 a=r4 b=r0 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t73 mm8 a=r4 b=r1 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t74 mm8 a=r5 b=r1 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t75 mm8 a=r4 b=r4 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t76 mm8 a=r5 b=r6 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t77 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t78 mm8 a=r2 b=r5 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t79 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t80 mm8 a=r6 b=r7 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t81 mm8 a=r4 b=r3 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t82 mm8 a=r6 b=r0 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t83 mm8 a=r2 b=r2 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t84 mm8 a=r0 b=r6 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t85 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t86 mm8 a=r3 b=r4 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t87 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t88 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t89 mm8 a=r4 b=r2 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t90 mm8 a=r6 b=r5 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t91 mm8 a=r6 b=r5 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t92 mm8 a=r2 b=r6 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t93 mm8 a=r2 b=r2 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t94 mm8 a=r2 b=r4 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t95 mm8 a=r5 b=r4 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t96 mm8 a=r0 b=r6 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t97 mm8 a=r0 b=r5 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t98 mm8 a=r2 b=r7 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t99 mm8 a=r1 b=r4 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t100 mm8 a=r2 b=r4 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t101 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t102 mm8 a=r5 b=r7 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t103 mm8 a=r4 b=r7 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t104 mm8 a=r2 b=r0 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t105 mm8 a=r6 b=r1 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t106 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t107 mm8 a=r2 b=r1 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t108 mm8 a=r4 b=r6 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t109 mm8 a=r2 b=r7 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t110 mm8 a=r1 b=r0 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t111 mm8 a=r6 b=r5 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t112 mm8 a=r5 b=r7 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t113 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t114 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t115 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t116 mm8 a=r7 b=r2 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t117 mm8 a=r7 b=r6 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t118 mm8 a=r0 b=r1 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t119 mm8 a=r4 b=r3 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t120 mm8 a=r5 b=r3 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t121 mm8 a=r2 b=r7 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t122 mm8 a=r0 b=r1 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t123 mm8 a=r5 b=r2 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t124 mm8 a=r5 b=r1 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t125 mm8 a=r7 b=r2 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t126 mm8 a=r6 b=r3 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t127 mm8 a=r6 b=r7 c=r5 c2=r1 - } - uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((ulong)hi << 32) | (ulong)lo; -} diff --git a/proto-cuda/packs-ca4/mx8_mm128/program_bound.metal b/proto-cuda/packs-ca4/mx8_mm128/program_bound.metal deleted file mode 100644 index 30fbce661..000000000 --- a/proto-cuda/packs-ca4/mx8_mm128/program_bound.metal +++ /dev/null @@ -1,259 +0,0 @@ -#include -using namespace metal; -// int8 tile reference (Counter ASIC 4.0 research): A = the 32 lanes' av as 8 x 16 bytes (lane l holds row l >> 2, bytes 4 (l & 3)..), -// B = bv as 16 x 8 (lane l holds column l >> 2); lane l gets C[l >> 2][2 (l & 3)] and C[l >> 2][2 (l & 3) + 1]; the PTX mma.m8n8k16 u8 layout. -static inline void mm8_ref(uint av, uint bv, uint lane, thread uint& d0, thread uint& d1) { - uint row = lane >> 2, col0 = 2u * (lane & 3u); - uint acc0 = 0u, acc1 = 0u; - for (uint j = 0u; j < 4u; ++j) { - uint aw = simd_shuffle(av, (ushort)(4u * row + j)); - uint bw0 = simd_shuffle(bv, (ushort)(4u * col0 + j)); - uint bw1 = simd_shuffle(bv, (ushort)(4u * (col0 + 1u) + j)); - for (uint t = 0u; t < 4u; ++t) { - uint ab = (aw >> (8u * t)) & 0xffu; - acc0 += ab * ((bw0 >> (8u * t)) & 0xffu); - acc1 += ab * ((bw1 >> (8u * t)) & 0xffu); - } - } - d0 = acc0; d1 = acc1; -} - - -#define MASK 0x0fffffffu -constant uint SEEDW[8] = { 0x67a9a7beu, 0x1a155b25u, 0xfddfb732u, 0x4b5af2e8u, 0xc55caf33u, 0xa27c13b7u, 0x06628a48u, 0x03852469u }; - -inline uint splitmix32(uint x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -inline uint rotl_imm(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 -inline uint rotr_var(uint x, uint n) { n &= 31u; return (x >> n) | (x << ((32u - n) & 31u)); } -inline uint ds_elem(uint i, uint d0, uint d1) { - uint x = i ^ d0; - x *= 0x9E3779B1u; x ^= x >> 15; - x += d1; - x *= 0x85EBCA77u; x ^= x >> 13; - x *= 0xC2B2AE3Du; x ^= x >> 16; - return x; -} - -// Header-bound variant: the init words come from buffer 3 (bind.rs), not from SEEDW. -kernel void igneum_hash_bound(device const uint* dataset [[buffer(0)]], - device ulong* out [[buffer(1)]], - constant uint& baseNonce [[buffer(2)]], - constant uint* initw [[buffer(3)]], - uint gid [[thread_position_in_grid]]) { - uint nonce = baseNonce + gid; - uint r0, r1, r2, r3, r4, r5, r6, r7; - uint lane = gid & 31u; - { uint x = nonce ^ initw[0]; x += 0x9e3779b9u * 1u; x = splitmix32(x); r0 = x ^ initw[1]; } - { uint x = nonce ^ initw[1]; x += 0x9e3779b9u * 2u; x = splitmix32(x); r1 = x ^ initw[2]; } - { uint x = nonce ^ initw[2]; x += 0x9e3779b9u * 3u; x = splitmix32(x); r2 = x ^ initw[3]; } - { uint x = nonce ^ initw[3]; x += 0x9e3779b9u * 4u; x = splitmix32(x); r3 = x ^ initw[4]; } - { uint x = nonce ^ initw[4]; x += 0x9e3779b9u * 5u; x = splitmix32(x); r4 = x ^ initw[5]; } - { uint x = nonce ^ initw[5]; x += 0x9e3779b9u * 6u; x = splitmix32(x); r5 = x ^ initw[6]; } - { uint x = nonce ^ initw[6]; x += 0x9e3779b9u * 7u; x = splitmix32(x); r6 = x ^ initw[7]; } - { uint x = nonce ^ initw[7]; x += 0x9e3779b9u * 8u; x = splitmix32(x); r7 = x ^ initw[0]; } - - for (uint it = 0u; it < 8u; ++it) { - uint sel = r0; - r2 = r3 * r4 + r2; // 0 - r2 = r1 * r1 + r2; // 1 - r2 = r3 * r2 + r2; // 2 - r3 = r3 ^ r5; // 3 - r7 = r7 ^ dataset[r2 & MASK]; // 4 - r5 = r5 ^ dataset[r7 & MASK]; // 5 - r1 = r1 ^ simd_shuffle_xor(r4, (ushort)8); // 6 - r7 = r7 ^ simd_shuffle_xor(r3, (ushort)8); // 7 - r1 = mulhi(r1, r5); // 8 - r6 = rotr_var(r6, r3); // 9 - r3 = r3 | r4; // 10 - r4 = r4 ^ dataset[r3 & MASK]; // 11 - r0 = mulhi(r0, r4); // 12 - r5 = r5 + r1 + select(0xc7934706u, 0xd3177981u, ((sel >> 30u) & 1u) != 0u); // 13 - r0 = r0 ^ dataset[r4 & MASK]; // 14 - r2 = r2 - r4; // 15 - r2 = r2 ^ dataset[r0 & MASK]; // 16 - r7 = r7 ^ dataset[r2 & MASK]; // 17 - r7 = r7 ^ simd_shuffle_xor(r3, (ushort)4); // 18 - r5 = r5 * r0; // 19 - r3 = r3 ^ simd_shuffle_xor(r4, (ushort)2); // 20 - r2 = r2 ^ simd_shuffle_xor(r4, (ushort)16); // 21 - r6 = mulhi(r6, r2); // 22 - r6 = r6 ^ dataset[r1 & MASK]; // 23 - r5 = r5 * r0; // 24 - r5 = rotl_imm(r5, 19u); // 25 - r7 = r7 ^ simd_shuffle_xor(r6, (ushort)2); // 26 - r0 = r0 ^ r5; // 27 - r0 = r0 ^ r4; // 28 - r3 = r3 - r0; // 29 - r5 = r5 * r1; // 30 - r7 = r7 ^ dataset[r2 & MASK]; // 31 - r1 = r1 ^ dataset[r0 & MASK]; // 32 - r5 = r5 ^ r6; // 33 - r5 = r5 ^ dataset[r1 & MASK]; // 34 - r0 = mulhi(r0, r5); // 35 - r5 = r5 ^ simd_shuffle_xor(r2, (ushort)4); // 36 - r7 = r7 ^ dataset[r0 & MASK]; // 37 - r3 = r3 + r1 + select(0x75ba2fadu, 0x230c005cu, ((sel >> 27u) & 1u) != 0u); // 38 - r1 = r1 ^ simd_shuffle_xor(r5, (ushort)4); // 39 - r2 = r2 ^ r5; // 40 - r3 = r6 * r3 + r3; // 41 - r6 = r6 - r7; // 42 - r7 = r7 ^ r0; // 43 - r1 = r1 ^ dataset[r7 & MASK]; // 44 - r2 = r2 * r3; // 45 - r1 = mulhi(r1, r5); // 46 - r4 = r4 - r3; // 47 - r2 = rotr_var(r2, r6); // 48 - r3 = r3 ^ dataset[r5 & MASK]; // 49 - r1 = r1 + r5 + select(0x81b8bc2cu, 0x1907970cu, ((sel >> 7u) & 1u) != 0u); // 50 - r0 = r0 * r2; // 51 - r0 = r0 + r2 + select(0x4f92b968u, 0x699fd448u, ((sel >> 6u) & 1u) != 0u); // 52 - r1 = r1 + r0 + select(0x2bb965afu, 0x77b1520du, ((sel >> 12u) & 1u) != 0u); // 53 - r7 = rotl_imm(r7, 14u); // 54 - r3 = r3 + r7 + select(0x7b0fe07au, 0xa54c55a0u, ((sel >> 1u) & 1u) != 0u); // 55 - r6 = r6 ^ dataset[r7 & MASK]; // 56 - r1 = rotr_var(r1, r5); // 57 - r5 = r5 ^ dataset[r4 & MASK]; // 58 - r6 = r6 ^ dataset[r2 & MASK]; // 59 - r3 = r5 * r0 + r3; // 60 - r5 = r5 + r7 + select(0xaf9dd72du, 0xad7493e7u, ((sel >> 31u) & 1u) != 0u); // 61 - r4 = r4 + r6 + select(0x89841d87u, 0x1e07c3d9u, ((sel >> 27u) & 1u) != 0u); // 62 - r5 = rotl_imm(r5, 19u); // 63 - // int8 tile block (Counter ASIC 4.0 research, experimental): 128 mma.m8n8k16 u8 tiles per iteration, both outputs consumed - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t0 mm8 a=r6 b=r5 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t1 mm8 a=r1 b=r1 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t2 mm8 a=r2 b=r5 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t3 mm8 a=r1 b=r5 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t4 mm8 a=r2 b=r0 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t5 mm8 a=r2 b=r6 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t6 mm8 a=r1 b=r4 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t7 mm8 a=r4 b=r6 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t8 mm8 a=r1 b=r1 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t9 mm8 a=r3 b=r7 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t10 mm8 a=r3 b=r0 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t11 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t12 mm8 a=r3 b=r1 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t13 mm8 a=r1 b=r5 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t14 mm8 a=r3 b=r3 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t15 mm8 a=r1 b=r0 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t16 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t17 mm8 a=r0 b=r5 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t18 mm8 a=r1 b=r1 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t19 mm8 a=r3 b=r3 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t20 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t21 mm8 a=r0 b=r3 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t22 mm8 a=r1 b=r7 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t23 mm8 a=r1 b=r2 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t24 mm8 a=r3 b=r4 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t25 mm8 a=r5 b=r0 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t26 mm8 a=r1 b=r0 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t27 mm8 a=r3 b=r2 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t28 mm8 a=r2 b=r4 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t29 mm8 a=r3 b=r4 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t30 mm8 a=r6 b=r5 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t31 mm8 a=r1 b=r7 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t32 mm8 a=r7 b=r6 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t33 mm8 a=r7 b=r1 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t34 mm8 a=r1 b=r0 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t35 mm8 a=r2 b=r4 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t36 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t37 mm8 a=r1 b=r7 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t38 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t39 mm8 a=r6 b=r2 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t40 mm8 a=r6 b=r1 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t41 mm8 a=r6 b=r0 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t42 mm8 a=r7 b=r4 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t43 mm8 a=r6 b=r1 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t44 mm8 a=r0 b=r6 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t45 mm8 a=r6 b=r5 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t46 mm8 a=r6 b=r6 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t47 mm8 a=r7 b=r3 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t48 mm8 a=r7 b=r7 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t49 mm8 a=r4 b=r5 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t50 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t51 mm8 a=r3 b=r4 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t52 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t53 mm8 a=r7 b=r7 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t54 mm8 a=r7 b=r6 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t55 mm8 a=r1 b=r7 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t56 mm8 a=r4 b=r0 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t57 mm8 a=r4 b=r3 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t58 mm8 a=r2 b=r3 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t59 mm8 a=r6 b=r7 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t60 mm8 a=r6 b=r0 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t61 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t62 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t63 mm8 a=r1 b=r6 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t64 mm8 a=r0 b=r5 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t65 mm8 a=r0 b=r1 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t66 mm8 a=r3 b=r5 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t67 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t68 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t69 mm8 a=r6 b=r7 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t70 mm8 a=r1 b=r7 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t71 mm8 a=r3 b=r0 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t72 mm8 a=r4 b=r0 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t73 mm8 a=r4 b=r1 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t74 mm8 a=r5 b=r1 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t75 mm8 a=r4 b=r4 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t76 mm8 a=r5 b=r6 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t77 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t78 mm8 a=r2 b=r5 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t79 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t80 mm8 a=r6 b=r7 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t81 mm8 a=r4 b=r3 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t82 mm8 a=r6 b=r0 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t83 mm8 a=r2 b=r2 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t84 mm8 a=r0 b=r6 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t85 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t86 mm8 a=r3 b=r4 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t87 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t88 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t89 mm8 a=r4 b=r2 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t90 mm8 a=r6 b=r5 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t91 mm8 a=r6 b=r5 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t92 mm8 a=r2 b=r6 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t93 mm8 a=r2 b=r2 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t94 mm8 a=r2 b=r4 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t95 mm8 a=r5 b=r4 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t96 mm8 a=r0 b=r6 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t97 mm8 a=r0 b=r5 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t98 mm8 a=r2 b=r7 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t99 mm8 a=r1 b=r4 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t100 mm8 a=r2 b=r4 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t101 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t102 mm8 a=r5 b=r7 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t103 mm8 a=r4 b=r7 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t104 mm8 a=r2 b=r0 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t105 mm8 a=r6 b=r1 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t106 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t107 mm8 a=r2 b=r1 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t108 mm8 a=r4 b=r6 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t109 mm8 a=r2 b=r7 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t110 mm8 a=r1 b=r0 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t111 mm8 a=r6 b=r5 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t112 mm8 a=r5 b=r7 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t113 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t114 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t115 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t116 mm8 a=r7 b=r2 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t117 mm8 a=r7 b=r6 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t118 mm8 a=r0 b=r1 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t119 mm8 a=r4 b=r3 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t120 mm8 a=r5 b=r3 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t121 mm8 a=r2 b=r7 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t122 mm8 a=r0 b=r1 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t123 mm8 a=r5 b=r2 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t124 mm8 a=r5 b=r1 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t125 mm8 a=r7 b=r2 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t126 mm8 a=r6 b=r3 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t127 mm8 a=r6 b=r7 c=r5 c2=r1 - } - uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((ulong)hi << 32) | (ulong)lo; -} diff --git a/proto-cuda/packs-ca4/mx8_mm128/vectors.h b/proto-cuda/packs-ca4/mx8_mm128/vectors.h deleted file mode 100644 index 64f736a01..000000000 --- a/proto-cuda/packs-ca4/mx8_mm128/vectors.h +++ /dev/null @@ -1,57 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// Expected outputs: igneum-pow (Rust) CPU interpreter, generator v2, memory-hard dataset -#pragma once -#ifdef __cplusplus -#include -#else -#include -#endif - -#define IGNEUM_VEC_WARPS 3 -static const uint32_t IGNEUM_VEC_BASE[IGNEUM_VEC_WARPS] = { 0u, 4096u, 1000000u }; -static const uint64_t IGNEUM_VEC_OUT[IGNEUM_VEC_WARPS][32] = { - { // base nonce 0 - 0x25b68ebd0e906b6dull, 0xd5072799aa3388f2ull, 0xcb9ef6af09df1727ull, 0x9640067bd458ba0full, 0xb425a8e6be6ebc66ull, 0xee0337da049776e0ull, 0x575cb409f02fe85cull, 0xf681e6a634ee274dull, - 0x6240bea3cd0618a3ull, 0x46542be4215eb40dull, 0xd473cfa0ee990014ull, 0x1edd1dde66d5ed15ull, 0x4e2614069b891edaull, 0xa0177b04b273df1eull, 0x8801079046cd297eull, 0xcbd7d0fc56746f2cull, - 0x6dd755062955ea76ull, 0x49c66be7bc453333ull, 0x77b6aa0a2e018a00ull, 0x03cd6c4473aaa4dfull, 0xc22947a71e2df6abull, 0x88dbfa375585588full, 0xee7ad5a0b9fd2cb0ull, 0x32e973e9d4d08f04ull, - 0x5216892aa97f054full, 0x0bb544fa1c3164a9ull, 0x287301a65944e03bull, 0x97ff3a15f4be15a1ull, 0x7efb80050078c725ull, 0x466e0db9cdf6cd16ull, 0x3edc67ee6fec9836ull, 0x719475da599c181cull - }, - { // base nonce 4096 - 0xe8a167c98997e9e4ull, 0x473e97b54279586full, 0xc936941504e34cdaull, 0xcc10b3fb38dbe2b1ull, 0xf5ec50127382b382ull, 0xfed36c8a20572550ull, 0xce2c26f4fc0a3b23ull, 0xbb007d40b8cb148full, - 0x828cf96280004a64ull, 0x0be4a5a3a24950faull, 0xa8fe47603236807dull, 0x5126fa0bfc2ff05dull, 0xc1c82fd752965fb2ull, 0xbb03e9a649e56fd6ull, 0x7665244f65ac6f68ull, 0xd7e98be5687212f5ull, - 0x158292648b58b97full, 0x17fe5c2db3fa4ffeull, 0xe3fff009c0baabfdull, 0x088db9de945c3641ull, 0xa9b4f2921081e576ull, 0x8e6ff7cd0b306b67ull, 0x3f297f8cd5f05e74ull, 0xcdc25fde19f03490ull, - 0x5347a7990e98f475ull, 0x054eaf4ed50cdb55ull, 0x6475d5d6ca0bbe31ull, 0xc49093e12468a24eull, 0x6bf26f2aa7164fc0ull, 0x6016f20b26bec311ull, 0x553d1d640b452da0ull, 0x14a02c647d6fd301ull - }, - { // base nonce 1000000 - 0x7bdfc082cefc7113ull, 0xbe8d8421159c699bull, 0x79462f47122923a3ull, 0xd8051e3d6c418791ull, 0xad6d08bc7fb8e839ull, 0xd16fe4bbadb350cfull, 0xe52871c076a7b822ull, 0x7caabc8c2924d1b6ull, - 0x11f56e922cd78df5ull, 0xdec94f81ca18bfb2ull, 0xfcc2d3a09cab249eull, 0x3d5157679833c75dull, 0xd030fbf56d44a09aull, 0x74d4aad3fdfdea27ull, 0x198020289b4880a4ull, 0x921135a661e458feull, - 0x77536f5c70aa1154ull, 0x32ed08793a3819e4ull, 0x316a1ca8cbf71df8ull, 0x01607d5115ba8a6full, 0xcf68787c5b013feaull, 0x8a2189bac924f312ull, 0x5b53bedb0dd0d6aaull, 0x7d1d537591d9394aull, - 0x1fe717f8a7c38824ull, 0xcd21f17ea407ab99ull, 0xcf36d53b85d645c7ull, 0x9c4dd4372c8513dcull, 0xe842a34932cf0fa5ull, 0x67d0f83d144c3293ull, 0x12c9e72434886b71ull, 0x6a62329215ace047ull - } -}; - -// Dataset self-test: dataset[0..15] and dataset[IGNEUM_MASK] (268435455). -static const uint32_t IGNEUM_DS_HEAD[16] = { - 0xfdad4319u, 0x1a7b68e1u, 0xde6db608u, 0x13d73892u, 0xd17f447au, 0xb2221ccfu, 0x9db004bdu, 0x57d7d367u, - 0xdbc4cf34u, 0x697c009au, 0xc43af1d4u, 0x97f12b2eu, 0x74c37cd0u, 0xc651ea15u, 0x665a6d29u, 0x22330a2du -}; -static const uint32_t IGNEUM_DS_LAST_INDEX = 268435455u; -static const uint32_t IGNEUM_DS_LAST = 0xa83e7aa6u; -// 64 sampled dataset words (index, value) computed on the Mac. -#define IGNEUM_DS_SAMPLES 64 -static const uint32_t IGNEUM_DS_SAMPLE_INDEX[IGNEUM_DS_SAMPLES] = { - 59471966u, 217795994u, 208353206u, 42483309u, 172547758u, 148076330u, 183853158u, 214389424u, 267488061u, 169781097u, 184093494u, 153880993u, 84977930u, 46426879u, 3093825u, 225364072u, 44593546u, 260713159u, 168250303u, 52384140u, 223401610u, 45554030u, 95410555u, 175039924u, 79171087u, 267580473u, 24168642u, 37981670u, 171551130u, 195559979u, 204611762u, 140997658u, 138925853u, 86637313u, 20736778u, 219665210u, 160430336u, 264654675u, 8013395u, 228945585u, 213884386u, 104419827u, 44185464u, 142737231u, 99284897u, 132475900u, 61861762u, 132056166u, 262388043u, 91878046u, 117353561u, 124768597u, 71352993u, 190698941u, 46055428u, 55281366u, 165145231u, 106810753u, 171985651u, 232085256u, 159510492u, 40072060u, 209107596u, 39023794u -}; -static const uint32_t IGNEUM_DS_SAMPLE_VALUE[IGNEUM_DS_SAMPLES] = { - 0x3230bc7bu, 0x7fbfe2c9u, 0xb2690991u, 0x1745c7c5u, 0x0ab0ccafu, 0x1bf87d6bu, 0x160139fdu, 0x719817acu, 0x0155df4bu, 0xbe1e86c3u, 0x680bcd6cu, 0x79c3dc6cu, 0x181e7e5fu, 0x0713a109u, 0xc705dd9fu, 0x3933b7a8u, 0xdd1c0431u, 0x50522b30u, 0xa0020b38u, 0xbff39e96u, 0x21b67e18u, 0x740f8db3u, 0x2baba568u, 0x2c9bef83u, 0x0ad9b671u, 0xc4327869u, 0x7b4fd7d0u, 0x2c29965fu, 0xec56f15fu, 0x61111746u, 0x303a1d6eu, 0xbddcfd1au, 0xf829a355u, 0x6d5df2a9u, 0x01ab8e44u, 0x06d13507u, 0xda8dcfc6u, 0x01a703e1u, 0xafe7d2c1u, 0xc091c3a2u, 0xac1814feu, 0x6e6ff62au, 0x8fdf01bau, 0xdd3f7159u, 0xdfa0d75cu, 0x26684c35u, 0x7f441e63u, 0x88df2570u, 0x8aa4d5ebu, 0xcc816c05u, 0x434df890u, 0xcd392ad6u, 0x1ab4cb63u, 0x595926fau, 0x7cd76b41u, 0x20cb95c4u, 0x13cf823fu, 0xf9daf901u, 0xff9af40au, 0x2c7dfa51u, 0x871206dbu, 0x938c116cu, 0xb64bf199u, 0x5751f874u -}; -// Cache self-test (memory-hard mode): cache[0..15], the last 16 words, and FNV-1a 64 over all 2^26 words. -static const uint32_t IGNEUM_CACHE_HEAD[16] = { - 0x355a86d2u, 0x7957db1cu, 0xd21772afu, 0x6fc1e09bu, 0xd55ce61du, 0x6e6a278bu, 0xd3f543ceu, 0x223d8e82u, - 0x143ab337u, 0x2e9f05bdu, 0x2eb389bfu, 0x0c6e449eu, 0x5cfa4222u, 0xba6560feu, 0x8e3e1aa4u, 0xdbcc1d53u -}; -static const uint32_t IGNEUM_CACHE_LAST[16] = { - 0x41190d91u, 0xbd277957u, 0x22ddbb49u, 0x6986f207u, 0xdf69a4d6u, 0x26401a3au, 0x818230fbu, 0xc417122du, - 0x3597b211u, 0xb553ce55u, 0xcf39cc0du, 0x3b7fc43au, 0x3fd43b00u, 0x67e1c80eu, 0xffa7ea7du, 0xca2960abu -}; -static const uint64_t IGNEUM_CACHE_FNV64 = 0x48c4f5bf24166b2eull; diff --git a/proto-cuda/packs-ca4/mx8_mm128/vectors.json b/proto-cuda/packs-ca4/mx8_mm128/vectors.json deleted file mode 100644 index 59bd1238f..000000000 --- a/proto-cuda/packs-ca4/mx8_mm128/vectors.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "seed": "igneum-genesis", - "day": "2026-10-03", - "dataset_mode": "memory-hard", - "dataset_log2_words": 28, - "mask": "0x0fffffff", - "lanes": 32, - "source": "igneum-pow (Rust) CPU interpreter, generator v2, memory-hard dataset", - "warps": [ - {"base_nonce": 0, "expected": [ - "0x25b68ebd0e906b6d", "0xd5072799aa3388f2", "0xcb9ef6af09df1727", "0x9640067bd458ba0f", "0xb425a8e6be6ebc66", "0xee0337da049776e0", "0x575cb409f02fe85c", "0xf681e6a634ee274d", - "0x6240bea3cd0618a3", "0x46542be4215eb40d", "0xd473cfa0ee990014", "0x1edd1dde66d5ed15", "0x4e2614069b891eda", "0xa0177b04b273df1e", "0x8801079046cd297e", "0xcbd7d0fc56746f2c", - "0x6dd755062955ea76", "0x49c66be7bc453333", "0x77b6aa0a2e018a00", "0x03cd6c4473aaa4df", "0xc22947a71e2df6ab", "0x88dbfa375585588f", "0xee7ad5a0b9fd2cb0", "0x32e973e9d4d08f04", - "0x5216892aa97f054f", "0x0bb544fa1c3164a9", "0x287301a65944e03b", "0x97ff3a15f4be15a1", "0x7efb80050078c725", "0x466e0db9cdf6cd16", "0x3edc67ee6fec9836", "0x719475da599c181c" - ]}, - {"base_nonce": 4096, "expected": [ - "0xe8a167c98997e9e4", "0x473e97b54279586f", "0xc936941504e34cda", "0xcc10b3fb38dbe2b1", "0xf5ec50127382b382", "0xfed36c8a20572550", "0xce2c26f4fc0a3b23", "0xbb007d40b8cb148f", - "0x828cf96280004a64", "0x0be4a5a3a24950fa", "0xa8fe47603236807d", "0x5126fa0bfc2ff05d", "0xc1c82fd752965fb2", "0xbb03e9a649e56fd6", "0x7665244f65ac6f68", "0xd7e98be5687212f5", - "0x158292648b58b97f", "0x17fe5c2db3fa4ffe", "0xe3fff009c0baabfd", "0x088db9de945c3641", "0xa9b4f2921081e576", "0x8e6ff7cd0b306b67", "0x3f297f8cd5f05e74", "0xcdc25fde19f03490", - "0x5347a7990e98f475", "0x054eaf4ed50cdb55", "0x6475d5d6ca0bbe31", "0xc49093e12468a24e", "0x6bf26f2aa7164fc0", "0x6016f20b26bec311", "0x553d1d640b452da0", "0x14a02c647d6fd301" - ]}, - {"base_nonce": 1000000, "expected": [ - "0x7bdfc082cefc7113", "0xbe8d8421159c699b", "0x79462f47122923a3", "0xd8051e3d6c418791", "0xad6d08bc7fb8e839", "0xd16fe4bbadb350cf", "0xe52871c076a7b822", "0x7caabc8c2924d1b6", - "0x11f56e922cd78df5", "0xdec94f81ca18bfb2", "0xfcc2d3a09cab249e", "0x3d5157679833c75d", "0xd030fbf56d44a09a", "0x74d4aad3fdfdea27", "0x198020289b4880a4", "0x921135a661e458fe", - "0x77536f5c70aa1154", "0x32ed08793a3819e4", "0x316a1ca8cbf71df8", "0x01607d5115ba8a6f", "0xcf68787c5b013fea", "0x8a2189bac924f312", "0x5b53bedb0dd0d6aa", "0x7d1d537591d9394a", - "0x1fe717f8a7c38824", "0xcd21f17ea407ab99", "0xcf36d53b85d645c7", "0x9c4dd4372c8513dc", "0xe842a34932cf0fa5", "0x67d0f83d144c3293", "0x12c9e72434886b71", "0x6a62329215ace047" - ]} - ], - "dataset_head": ["0xfdad4319", "0x1a7b68e1", "0xde6db608", "0x13d73892", "0xd17f447a", "0xb2221ccf", "0x9db004bd", "0x57d7d367", "0xdbc4cf34", "0x697c009a", "0xc43af1d4", "0x97f12b2e", "0x74c37cd0", "0xc651ea15", "0x665a6d29", "0x22330a2d"], - "dataset_last_index": 268435455, - "dataset_last": "0xa83e7aa6", - "dataset_samples": [{"index": 59471966, "value": "0x3230bc7b"}, {"index": 217795994, "value": "0x7fbfe2c9"}, {"index": 208353206, "value": "0xb2690991"}, {"index": 42483309, "value": "0x1745c7c5"}, {"index": 172547758, "value": "0x0ab0ccaf"}, {"index": 148076330, "value": "0x1bf87d6b"}, {"index": 183853158, "value": "0x160139fd"}, {"index": 214389424, "value": "0x719817ac"}, {"index": 267488061, "value": "0x0155df4b"}, {"index": 169781097, "value": "0xbe1e86c3"}, {"index": 184093494, "value": "0x680bcd6c"}, {"index": 153880993, "value": "0x79c3dc6c"}, {"index": 84977930, "value": "0x181e7e5f"}, {"index": 46426879, "value": "0x0713a109"}, {"index": 3093825, "value": "0xc705dd9f"}, {"index": 225364072, "value": "0x3933b7a8"}, {"index": 44593546, "value": "0xdd1c0431"}, {"index": 260713159, "value": "0x50522b30"}, {"index": 168250303, "value": "0xa0020b38"}, {"index": 52384140, "value": "0xbff39e96"}, {"index": 223401610, "value": "0x21b67e18"}, {"index": 45554030, "value": "0x740f8db3"}, {"index": 95410555, "value": "0x2baba568"}, {"index": 175039924, "value": "0x2c9bef83"}, {"index": 79171087, "value": "0x0ad9b671"}, {"index": 267580473, "value": "0xc4327869"}, {"index": 24168642, "value": "0x7b4fd7d0"}, {"index": 37981670, "value": "0x2c29965f"}, {"index": 171551130, "value": "0xec56f15f"}, {"index": 195559979, "value": "0x61111746"}, {"index": 204611762, "value": "0x303a1d6e"}, {"index": 140997658, "value": "0xbddcfd1a"}, {"index": 138925853, "value": "0xf829a355"}, {"index": 86637313, "value": "0x6d5df2a9"}, {"index": 20736778, "value": "0x01ab8e44"}, {"index": 219665210, "value": "0x06d13507"}, {"index": 160430336, "value": "0xda8dcfc6"}, {"index": 264654675, "value": "0x01a703e1"}, {"index": 8013395, "value": "0xafe7d2c1"}, {"index": 228945585, "value": "0xc091c3a2"}, {"index": 213884386, "value": "0xac1814fe"}, {"index": 104419827, "value": "0x6e6ff62a"}, {"index": 44185464, "value": "0x8fdf01ba"}, {"index": 142737231, "value": "0xdd3f7159"}, {"index": 99284897, "value": "0xdfa0d75c"}, {"index": 132475900, "value": "0x26684c35"}, {"index": 61861762, "value": "0x7f441e63"}, {"index": 132056166, "value": "0x88df2570"}, {"index": 262388043, "value": "0x8aa4d5eb"}, {"index": 91878046, "value": "0xcc816c05"}, {"index": 117353561, "value": "0x434df890"}, {"index": 124768597, "value": "0xcd392ad6"}, {"index": 71352993, "value": "0x1ab4cb63"}, {"index": 190698941, "value": "0x595926fa"}, {"index": 46055428, "value": "0x7cd76b41"}, {"index": 55281366, "value": "0x20cb95c4"}, {"index": 165145231, "value": "0x13cf823f"}, {"index": 106810753, "value": "0xf9daf901"}, {"index": 171985651, "value": "0xff9af40a"}, {"index": 232085256, "value": "0x2c7dfa51"}, {"index": 159510492, "value": "0x871206db"}, {"index": 40072060, "value": "0x938c116c"}, {"index": 209107596, "value": "0xb64bf199"}, {"index": 39023794, "value": "0x5751f874"}], - "cache_head": ["0x355a86d2", "0x7957db1c", "0xd21772af", "0x6fc1e09b", "0xd55ce61d", "0x6e6a278b", "0xd3f543ce", "0x223d8e82", "0x143ab337", "0x2e9f05bd", "0x2eb389bf", "0x0c6e449e", "0x5cfa4222", "0xba6560fe", "0x8e3e1aa4", "0xdbcc1d53"], - "cache_last_line": ["0x41190d91", "0xbd277957", "0x22ddbb49", "0x6986f207", "0xdf69a4d6", "0x26401a3a", "0x818230fb", "0xc417122d", "0x3597b211", "0xb553ce55", "0xcf39cc0d", "0x3b7fc43a", "0x3fd43b00", "0x67e1c80e", "0xffa7ea7d", "0xca2960ab"], - "cache_fnv1a64": "0x48c4f5bf24166b2e" -} diff --git a/proto-cuda/packs-ca4/mx8_mm1430/kernel.cl b/proto-cuda/packs-ca4/mx8_mm1430/kernel.cl deleted file mode 100644 index ccb2cefd2..000000000 --- a/proto-cuda/packs-ca4/mx8_mm1430/kernel.cl +++ /dev/null @@ -1,1717 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// OpenCL C twin of the Metal kernel for the same seed (see proto-opencl/README.md, WAVEFRONT.md and program.metal). -// Built from source at runtime by proto-opencl/host.c, which passes these defines: -// IGNEUM_GROUP work-group size of igneum_hash, a multiple of 32 (default 32: one work-group = one 32-lane unit) -// IGNEUM_EXCHANGE 0 = local-memory exchange with a barrier (any device, any wave width; the default) -// 1 = sub_group_shuffle_xor (cl_khr_subgroup_shuffle), only with IGNEUM_GROUP 32 and a sub-group size of exactly 32 -// 2 = intel_sub_group_shuffle_xor (cl_intel_subgroups), same condition -// The verification unit is always 32 lanes. A 64-wide hardware wave (AMD GCN/CDNA, RDNA in wave64) runs two units; -// the exchange masks are 1, 2, 4, 8, 16, so every partner lane lies inside the lane's own aligned run of 32. -#ifndef IGNEUM_GROUP -#define IGNEUM_GROUP 32 -#endif -#ifndef IGNEUM_EXCHANGE -#define IGNEUM_EXCHANGE 0 -#endif -#ifdef __OPENCL_VERSION__ -#define IGNEUM_KERNEL_HASH __kernel __attribute__((reqd_work_group_size(IGNEUM_GROUP, 1, 1))) -#define IGNEUM_LOCAL_WORDS(name, n) __local uint name[n] -#if IGNEUM_EXCHANGE == 1 -#ifdef cl_khr_subgroups -#pragma OPENCL EXTENSION cl_khr_subgroups : enable -#endif -#ifdef cl_khr_subgroup_shuffle -#pragma OPENCL EXTENSION cl_khr_subgroup_shuffle : enable -#endif -#elif IGNEUM_EXCHANGE == 2 -#pragma OPENCL EXTENSION cl_intel_subgroups : enable -#endif -#else -// Not an OpenCL compiler: proto-opencl/emu compiles this file as C++ and supplies the built-ins and these two macros. -#include "emu_opencl.h" -#endif - -#if IGNEUM_EXCHANGE == 1 -#define IGNEUM_SHFL_XOR(dst, a, m) dst = sub_group_shuffle_xor((a), (uint)(m)) -#define IGNEUM_BCAST0(dst, a) dst = sub_group_broadcast((a), 0u) -#define IGNEUM_SHFL_IDX(dst, a, i) dst = sub_group_shuffle((a), (uint)(i)) -#elif IGNEUM_EXCHANGE == 2 -#define IGNEUM_SHFL_XOR(dst, a, m) dst = intel_sub_group_shuffle_xor((a), (uint)(m)) -#define IGNEUM_BCAST0(dst, a) dst = sub_group_broadcast((a), 0u) -#define IGNEUM_SHFL_IDX(dst, a, i) dst = intel_sub_group_shuffle((a), (uint)(i)) -#else -// Local-memory exchange. Two buffers of IGNEUM_GROUP words alternate (xk counts exchanges), so one barrier per -// exchange is enough: a lane can only overwrite buffer b at exchange k+2 after passing barrier k+1, and every lane -// reaches barrier k+1 only after its read of buffer b at exchange k. The partner lid ^ m stays inside the lane's -// aligned run of 32 because m < 32. Control flow is uniform, so every work-item reaches every barrier. -#define IGNEUM_SHFL_XOR(dst, a, m) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid ^ (uint)(m))]; xk += 1u; } -#define IGNEUM_BCAST0(dst, a) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid & ~31u)]; xk += 1u; } -#define IGNEUM_SHFL_IDX(dst, a, i) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid & ~31u) + (uint)(i)]; xk += 1u; } -#endif - -// int8 tile reference (Counter ASIC 4.0 research): 12 index shuffles and 32 byte products per lane; the PTX mma.m8n8k16 u8 layout. -#define IGNEUM_MM8_REF(av, bv, d0, d1) { uint row_ = lane >> 2, col0_ = 2u * (lane & 3u); uint acc0_ = 0u, acc1_ = 0u; for (uint j_ = 0u; j_ < 4u; ++j_) { uint aw_, bw0_, bw1_; IGNEUM_SHFL_IDX(aw_, (av), 4u * row_ + j_); IGNEUM_SHFL_IDX(bw0_, (bv), 4u * col0_ + j_); IGNEUM_SHFL_IDX(bw1_, (bv), 4u * (col0_ + 1u) + j_); for (uint t_ = 0u; t_ < 4u; ++t_) { uint ab_ = (aw_ >> (8u * t_)) & 0xffu; acc0_ += ab_ * ((bw0_ >> (8u * t_)) & 0xffu); acc1_ += ab_ * ((bw1_ >> (8u * t_)) & 0xffu); } } d0 = acc0_; d1 = acc1_; } - -static inline uint splitmix32(uint x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -// n is a literal in 1..31 at every call site. OpenCL rotate() rotates left by n modulo 32. -static inline uint rotl_imm(uint x, uint n) { return rotate(x, n); } -// Right rotation by n modulo 32 as a left rotation by (32 - n) modulo 32; n == 0 gives x. -static inline uint rotr_var(uint x, uint n) { return rotate(x, (0u - n) & 31u); } -static inline uint ds_elem(uint i, uint d0, uint d1) { - uint x = i ^ d0; - x *= 0x9E3779B1u; x ^= x >> 15; - x += d1; - x *= 0x85EBCA77u; x ^= x >> 13; - x *= 0xC2B2AE3Du; x ^= x >> 16; - return x; -} - -// Memory-hard dataset core (MEMHARD.md). Cache: 2^26 words in 2^16 segments of 64 chained ChaCha12 lines. -// Item: 8 rounds of 8 x seed-parameterised mixer + one 64-byte cache read, then 8 x final mixer (class v3, mixer multiplier 8, -// docs/plans/mixer-x4.md: the round key of application j of round r is 0x9E3779B9 * (r * m + j + 1)). All parameters are literals. -#define MH_CACHE_LINE_MASK 0x003fffffu -#define MH_SEGMENT_LINES 64u -#define MH_QR(a, b, c, d, r1, r2, r3, r4) { a += b; d ^= a; d = mh_rotl(d, r1); c += d; b ^= c; b = mh_rotl(b, r2); a += b; d ^= a; d = mh_rotl(d, r3); c += d; b ^= c; b = mh_rotl(b, r4); } -static inline uint mh_rotl(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 at every call site - -// y = ChaCha12 core(x) + x -static inline void mh_chacha_block(const uint* x, uint* y) { - for (uint i = 0u; i < 16u; ++i) y[i] = x[i]; - for (uint r = 0u; r < 6u; ++r) { - MH_QR(y[0], y[4], y[8], y[12], 16u, 12u, 8u, 7u) MH_QR(y[1], y[5], y[9], y[13], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[6], y[10], y[14], 16u, 12u, 8u, 7u) MH_QR(y[3], y[7], y[11], y[15], 16u, 12u, 8u, 7u) - MH_QR(y[0], y[5], y[10], y[15], 16u, 12u, 8u, 7u) MH_QR(y[1], y[6], y[11], y[12], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[7], y[8], y[13], 16u, 12u, 8u, 7u) MH_QR(y[3], y[4], y[9], y[14], 16u, 12u, 8u, 7u) - } - for (uint i = 0u; i < 16u; ++i) y[i] += x[i]; -} - -// One cache segment: 64 chained lines written at cache[seg * 1024]. in_j = prev ^ (sigma || K || seg || j || tag), prev_0 = 0. -static inline void mh_cache_segment(__global uint* cache, uint seg) { - uint prev[16]; uint x[16]; uint y[16]; - for (uint i = 0u; i < 16u; ++i) prev[i] = 0u; - for (uint j = 0u; j < MH_SEGMENT_LINES; ++j) { - x[0] = 0x61707865u ^ prev[0]; x[1] = 0x3320646eu ^ prev[1]; x[2] = 0x79622d32u ^ prev[2]; x[3] = 0x6b206574u ^ prev[3]; - x[4] = 0x3067619fu ^ prev[4]; - x[5] = 0x3c269176u ^ prev[5]; - x[6] = 0x84a03b03u ^ prev[6]; - x[7] = 0xf8c63294u ^ prev[7]; - x[8] = 0xff977c5bu ^ prev[8]; - x[9] = 0xe60def3eu ^ prev[9]; - x[10] = 0x63630141u ^ prev[10]; - x[11] = 0xb8fbcb58u ^ prev[11]; - x[12] = seg ^ prev[12]; x[13] = j ^ prev[13]; x[14] = 0x49676e65u ^ prev[14]; x[15] = 0x756d4d48u ^ prev[15]; - mh_chacha_block(x, y); - __global uint* line = cache + ((seg * MH_SEGMENT_LINES + j) * 16u); - for (uint i = 0u; i < 16u; ++i) { line[i] = y[i]; prev[i] = y[i]; } - } -} - -// M_r: per word (s ^ (RC + rk)) * MUL, then a column round and a diagonal round with the seed-drawn rotations. -static inline void mh_mixer(uint* s, uint rk) { - s[0] = (s[0] ^ (0xbab68293u + rk)) * 0x42146205u; - s[1] = (s[1] ^ (0xcc162340u + rk)) * 0x52cbe0fbu; - s[2] = (s[2] ^ (0x6ce151ccu + rk)) * 0x7ecf4a03u; - s[3] = (s[3] ^ (0xe62b8997u + rk)) * 0x6728907fu; - s[4] = (s[4] ^ (0xc9c80297u + rk)) * 0xd81d9751u; - s[5] = (s[5] ^ (0xf74a1654u + rk)) * 0x132952c3u; - s[6] = (s[6] ^ (0x3d704af5u + rk)) * 0xf60de277u; - s[7] = (s[7] ^ (0x3cf522b7u + rk)) * 0x05358035u; - s[8] = (s[8] ^ (0x2b9cac04u + rk)) * 0xbaf6499du; - s[9] = (s[9] ^ (0xa880ac10u + rk)) * 0xe4db9667u; - s[10] = (s[10] ^ (0x13e5dd1du + rk)) * 0x3e98f45du; - s[11] = (s[11] ^ (0x6fc3e233u + rk)) * 0xd0004eddu; - s[12] = (s[12] ^ (0x2d83eeacu + rk)) * 0x2691630du; - s[13] = (s[13] ^ (0x9006e8bfu + rk)) * 0x9beb3bcfu; - s[14] = (s[14] ^ (0x2c4b5362u + rk)) * 0xab310379u; - s[15] = (s[15] ^ (0x31b49ee2u + rk)) * 0x99cfb423u; - MH_QR(s[0], s[4], s[8], s[12], 20u, 20u, 19u, 4u) MH_QR(s[1], s[5], s[9], s[13], 20u, 20u, 19u, 4u) - MH_QR(s[2], s[6], s[10], s[14], 20u, 20u, 19u, 4u) MH_QR(s[3], s[7], s[11], s[15], 20u, 20u, 19u, 4u) - MH_QR(s[0], s[5], s[10], s[15], 26u, 3u, 3u, 27u) MH_QR(s[1], s[6], s[11], s[12], 26u, 3u, 3u, 27u) - MH_QR(s[2], s[7], s[8], s[13], 26u, 3u, 3u, 27u) MH_QR(s[3], s[4], s[9], s[14], 26u, 3u, 3u, 27u) -} - -// Item t: 16 words. s = (K, t * MUL[i] + RC[i]); 8 rounds of 8 x mixer + cache line s[0] & mask; 8 x final mixer. -static inline void mh_item(__global const uint* cache, uint t, uint* s) { - s[0] = 0x3067619fu; - s[1] = 0x3c269176u; - s[2] = 0x84a03b03u; - s[3] = 0xf8c63294u; - s[4] = 0xff977c5bu; - s[5] = 0xe60def3eu; - s[6] = 0x63630141u; - s[7] = 0xb8fbcb58u; - s[8] = t * 0x42146205u + 0xbab68293u; - s[9] = t * 0x52cbe0fbu + 0xcc162340u; - s[10] = t * 0x7ecf4a03u + 0x6ce151ccu; - s[11] = t * 0x6728907fu + 0xe62b8997u; - s[12] = t * 0xd81d9751u + 0xc9c80297u; - s[13] = t * 0x132952c3u + 0xf74a1654u; - s[14] = t * 0xf60de277u + 0x3d704af5u; - s[15] = t * 0x05358035u + 0x3cf522b7u; - for (uint r = 0u; r < 8u; ++r) { - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (r * 8u + j + 1u)); - __global const uint* line = cache + ((s[0] & MH_CACHE_LINE_MASK) * 16u); - for (uint i = 0u; i < 16u; ++i) s[i] ^= line[i]; - } - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (64u + j + 1u)); -} -// dataset[w] without the dataset: derive item w >> 4 and take word w & 15. -static inline uint mh_word(__global const uint* cache, uint w) { uint s[16]; mh_item(cache, w >> 4u, s); return s[w & 15u]; } - -// Memory-hard dataset (MEMHARD.md). One work-item per cache segment; one work-item per 64-byte dataset item. -// The same constants as memhard.h in this pack (one emitter, three dialects). -__kernel void igneum_cache_fill(__global uint* cache, uint nSegments) { - uint seg = (uint)get_global_id(0); - if (seg < nSegments) mh_cache_segment(cache, seg); -} -__kernel void igneum_build(__global uint* ds, __global const uint* cache, uint nItems) { - uint t = (uint)get_global_id(0); - if (t < nItems) { - uint s[16]; - mh_item(cache, t, s); - __global uint* d = ds + ((ulong)t * 16u); - for (uint i = 0u; i < 16u; ++i) d[i] = s[i]; - } -} - -// One hash per work-item. IGNEUM_GROUP is a multiple of 32; lane = lid & 31 and every exchange stays inside the -// lane's own aligned run of 32 work-items, exactly like simd_shuffle_xor inside a 32-wide Metal SIMD group and -// __shfl_xor_sync inside a CUDA warp. Control flow is uniform (no branches at all). -IGNEUM_KERNEL_HASH void igneum_hash(__global const uint* ds, __global ulong* out, uint baseNonce, uint mask) { - uint gid = (uint)get_global_id(0); - uint lid = (uint)get_local_id(0); - uint nonce = baseNonce + gid; - uint r0, r1, r2, r3, r4, r5, r6, r7; -#if IGNEUM_EXCHANGE == 0 - IGNEUM_LOCAL_WORDS(xch, 2 * IGNEUM_GROUP); - uint xk = 0u; -#else - (void)lid; -#endif - uint lane = lid & 31u; - { uint x = nonce ^ 0x67a9a7beu; x += 0x9e3779b9u; x = splitmix32(x); r0 = x ^ 0x1a155b25u; } // SEEDW[0], 0x9e3779b9u * 1u, SEEDW[1] - { uint x = nonce ^ 0x1a155b25u; x += 0x3c6ef372u; x = splitmix32(x); r1 = x ^ 0xfddfb732u; } // SEEDW[1], 0x9e3779b9u * 2u, SEEDW[2] - { uint x = nonce ^ 0xfddfb732u; x += 0xdaa66d2bu; x = splitmix32(x); r2 = x ^ 0x4b5af2e8u; } // SEEDW[2], 0x9e3779b9u * 3u, SEEDW[3] - { uint x = nonce ^ 0x4b5af2e8u; x += 0x78dde6e4u; x = splitmix32(x); r3 = x ^ 0xc55caf33u; } // SEEDW[3], 0x9e3779b9u * 4u, SEEDW[4] - { uint x = nonce ^ 0xc55caf33u; x += 0x1715609du; x = splitmix32(x); r4 = x ^ 0xa27c13b7u; } // SEEDW[4], 0x9e3779b9u * 5u, SEEDW[5] - { uint x = nonce ^ 0xa27c13b7u; x += 0xb54cda56u; x = splitmix32(x); r5 = x ^ 0x06628a48u; } // SEEDW[5], 0x9e3779b9u * 6u, SEEDW[6] - { uint x = nonce ^ 0x06628a48u; x += 0x5384540fu; x = splitmix32(x); r6 = x ^ 0x03852469u; } // SEEDW[6], 0x9e3779b9u * 7u, SEEDW[7] - { uint x = nonce ^ 0x03852469u; x += 0xf1bbcdc8u; x = splitmix32(x); r7 = x ^ 0x67a9a7beu; } // SEEDW[7], 0x9e3779b9u * 8u, SEEDW[0] - - for (uint it = 0u; it < 8u; ++it) { - uint sel = r0; - r2 = r3 * r4 + r2; // 0 mad - r2 = r1 * r1 + r2; // 1 mad - r2 = r3 * r2 + r2; // 2 mad - r3 = r3 ^ r5; // 3 xor - r7 = r7 ^ ds[r2 & mask]; // 4 load - r5 = r5 ^ ds[r7 & mask]; // 5 load - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 8u); r1 = r1 ^ t_; } // 6 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 8u); r7 = r7 ^ t_; } // 7 shfl - r1 = mul_hi(r1, r5); // 8 mulhi - r6 = rotr_var(r6, r3); // 9 rotr - r3 = r3 | r4; // 10 or - r4 = r4 ^ ds[r3 & mask]; // 11 load - r0 = mul_hi(r0, r4); // 12 mulhi - r5 = r5 + r1 + ((((sel >> 30u) & 1u) != 0u) ? 0xd3177981u : 0xc7934706u); // 13 add - r0 = r0 ^ ds[r4 & mask]; // 14 load - r2 = r2 - r4; // 15 sub - r2 = r2 ^ ds[r0 & mask]; // 16 load - r7 = r7 ^ ds[r2 & mask]; // 17 load - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 4u); r7 = r7 ^ t_; } // 18 shfl - r5 = r5 * r0; // 19 mul - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 2u); r3 = r3 ^ t_; } // 20 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 16u); r2 = r2 ^ t_; } // 21 shfl - r6 = mul_hi(r6, r2); // 22 mulhi - r6 = r6 ^ ds[r1 & mask]; // 23 load - r5 = r5 * r0; // 24 mul - r5 = rotl_imm(r5, 19u); // 25 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 2u); r7 = r7 ^ t_; } // 26 shfl - r0 = r0 ^ r5; // 27 xor - r0 = r0 ^ r4; // 28 xor - r3 = r3 - r0; // 29 sub - r5 = r5 * r1; // 30 mul - r7 = r7 ^ ds[r2 & mask]; // 31 load - r1 = r1 ^ ds[r0 & mask]; // 32 load - r5 = r5 ^ r6; // 33 xor - r5 = r5 ^ ds[r1 & mask]; // 34 load - r0 = mul_hi(r0, r5); // 35 mulhi - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 4u); r5 = r5 ^ t_; } // 36 shfl - r7 = r7 ^ ds[r0 & mask]; // 37 load - r3 = r3 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0x230c005cu : 0x75ba2fadu); // 38 add - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 4u); r1 = r1 ^ t_; } // 39 shfl - r2 = r2 ^ r5; // 40 xor - r3 = r6 * r3 + r3; // 41 mad - r6 = r6 - r7; // 42 sub - r7 = r7 ^ r0; // 43 xor - r1 = r1 ^ ds[r7 & mask]; // 44 load - r2 = r2 * r3; // 45 mul - r1 = mul_hi(r1, r5); // 46 mulhi - r4 = r4 - r3; // 47 sub - r2 = rotr_var(r2, r6); // 48 rotr - r3 = r3 ^ ds[r5 & mask]; // 49 load - r1 = r1 + r5 + ((((sel >> 7u) & 1u) != 0u) ? 0x1907970cu : 0x81b8bc2cu); // 50 add - r0 = r0 * r2; // 51 mul - r0 = r0 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x699fd448u : 0x4f92b968u); // 52 add - r1 = r1 + r0 + ((((sel >> 12u) & 1u) != 0u) ? 0x77b1520du : 0x2bb965afu); // 53 add - r7 = rotl_imm(r7, 14u); // 54 rotl - r3 = r3 + r7 + ((((sel >> 1u) & 1u) != 0u) ? 0xa54c55a0u : 0x7b0fe07au); // 55 add - r6 = r6 ^ ds[r7 & mask]; // 56 load - r1 = rotr_var(r1, r5); // 57 rotr - r5 = r5 ^ ds[r4 & mask]; // 58 load - r6 = r6 ^ ds[r2 & mask]; // 59 load - r3 = r5 * r0 + r3; // 60 mad - r5 = r5 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0xad7493e7u : 0xaf9dd72du); // 61 add - r4 = r4 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x1e07c3d9u : 0x89841d87u); // 62 add - r5 = rotl_imm(r5, 19u); // 63 rotl - // int8 tile block (Counter ASIC 4.0 research, experimental): 1430 mma.m8n8k16 u8 tiles per iteration, both outputs consumed - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r2 += d0_; r3 += d1_; } // t0 mm8 a=r6 b=r5 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r4 += d0_; r5 += d1_; } // t1 mm8 a=r1 b=r1 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r0 += d0_; r7 += d1_; } // t2 mm8 a=r2 b=r5 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r3 += d1_; } // t3 mm8 a=r1 b=r5 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r2 += d0_; r6 += d1_; } // t4 mm8 a=r2 b=r0 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r7 += d0_; r3 += d1_; } // t5 mm8 a=r2 b=r6 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t6 mm8 a=r1 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r0 += d0_; r4 += d1_; } // t7 mm8 a=r4 b=r6 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r1 += d0_; r5 += d1_; } // t8 mm8 a=r1 b=r1 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r4 += d0_; r6 += d1_; } // t9 mm8 a=r3 b=r7 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r0 += d0_; r4 += d1_; } // t10 mm8 a=r3 b=r0 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r0 += d0_; r6 += d1_; } // t11 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r6 += d0_; r0 += d1_; } // t12 mm8 a=r3 b=r1 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r5 += d1_; } // t13 mm8 a=r1 b=r5 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r3 += d0_; r4 += d1_; } // t14 mm8 a=r3 b=r3 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r5 += d0_; r4 += d1_; } // t15 mm8 a=r1 b=r0 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r0 += d0_; r4 += d1_; } // t16 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r7 += d1_; } // t17 mm8 a=r0 b=r5 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r2 += d0_; r0 += d1_; } // t18 mm8 a=r1 b=r1 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r2 += d0_; r4 += d1_; } // t19 mm8 a=r3 b=r3 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r6 += d0_; r1 += d1_; } // t20 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r6 += d0_; r0 += d1_; } // t21 mm8 a=r0 b=r3 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r4 += d1_; } // t22 mm8 a=r1 b=r7 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r0 += d0_; r3 += d1_; } // t23 mm8 a=r1 b=r2 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r0 += d0_; r2 += d1_; } // t24 mm8 a=r3 b=r4 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r3 += d0_; r7 += d1_; } // t25 mm8 a=r5 b=r0 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t26 mm8 a=r1 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r0 += d0_; r5 += d1_; } // t27 mm8 a=r3 b=r2 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r5 += d1_; } // t28 mm8 a=r2 b=r4 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r1 += d0_; r5 += d1_; } // t29 mm8 a=r3 b=r4 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r6 += d0_; r3 += d1_; } // t30 mm8 a=r6 b=r5 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t31 mm8 a=r1 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r3 += d0_; r0 += d1_; } // t32 mm8 a=r7 b=r6 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r4 += d0_; r6 += d1_; } // t33 mm8 a=r7 b=r1 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r5 += d0_; r1 += d1_; } // t34 mm8 a=r1 b=r0 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t35 mm8 a=r2 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r7 += d0_; r1 += d1_; } // t36 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r0 += d0_; r2 += d1_; } // t37 mm8 a=r1 b=r7 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r2 += d0_; r4 += d1_; } // t38 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r2 += d0_; r5 += d1_; } // t39 mm8 a=r6 b=r2 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r4 += d0_; r1 += d1_; } // t40 mm8 a=r6 b=r1 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r6 += d0_; r2 += d1_; } // t41 mm8 a=r6 b=r0 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r5 += d0_; r1 += d1_; } // t42 mm8 a=r7 b=r4 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r6 += d0_; r0 += d1_; } // t43 mm8 a=r6 b=r1 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r4 += d0_; r1 += d1_; } // t44 mm8 a=r0 b=r6 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r6 += d0_; r5 += d1_; } // t45 mm8 a=r6 b=r5 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r1 += d0_; r0 += d1_; } // t46 mm8 a=r6 b=r6 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r3 += d0_; r1 += d1_; } // t47 mm8 a=r7 b=r3 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t48 mm8 a=r7 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r1 += d0_; r7 += d1_; } // t49 mm8 a=r4 b=r5 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r4 += d0_; r5 += d1_; } // t50 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r0 += d0_; r3 += d1_; } // t51 mm8 a=r3 b=r4 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r7 += d0_; r4 += d1_; } // t52 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r1 += d0_; r3 += d1_; } // t53 mm8 a=r7 b=r7 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t54 mm8 a=r7 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r3 += d0_; r7 += d1_; } // t55 mm8 a=r1 b=r7 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t56 mm8 a=r4 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r6 += d0_; r7 += d1_; } // t57 mm8 a=r4 b=r3 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r6 += d0_; r0 += d1_; } // t58 mm8 a=r2 b=r3 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r7 += d1_; } // t59 mm8 a=r6 b=r7 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t60 mm8 a=r6 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r2 += d0_; r6 += d1_; } // t61 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r6 += d1_; } // t62 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t63 mm8 a=r1 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r5 += d0_; r3 += d1_; } // t64 mm8 a=r0 b=r5 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r2 += d0_; r4 += d1_; } // t65 mm8 a=r0 b=r1 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r1 += d0_; r3 += d1_; } // t66 mm8 a=r3 b=r5 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r3 += d0_; r4 += d1_; } // t67 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r2 += d1_; } // t68 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r4 += d1_; } // t69 mm8 a=r6 b=r7 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r1 += d0_; r5 += d1_; } // t70 mm8 a=r1 b=r7 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r1 += d0_; r0 += d1_; } // t71 mm8 a=r3 b=r0 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r3 += d0_; r6 += d1_; } // t72 mm8 a=r4 b=r0 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r0 += d0_; r2 += d1_; } // t73 mm8 a=r4 b=r1 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r3 += d0_; r4 += d1_; } // t74 mm8 a=r5 b=r1 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r7 += d0_; r3 += d1_; } // t75 mm8 a=r4 b=r4 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r1 += d0_; r7 += d1_; } // t76 mm8 a=r5 b=r6 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r3 += d0_; r2 += d1_; } // t77 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r0 += d0_; r5 += d1_; } // t78 mm8 a=r2 b=r5 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t79 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r3 += d0_; r4 += d1_; } // t80 mm8 a=r6 b=r7 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r1 += d0_; r6 += d1_; } // t81 mm8 a=r4 b=r3 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r1 += d0_; r7 += d1_; } // t82 mm8 a=r6 b=r0 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r1 += d0_; r6 += d1_; } // t83 mm8 a=r2 b=r2 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t84 mm8 a=r0 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t85 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r2 += d0_; r3 += d1_; } // t86 mm8 a=r3 b=r4 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r3 += d0_; r7 += d1_; } // t87 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r5 += d0_; r3 += d1_; } // t88 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r2 += d0_; r1 += d1_; } // t89 mm8 a=r4 b=r2 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r3 += d0_; r6 += d1_; } // t90 mm8 a=r6 b=r5 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r5 += d0_; r4 += d1_; } // t91 mm8 a=r6 b=r5 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r2 += d0_; r3 += d1_; } // t92 mm8 a=r2 b=r6 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r5 += d0_; r7 += d1_; } // t93 mm8 a=r2 b=r2 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r7 += d0_; r3 += d1_; } // t94 mm8 a=r2 b=r4 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r1 += d0_; r6 += d1_; } // t95 mm8 a=r5 b=r4 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r3 += d0_; r1 += d1_; } // t96 mm8 a=r0 b=r6 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r2 += d1_; } // t97 mm8 a=r0 b=r5 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t98 mm8 a=r2 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r0 += d0_; r2 += d1_; } // t99 mm8 a=r1 b=r4 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r2 += d0_; r5 += d1_; } // t100 mm8 a=r2 b=r4 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r7 += d1_; } // t101 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t102 mm8 a=r5 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t103 mm8 a=r4 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r3 += d0_; r7 += d1_; } // t104 mm8 a=r2 b=r0 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r0 += d0_; r5 += d1_; } // t105 mm8 a=r6 b=r1 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r0 += d0_; r5 += d1_; } // t106 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r4 += d0_; r2 += d1_; } // t107 mm8 a=r2 b=r1 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t108 mm8 a=r4 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r5 += d0_; r3 += d1_; } // t109 mm8 a=r2 b=r7 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r0 += d0_; r2 += d1_; } // t110 mm8 a=r1 b=r0 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r4 += d0_; r6 += d1_; } // t111 mm8 a=r6 b=r5 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r4 += d0_; r2 += d1_; } // t112 mm8 a=r5 b=r7 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t113 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t114 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r1 += d0_; r4 += d1_; } // t115 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r5 += d0_; r0 += d1_; } // t116 mm8 a=r7 b=r2 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r7 += d0_; r6 += d1_; } // t117 mm8 a=r7 b=r6 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r2 += d0_; r1 += d1_; } // t118 mm8 a=r0 b=r1 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r5 += d0_; r3 += d1_; } // t119 mm8 a=r4 b=r3 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r7 += d0_; r2 += d1_; } // t120 mm8 a=r5 b=r3 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r1 += d1_; } // t121 mm8 a=r2 b=r7 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r3 += d0_; r0 += d1_; } // t122 mm8 a=r0 b=r1 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r3 += d0_; r5 += d1_; } // t123 mm8 a=r5 b=r2 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t124 mm8 a=r5 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r7 += d0_; r2 += d1_; } // t125 mm8 a=r7 b=r2 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r3 += d0_; r2 += d1_; } // t126 mm8 a=r6 b=r3 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r1 += d1_; } // t127 mm8 a=r6 b=r7 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r4 += d0_; r1 += d1_; } // t128 mm8 a=r6 b=r7 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r6 += d0_; r3 += d1_; } // t129 mm8 a=r3 b=r2 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r2 += d0_; r1 += d1_; } // t130 mm8 a=r0 b=r4 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r3 += d0_; r1 += d1_; } // t131 mm8 a=r3 b=r5 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r0 += d0_; r6 += d1_; } // t132 mm8 a=r0 b=r7 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r0 += d0_; r1 += d1_; } // t133 mm8 a=r2 b=r6 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r3 += d0_; r1 += d1_; } // t134 mm8 a=r5 b=r6 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r4 += d0_; r5 += d1_; } // t135 mm8 a=r0 b=r2 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r1 += d0_; r5 += d1_; } // t136 mm8 a=r3 b=r1 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r4 += d0_; r6 += d1_; } // t137 mm8 a=r7 b=r0 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r4 += d0_; r2 += d1_; } // t138 mm8 a=r2 b=r2 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r2 += d0_; r7 += d1_; } // t139 mm8 a=r0 b=r4 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r6 += d1_; } // t140 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r1 += d0_; r2 += d1_; } // t141 mm8 a=r5 b=r5 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r6 += d0_; r2 += d1_; } // t142 mm8 a=r2 b=r2 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r7 += d0_; r1 += d1_; } // t143 mm8 a=r4 b=r3 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r1 += d0_; r7 += d1_; } // t144 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r6 += d0_; r5 += d1_; } // t145 mm8 a=r7 b=r2 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r7 += d1_; } // t146 mm8 a=r2 b=r7 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r3 += d0_; r0 += d1_; } // t147 mm8 a=r3 b=r1 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r0 += d0_; r6 += d1_; } // t148 mm8 a=r2 b=r1 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r7 += d0_; r1 += d1_; } // t149 mm8 a=r4 b=r6 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r7 += d0_; r0 += d1_; } // t150 mm8 a=r3 b=r6 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r2 += d0_; r4 += d1_; } // t151 mm8 a=r1 b=r4 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r5 += d0_; r1 += d1_; } // t152 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r2 += d0_; r3 += d1_; } // t153 mm8 a=r0 b=r3 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r0 += d0_; r4 += d1_; } // t154 mm8 a=r7 b=r3 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r7 += d0_; r5 += d1_; } // t155 mm8 a=r1 b=r3 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r2 += d0_; r3 += d1_; } // t156 mm8 a=r6 b=r3 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r4 += d0_; r5 += d1_; } // t157 mm8 a=r6 b=r4 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t158 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r5 += d0_; r2 += d1_; } // t159 mm8 a=r1 b=r0 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r5 += d0_; r7 += d1_; } // t160 mm8 a=r0 b=r1 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r7 += d0_; r0 += d1_; } // t161 mm8 a=r4 b=r0 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r6 += d0_; r1 += d1_; } // t162 mm8 a=r6 b=r6 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r7 += d0_; r3 += d1_; } // t163 mm8 a=r5 b=r3 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t164 mm8 a=r0 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r1 += d0_; r5 += d1_; } // t165 mm8 a=r6 b=r5 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r4 += d0_; r0 += d1_; } // t166 mm8 a=r6 b=r5 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r3 += d0_; r7 += d1_; } // t167 mm8 a=r4 b=r5 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r0 += d0_; r7 += d1_; } // t168 mm8 a=r3 b=r6 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r4 += d0_; r3 += d1_; } // t169 mm8 a=r6 b=r2 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t170 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r2 += d0_; r3 += d1_; } // t171 mm8 a=r6 b=r3 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r5 += d0_; r2 += d1_; } // t172 mm8 a=r7 b=r3 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t173 mm8 a=r0 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r5 += d0_; r6 += d1_; } // t174 mm8 a=r6 b=r6 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r3 += d0_; r4 += d1_; } // t175 mm8 a=r1 b=r6 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r7 += d0_; r5 += d1_; } // t176 mm8 a=r2 b=r4 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r7 += d0_; r2 += d1_; } // t177 mm8 a=r1 b=r7 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r7 += d0_; r3 += d1_; } // t178 mm8 a=r4 b=r2 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r5 += d0_; r1 += d1_; } // t179 mm8 a=r5 b=r2 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r2 += d0_; r1 += d1_; } // t180 mm8 a=r0 b=r4 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r2 += d0_; r4 += d1_; } // t181 mm8 a=r3 b=r5 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r4 += d0_; r3 += d1_; } // t182 mm8 a=r1 b=r1 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r0 += d0_; r7 += d1_; } // t183 mm8 a=r3 b=r1 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r0 += d0_; r7 += d1_; } // t184 mm8 a=r7 b=r4 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r6 += d0_; r3 += d1_; } // t185 mm8 a=r5 b=r0 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r7 += d0_; r6 += d1_; } // t186 mm8 a=r2 b=r5 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r0 += d0_; r7 += d1_; } // t187 mm8 a=r3 b=r0 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r5 += d0_; r0 += d1_; } // t188 mm8 a=r1 b=r4 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r5 += d0_; r4 += d1_; } // t189 mm8 a=r7 b=r2 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r0 += d1_; } // t190 mm8 a=r1 b=r5 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t191 mm8 a=r2 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t192 mm8 a=r5 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r4 += d0_; r6 += d1_; } // t193 mm8 a=r2 b=r1 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r0 += d0_; r5 += d1_; } // t194 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r4 += d1_; } // t195 mm8 a=r2 b=r7 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r2 += d0_; r5 += d1_; } // t196 mm8 a=r5 b=r6 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r6 += d0_; r5 += d1_; } // t197 mm8 a=r5 b=r3 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r7 += d0_; r1 += d1_; } // t198 mm8 a=r1 b=r0 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t199 mm8 a=r2 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t200 mm8 a=r5 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r2 += d0_; r1 += d1_; } // t201 mm8 a=r7 b=r2 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r5 += d0_; r6 += d1_; } // t202 mm8 a=r2 b=r5 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r6 += d1_; } // t203 mm8 a=r0 b=r5 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r3 += d0_; r7 += d1_; } // t204 mm8 a=r4 b=r7 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r1 += d0_; r3 += d1_; } // t205 mm8 a=r3 b=r7 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r7 += d0_; r3 += d1_; } // t206 mm8 a=r0 b=r6 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r1 += d0_; r4 += d1_; } // t207 mm8 a=r7 b=r7 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r6 += d0_; r5 += d1_; } // t208 mm8 a=r0 b=r1 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r6 += d0_; r2 += d1_; } // t209 mm8 a=r4 b=r4 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t210 mm8 a=r7 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r5 += d0_; r6 += d1_; } // t211 mm8 a=r1 b=r5 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r5 += d0_; r3 += d1_; } // t212 mm8 a=r2 b=r4 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r3 += d0_; r5 += d1_; } // t213 mm8 a=r2 b=r2 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r5 += d0_; r7 += d1_; } // t214 mm8 a=r4 b=r6 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r2 += d0_; r6 += d1_; } // t215 mm8 a=r7 b=r3 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r3 += d0_; r4 += d1_; } // t216 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r0 += d0_; r6 += d1_; } // t217 mm8 a=r5 b=r7 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r0 += d1_; } // t218 mm8 a=r7 b=r1 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r3 += d0_; r1 += d1_; } // t219 mm8 a=r2 b=r1 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r4 += d0_; r5 += d1_; } // t220 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r2 += d0_; r1 += d1_; } // t221 mm8 a=r7 b=r1 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r2 += d0_; r1 += d1_; } // t222 mm8 a=r0 b=r6 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r7 += d0_; r0 += d1_; } // t223 mm8 a=r2 b=r2 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r6 += d0_; r2 += d1_; } // t224 mm8 a=r1 b=r1 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r4 += d1_; } // t225 mm8 a=r2 b=r7 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t226 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r5 += d0_; r6 += d1_; } // t227 mm8 a=r3 b=r5 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r3 += d0_; r1 += d1_; } // t228 mm8 a=r7 b=r1 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r1 += d0_; r4 += d1_; } // t229 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r5 += d0_; r6 += d1_; } // t230 mm8 a=r0 b=r3 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r4 += d0_; r0 += d1_; } // t231 mm8 a=r1 b=r3 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r0 += d0_; r1 += d1_; } // t232 mm8 a=r5 b=r4 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r0 += d0_; r4 += d1_; } // t233 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r2 += d0_; r1 += d1_; } // t234 mm8 a=r2 b=r0 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t235 mm8 a=r3 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r0 += d0_; r5 += d1_; } // t236 mm8 a=r1 b=r7 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r6 += d0_; r3 += d1_; } // t237 mm8 a=r3 b=r5 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r0 += d0_; r2 += d1_; } // t238 mm8 a=r2 b=r0 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r1 += d0_; r0 += d1_; } // t239 mm8 a=r5 b=r7 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r4 += d0_; r6 += d1_; } // t240 mm8 a=r2 b=r7 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r3 += d0_; r7 += d1_; } // t241 mm8 a=r0 b=r1 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r2 += d0_; r7 += d1_; } // t242 mm8 a=r0 b=r2 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r1 += d0_; r5 += d1_; } // t243 mm8 a=r4 b=r4 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r0 += d0_; r7 += d1_; } // t244 mm8 a=r7 b=r6 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r2 += d0_; r0 += d1_; } // t245 mm8 a=r4 b=r4 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t246 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r2 += d0_; r4 += d1_; } // t247 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r7 += d0_; r3 += d1_; } // t248 mm8 a=r7 b=r5 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r0 += d0_; r5 += d1_; } // t249 mm8 a=r4 b=r4 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r0 += d0_; r1 += d1_; } // t250 mm8 a=r0 b=r6 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t251 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r5 += d0_; r4 += d1_; } // t252 mm8 a=r0 b=r2 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r7 += d0_; r4 += d1_; } // t253 mm8 a=r5 b=r5 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t254 mm8 a=r6 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r0 += d0_; r3 += d1_; } // t255 mm8 a=r5 b=r0 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r3 += d0_; r4 += d1_; } // t256 mm8 a=r2 b=r1 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r6 += d0_; r0 += d1_; } // t257 mm8 a=r4 b=r3 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r7 += d0_; r3 += d1_; } // t258 mm8 a=r4 b=r1 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r3 += d0_; r0 += d1_; } // t259 mm8 a=r5 b=r5 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r3 += d0_; r0 += d1_; } // t260 mm8 a=r3 b=r2 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r3 += d0_; r2 += d1_; } // t261 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r2 += d0_; r5 += d1_; } // t262 mm8 a=r6 b=r4 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r2 += d0_; r1 += d1_; } // t263 mm8 a=r6 b=r4 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r1 += d0_; r4 += d1_; } // t264 mm8 a=r7 b=r2 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r3 += d0_; r1 += d1_; } // t265 mm8 a=r6 b=r1 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r7 += d0_; r0 += d1_; } // t266 mm8 a=r0 b=r0 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r3 += d0_; r4 += d1_; } // t267 mm8 a=r6 b=r3 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r6 += d0_; r5 += d1_; } // t268 mm8 a=r1 b=r0 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r7 += d0_; r3 += d1_; } // t269 mm8 a=r7 b=r7 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r7 += d0_; r0 += d1_; } // t270 mm8 a=r6 b=r2 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r5 += d0_; r3 += d1_; } // t271 mm8 a=r0 b=r3 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r0 += d0_; r3 += d1_; } // t272 mm8 a=r6 b=r0 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r7 += d1_; } // t273 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r7 += d0_; r6 += d1_; } // t274 mm8 a=r1 b=r0 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r1 += d0_; r4 += d1_; } // t275 mm8 a=r1 b=r3 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r6 += d0_; r2 += d1_; } // t276 mm8 a=r1 b=r5 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r4 += d0_; r7 += d1_; } // t277 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r6 += d0_; r7 += d1_; } // t278 mm8 a=r0 b=r0 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r2 += d0_; r4 += d1_; } // t279 mm8 a=r2 b=r4 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r3 += d0_; r5 += d1_; } // t280 mm8 a=r7 b=r2 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r4 += d0_; r2 += d1_; } // t281 mm8 a=r1 b=r0 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r5 += d0_; r1 += d1_; } // t282 mm8 a=r0 b=r1 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r7 += d0_; r2 += d1_; } // t283 mm8 a=r0 b=r2 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r0 += d0_; r2 += d1_; } // t284 mm8 a=r1 b=r5 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r5 += d0_; r0 += d1_; } // t285 mm8 a=r4 b=r7 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r2 += d0_; r0 += d1_; } // t286 mm8 a=r1 b=r0 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r7 += d0_; r4 += d1_; } // t287 mm8 a=r7 b=r1 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t288 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r5 += d0_; r4 += d1_; } // t289 mm8 a=r0 b=r5 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r6 += d0_; r7 += d1_; } // t290 mm8 a=r3 b=r2 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r7 += d0_; r1 += d1_; } // t291 mm8 a=r7 b=r4 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t292 mm8 a=r6 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r6 += d0_; r4 += d1_; } // t293 mm8 a=r5 b=r5 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r1 += d0_; r6 += d1_; } // t294 mm8 a=r5 b=r7 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r1 += d0_; r3 += d1_; } // t295 mm8 a=r0 b=r7 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r5 += d0_; r6 += d1_; } // t296 mm8 a=r6 b=r1 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r3 += d0_; r5 += d1_; } // t297 mm8 a=r4 b=r5 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r3 += d0_; r1 += d1_; } // t298 mm8 a=r6 b=r0 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r2 += d0_; r3 += d1_; } // t299 mm8 a=r6 b=r0 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r4 += d0_; r2 += d1_; } // t300 mm8 a=r6 b=r4 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r6 += d0_; r3 += d1_; } // t301 mm8 a=r6 b=r4 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r6 += d0_; r4 += d1_; } // t302 mm8 a=r7 b=r6 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r3 += d0_; r6 += d1_; } // t303 mm8 a=r3 b=r2 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r2 += d0_; r4 += d1_; } // t304 mm8 a=r6 b=r5 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r0 += d0_; r2 += d1_; } // t305 mm8 a=r4 b=r5 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r0 += d0_; r2 += d1_; } // t306 mm8 a=r3 b=r1 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r5 += d0_; r0 += d1_; } // t307 mm8 a=r4 b=r3 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r2 += d0_; r6 += d1_; } // t308 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r1 += d0_; r4 += d1_; } // t309 mm8 a=r2 b=r1 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r2 += d0_; r3 += d1_; } // t310 mm8 a=r6 b=r1 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r3 += d0_; r2 += d1_; } // t311 mm8 a=r4 b=r6 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t312 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r7 += d0_; r1 += d1_; } // t313 mm8 a=r2 b=r6 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r2 += d0_; r1 += d1_; } // t314 mm8 a=r0 b=r3 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r6 += d0_; r2 += d1_; } // t315 mm8 a=r7 b=r1 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r1 += d0_; r6 += d1_; } // t316 mm8 a=r2 b=r4 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r3 += d0_; r6 += d1_; } // t317 mm8 a=r1 b=r6 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r1 += d0_; r0 += d1_; } // t318 mm8 a=r2 b=r1 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r6 += d0_; r3 += d1_; } // t319 mm8 a=r0 b=r2 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r6 += d0_; r2 += d1_; } // t320 mm8 a=r5 b=r2 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t321 mm8 a=r7 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r7 += d0_; r1 += d1_; } // t322 mm8 a=r2 b=r4 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r2 += d0_; r3 += d1_; } // t323 mm8 a=r4 b=r3 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r2 += d0_; r0 += d1_; } // t324 mm8 a=r5 b=r7 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r7 += d0_; r2 += d1_; } // t325 mm8 a=r6 b=r2 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r7 += d0_; r6 += d1_; } // t326 mm8 a=r6 b=r1 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r3 += d0_; r0 += d1_; } // t327 mm8 a=r1 b=r5 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r1 += d0_; r6 += d1_; } // t328 mm8 a=r0 b=r1 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r7 += d0_; r1 += d1_; } // t329 mm8 a=r0 b=r3 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r4 += d0_; r0 += d1_; } // t330 mm8 a=r6 b=r7 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t331 mm8 a=r4 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r6 += d0_; r3 += d1_; } // t332 mm8 a=r4 b=r4 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r4 += d0_; r2 += d1_; } // t333 mm8 a=r4 b=r6 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r0 += d0_; r1 += d1_; } // t334 mm8 a=r2 b=r2 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r2 += d1_; } // t335 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r2 += d0_; r3 += d1_; } // t336 mm8 a=r4 b=r0 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r0 += d0_; r3 += d1_; } // t337 mm8 a=r1 b=r4 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r2 += d0_; r7 += d1_; } // t338 mm8 a=r3 b=r7 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r7 += d0_; r6 += d1_; } // t339 mm8 a=r5 b=r1 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r7 += d0_; r5 += d1_; } // t340 mm8 a=r5 b=r0 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r7 += d0_; r5 += d1_; } // t341 mm8 a=r6 b=r6 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r0 += d0_; r4 += d1_; } // t342 mm8 a=r4 b=r7 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r4 += d0_; r6 += d1_; } // t343 mm8 a=r0 b=r4 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r4 += d0_; r6 += d1_; } // t344 mm8 a=r2 b=r5 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t345 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t346 mm8 a=r6 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r3 += d0_; r5 += d1_; } // t347 mm8 a=r5 b=r3 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r4 += d0_; r5 += d1_; } // t348 mm8 a=r0 b=r4 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r4 += d0_; r7 += d1_; } // t349 mm8 a=r6 b=r1 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r3 += d0_; r5 += d1_; } // t350 mm8 a=r1 b=r5 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r4 += d0_; r5 += d1_; } // t351 mm8 a=r6 b=r4 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r6 += d0_; r0 += d1_; } // t352 mm8 a=r4 b=r1 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r1 += d0_; r4 += d1_; } // t353 mm8 a=r7 b=r3 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r1 += d0_; r5 += d1_; } // t354 mm8 a=r4 b=r1 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r0 += d0_; r1 += d1_; } // t355 mm8 a=r4 b=r5 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r0 += d0_; r4 += d1_; } // t356 mm8 a=r7 b=r6 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r7 += d0_; r5 += d1_; } // t357 mm8 a=r7 b=r3 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r5 += d0_; r7 += d1_; } // t358 mm8 a=r4 b=r7 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r3 += d0_; r5 += d1_; } // t359 mm8 a=r2 b=r7 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r0 += d0_; r2 += d1_; } // t360 mm8 a=r4 b=r4 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r5 += d0_; r6 += d1_; } // t361 mm8 a=r2 b=r2 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r3 += d0_; r0 += d1_; } // t362 mm8 a=r2 b=r7 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r5 += d0_; r0 += d1_; } // t363 mm8 a=r7 b=r6 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r0 += d0_; r5 += d1_; } // t364 mm8 a=r6 b=r4 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r5 += d0_; r4 += d1_; } // t365 mm8 a=r6 b=r6 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r2 += d0_; r5 += d1_; } // t366 mm8 a=r7 b=r7 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t367 mm8 a=r4 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r3 += d0_; r5 += d1_; } // t368 mm8 a=r7 b=r0 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r6 += d0_; r0 += d1_; } // t369 mm8 a=r1 b=r0 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r7 += d0_; r4 += d1_; } // t370 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r2 += d0_; r6 += d1_; } // t371 mm8 a=r5 b=r4 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r2 += d1_; } // t372 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r2 += d0_; r4 += d1_; } // t373 mm8 a=r4 b=r6 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t374 mm8 a=r1 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r6 += d0_; r2 += d1_; } // t375 mm8 a=r6 b=r5 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t376 mm8 a=r3 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r6 += d0_; r4 += d1_; } // t377 mm8 a=r6 b=r4 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r2 += d0_; r4 += d1_; } // t378 mm8 a=r6 b=r3 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t379 mm8 a=r3 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r6 += d0_; r2 += d1_; } // t380 mm8 a=r2 b=r0 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r1 += d0_; r2 += d1_; } // t381 mm8 a=r3 b=r5 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r2 += d0_; r4 += d1_; } // t382 mm8 a=r5 b=r2 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r0 += d0_; r2 += d1_; } // t383 mm8 a=r1 b=r2 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r1 += d0_; r0 += d1_; } // t384 mm8 a=r3 b=r1 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r7 += d0_; r5 += d1_; } // t385 mm8 a=r6 b=r0 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r1 += d0_; r7 += d1_; } // t386 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r2 += d0_; r4 += d1_; } // t387 mm8 a=r3 b=r7 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r1 += d0_; r2 += d1_; } // t388 mm8 a=r3 b=r0 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r5 += d0_; r3 += d1_; } // t389 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r1 += d0_; r4 += d1_; } // t390 mm8 a=r5 b=r4 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t391 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r3 += d0_; r0 += d1_; } // t392 mm8 a=r2 b=r3 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r5 += d0_; r0 += d1_; } // t393 mm8 a=r3 b=r6 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r1 += d0_; r2 += d1_; } // t394 mm8 a=r0 b=r7 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r2 += d0_; r4 += d1_; } // t395 mm8 a=r0 b=r7 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r1 += d0_; r4 += d1_; } // t396 mm8 a=r7 b=r7 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t397 mm8 a=r5 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r7 += d0_; r0 += d1_; } // t398 mm8 a=r6 b=r5 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r3 += d0_; r5 += d1_; } // t399 mm8 a=r6 b=r3 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r0 += d0_; r4 += d1_; } // t400 mm8 a=r2 b=r7 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r2 += d0_; r1 += d1_; } // t401 mm8 a=r7 b=r6 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r7 += d0_; r1 += d1_; } // t402 mm8 a=r0 b=r0 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r1 += d0_; r0 += d1_; } // t403 mm8 a=r7 b=r4 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r1 += d0_; r4 += d1_; } // t404 mm8 a=r1 b=r1 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r3 += d0_; r6 += d1_; } // t405 mm8 a=r6 b=r6 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r4 += d0_; r3 += d1_; } // t406 mm8 a=r6 b=r1 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r2 += d0_; r4 += d1_; } // t407 mm8 a=r5 b=r7 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t408 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r4 += d0_; r7 += d1_; } // t409 mm8 a=r3 b=r4 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r5 += d0_; r2 += d1_; } // t410 mm8 a=r0 b=r5 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r5 += d0_; r1 += d1_; } // t411 mm8 a=r2 b=r0 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t412 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r2 += d1_; } // t413 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r0 += d0_; r3 += d1_; } // t414 mm8 a=r7 b=r1 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r5 += d0_; r1 += d1_; } // t415 mm8 a=r4 b=r7 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r3 += d0_; r5 += d1_; } // t416 mm8 a=r5 b=r4 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t417 mm8 a=r6 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r1 += d0_; r5 += d1_; } // t418 mm8 a=r1 b=r3 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r0 += d0_; r6 += d1_; } // t419 mm8 a=r5 b=r0 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r7 += d0_; r6 += d1_; } // t420 mm8 a=r4 b=r2 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r3 += d0_; r7 += d1_; } // t421 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t422 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r0 += d1_; } // t423 mm8 a=r1 b=r7 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r6 += d0_; r5 += d1_; } // t424 mm8 a=r5 b=r1 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t425 mm8 a=r2 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r5 += d0_; r0 += d1_; } // t426 mm8 a=r1 b=r6 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t427 mm8 a=r6 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r7 += d0_; r4 += d1_; } // t428 mm8 a=r6 b=r2 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r7 += d0_; r0 += d1_; } // t429 mm8 a=r3 b=r3 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r3 += d0_; r5 += d1_; } // t430 mm8 a=r7 b=r7 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r1 += d0_; r0 += d1_; } // t431 mm8 a=r6 b=r4 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r3 += d0_; r4 += d1_; } // t432 mm8 a=r3 b=r2 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r5 += d0_; r0 += d1_; } // t433 mm8 a=r3 b=r0 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r5 += d0_; r3 += d1_; } // t434 mm8 a=r0 b=r7 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r6 += d0_; r3 += d1_; } // t435 mm8 a=r7 b=r5 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r0 += d0_; r5 += d1_; } // t436 mm8 a=r6 b=r4 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r3 += d0_; r4 += d1_; } // t437 mm8 a=r0 b=r7 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r4 += d0_; r7 += d1_; } // t438 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t439 mm8 a=r4 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r3 += d0_; r2 += d1_; } // t440 mm8 a=r3 b=r5 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r1 += d0_; r4 += d1_; } // t441 mm8 a=r5 b=r1 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r7 += d0_; r5 += d1_; } // t442 mm8 a=r1 b=r2 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r5 += d0_; r2 += d1_; } // t443 mm8 a=r1 b=r7 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r1 += d0_; r0 += d1_; } // t444 mm8 a=r3 b=r7 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r7 += d0_; r0 += d1_; } // t445 mm8 a=r7 b=r5 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r2 += d0_; r1 += d1_; } // t446 mm8 a=r3 b=r1 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r4 += d0_; r7 += d1_; } // t447 mm8 a=r6 b=r7 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r4 += d0_; r5 += d1_; } // t448 mm8 a=r5 b=r2 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r3 += d1_; } // t449 mm8 a=r7 b=r1 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r7 += d0_; r3 += d1_; } // t450 mm8 a=r7 b=r5 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r1 += d0_; r7 += d1_; } // t451 mm8 a=r0 b=r3 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r7 += d0_; r1 += d1_; } // t452 mm8 a=r2 b=r4 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r7 += d0_; r1 += d1_; } // t453 mm8 a=r5 b=r7 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r7 += d0_; r1 += d1_; } // t454 mm8 a=r7 b=r7 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t455 mm8 a=r7 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r5 += d0_; r6 += d1_; } // t456 mm8 a=r6 b=r5 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r1 += d0_; r5 += d1_; } // t457 mm8 a=r7 b=r7 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r4 += d0_; r1 += d1_; } // t458 mm8 a=r5 b=r1 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r1 += d0_; r0 += d1_; } // t459 mm8 a=r6 b=r7 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t460 mm8 a=r7 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r6 += d0_; r2 += d1_; } // t461 mm8 a=r0 b=r6 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t462 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r3 += d0_; r2 += d1_; } // t463 mm8 a=r2 b=r0 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r0 += d0_; r1 += d1_; } // t464 mm8 a=r0 b=r3 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r0 += d0_; r5 += d1_; } // t465 mm8 a=r1 b=r7 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r4 += d0_; r1 += d1_; } // t466 mm8 a=r1 b=r2 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r1 += d0_; r6 += d1_; } // t467 mm8 a=r1 b=r0 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r6 += d0_; r2 += d1_; } // t468 mm8 a=r4 b=r1 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r0 += d0_; r1 += d1_; } // t469 mm8 a=r2 b=r0 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r2 += d0_; r0 += d1_; } // t470 mm8 a=r3 b=r1 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r7 += d0_; r1 += d1_; } // t471 mm8 a=r2 b=r2 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t472 mm8 a=r1 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r1 += d1_; } // t473 mm8 a=r2 b=r7 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r6 += d0_; r3 += d1_; } // t474 mm8 a=r5 b=r3 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r6 += d0_; r0 += d1_; } // t475 mm8 a=r7 b=r6 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r5 += d0_; r6 += d1_; } // t476 mm8 a=r3 b=r0 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r4 += d0_; r6 += d1_; } // t477 mm8 a=r3 b=r1 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r2 += d0_; r5 += d1_; } // t478 mm8 a=r0 b=r6 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r7 += d0_; r1 += d1_; } // t479 mm8 a=r7 b=r6 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r1 += d0_; r0 += d1_; } // t480 mm8 a=r6 b=r0 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r7 += d1_; } // t481 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r0 += d0_; r6 += d1_; } // t482 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r7 += d0_; r6 += d1_; } // t483 mm8 a=r7 b=r0 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r6 += d0_; r4 += d1_; } // t484 mm8 a=r3 b=r7 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r1 += d0_; r2 += d1_; } // t485 mm8 a=r6 b=r4 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r2 += d0_; r5 += d1_; } // t486 mm8 a=r6 b=r5 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r4 += d0_; r5 += d1_; } // t487 mm8 a=r2 b=r5 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r1 += d0_; r2 += d1_; } // t488 mm8 a=r1 b=r2 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r6 += d0_; r4 += d1_; } // t489 mm8 a=r5 b=r2 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r2 += d0_; r6 += d1_; } // t490 mm8 a=r7 b=r1 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r0 += d0_; r6 += d1_; } // t491 mm8 a=r0 b=r0 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r6 += d0_; r4 += d1_; } // t492 mm8 a=r3 b=r1 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r7 += d0_; r6 += d1_; } // t493 mm8 a=r6 b=r1 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r3 += d0_; r1 += d1_; } // t494 mm8 a=r5 b=r6 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r7 += d0_; r2 += d1_; } // t495 mm8 a=r2 b=r2 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r5 += d0_; r0 += d1_; } // t496 mm8 a=r3 b=r5 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r2 += d0_; r0 += d1_; } // t497 mm8 a=r3 b=r2 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r6 += d0_; r2 += d1_; } // t498 mm8 a=r3 b=r1 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r1 += d0_; r5 += d1_; } // t499 mm8 a=r1 b=r6 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r1 += d0_; r5 += d1_; } // t500 mm8 a=r7 b=r6 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r1 += d0_; r0 += d1_; } // t501 mm8 a=r4 b=r4 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r7 += d0_; r6 += d1_; } // t502 mm8 a=r5 b=r7 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r3 += d0_; r4 += d1_; } // t503 mm8 a=r7 b=r5 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r4 += d0_; r2 += d1_; } // t504 mm8 a=r2 b=r0 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r0 += d0_; r1 += d1_; } // t505 mm8 a=r4 b=r0 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r7 += d0_; r0 += d1_; } // t506 mm8 a=r0 b=r2 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r4 += d0_; r7 += d1_; } // t507 mm8 a=r5 b=r6 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r4 += d0_; r7 += d1_; } // t508 mm8 a=r7 b=r3 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r6 += d0_; r0 += d1_; } // t509 mm8 a=r4 b=r4 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t510 mm8 a=r0 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t511 mm8 a=r2 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r4 += d0_; r1 += d1_; } // t512 mm8 a=r2 b=r7 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r5 += d0_; r7 += d1_; } // t513 mm8 a=r0 b=r3 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r4 += d0_; r2 += d1_; } // t514 mm8 a=r6 b=r7 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r0 += d0_; r4 += d1_; } // t515 mm8 a=r5 b=r2 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r2 += d0_; r4 += d1_; } // t516 mm8 a=r0 b=r2 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r6 += d0_; r7 += d1_; } // t517 mm8 a=r6 b=r7 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r0 += d0_; r6 += d1_; } // t518 mm8 a=r1 b=r4 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r7 += d0_; r4 += d1_; } // t519 mm8 a=r6 b=r0 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r0 += d0_; r2 += d1_; } // t520 mm8 a=r1 b=r2 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r0 += d0_; r2 += d1_; } // t521 mm8 a=r0 b=r4 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r6 += d0_; r7 += d1_; } // t522 mm8 a=r7 b=r2 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r7 += d0_; r0 += d1_; } // t523 mm8 a=r5 b=r3 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r7 += d0_; r0 += d1_; } // t524 mm8 a=r0 b=r4 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r5 += d0_; r1 += d1_; } // t525 mm8 a=r1 b=r5 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r6 += d1_; } // t526 mm8 a=r7 b=r1 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r6 += d0_; r7 += d1_; } // t527 mm8 a=r5 b=r1 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r3 += d0_; r2 += d1_; } // t528 mm8 a=r4 b=r1 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r2 += d0_; r1 += d1_; } // t529 mm8 a=r4 b=r2 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r6 += d0_; r4 += d1_; } // t530 mm8 a=r0 b=r2 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r3 += d0_; r4 += d1_; } // t531 mm8 a=r7 b=r0 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r2 += d0_; r7 += d1_; } // t532 mm8 a=r7 b=r2 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r6 += d0_; r4 += d1_; } // t533 mm8 a=r3 b=r2 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r3 += d0_; r6 += d1_; } // t534 mm8 a=r4 b=r3 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r1 += d0_; r6 += d1_; } // t535 mm8 a=r0 b=r6 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r4 += d0_; r6 += d1_; } // t536 mm8 a=r3 b=r1 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r4 += d0_; r5 += d1_; } // t537 mm8 a=r1 b=r6 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r6 += d0_; r7 += d1_; } // t538 mm8 a=r6 b=r6 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t539 mm8 a=r4 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r2 += d0_; r4 += d1_; } // t540 mm8 a=r1 b=r4 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r3 += d0_; r6 += d1_; } // t541 mm8 a=r5 b=r3 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r0 += d0_; r7 += d1_; } // t542 mm8 a=r7 b=r7 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r3 += d0_; r7 += d1_; } // t543 mm8 a=r4 b=r2 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r3 += d0_; r6 += d1_; } // t544 mm8 a=r5 b=r7 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r2 += d0_; r1 += d1_; } // t545 mm8 a=r5 b=r2 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r5 += d0_; r7 += d1_; } // t546 mm8 a=r7 b=r1 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r2 += d0_; r1 += d1_; } // t547 mm8 a=r3 b=r5 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r5 += d0_; r4 += d1_; } // t548 mm8 a=r6 b=r3 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r3 += d0_; r7 += d1_; } // t549 mm8 a=r2 b=r2 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r5 += d0_; r2 += d1_; } // t550 mm8 a=r5 b=r5 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r4 += d0_; r0 += d1_; } // t551 mm8 a=r7 b=r6 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r1 += d0_; r2 += d1_; } // t552 mm8 a=r6 b=r7 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r6 += d0_; r7 += d1_; } // t553 mm8 a=r6 b=r3 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r7 += d0_; r0 += d1_; } // t554 mm8 a=r4 b=r2 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r0 += d0_; r1 += d1_; } // t555 mm8 a=r2 b=r7 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r0 += d0_; r7 += d1_; } // t556 mm8 a=r3 b=r6 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r7 += d0_; r0 += d1_; } // t557 mm8 a=r7 b=r7 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r5 += d0_; r2 += d1_; } // t558 mm8 a=r5 b=r0 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r5 += d0_; r1 += d1_; } // t559 mm8 a=r4 b=r1 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r5 += d0_; r4 += d1_; } // t560 mm8 a=r5 b=r4 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t561 mm8 a=r6 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r7 += d0_; r0 += d1_; } // t562 mm8 a=r1 b=r2 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r3 += d0_; r2 += d1_; } // t563 mm8 a=r5 b=r3 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r4 += d0_; r6 += d1_; } // t564 mm8 a=r4 b=r3 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r2 += d0_; r7 += d1_; } // t565 mm8 a=r6 b=r0 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r5 += d0_; r0 += d1_; } // t566 mm8 a=r1 b=r6 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r2 += d0_; r0 += d1_; } // t567 mm8 a=r3 b=r6 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r3 += d0_; r1 += d1_; } // t568 mm8 a=r6 b=r5 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r2 += d0_; r5 += d1_; } // t569 mm8 a=r3 b=r7 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r3 += d0_; r2 += d1_; } // t570 mm8 a=r5 b=r1 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r2 += d0_; r6 += d1_; } // t571 mm8 a=r6 b=r0 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r3 += d0_; r1 += d1_; } // t572 mm8 a=r5 b=r0 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r4 += d0_; r3 += d1_; } // t573 mm8 a=r3 b=r0 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r0 += d0_; r5 += d1_; } // t574 mm8 a=r2 b=r2 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r6 += d0_; r0 += d1_; } // t575 mm8 a=r3 b=r1 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r0 += d0_; r4 += d1_; } // t576 mm8 a=r2 b=r3 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r6 += d0_; r1 += d1_; } // t577 mm8 a=r6 b=r3 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r2 += d0_; r4 += d1_; } // t578 mm8 a=r5 b=r6 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r7 += d0_; r4 += d1_; } // t579 mm8 a=r4 b=r7 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r5 += d0_; r7 += d1_; } // t580 mm8 a=r7 b=r1 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r3 += d0_; r4 += d1_; } // t581 mm8 a=r1 b=r6 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r1 += d0_; r3 += d1_; } // t582 mm8 a=r2 b=r1 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t583 mm8 a=r5 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r0 += d0_; r4 += d1_; } // t584 mm8 a=r1 b=r7 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r5 += d0_; r2 += d1_; } // t585 mm8 a=r0 b=r4 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r1 += d0_; r2 += d1_; } // t586 mm8 a=r4 b=r4 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r6 += d0_; r3 += d1_; } // t587 mm8 a=r0 b=r4 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r7 += d0_; r0 += d1_; } // t588 mm8 a=r5 b=r5 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r3 += d0_; r5 += d1_; } // t589 mm8 a=r3 b=r6 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r5 += d0_; r0 += d1_; } // t590 mm8 a=r3 b=r2 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r0 += d0_; r5 += d1_; } // t591 mm8 a=r5 b=r1 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r7 += d0_; r4 += d1_; } // t592 mm8 a=r5 b=r3 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r6 += d0_; r4 += d1_; } // t593 mm8 a=r5 b=r3 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r4 += d0_; r0 += d1_; } // t594 mm8 a=r6 b=r7 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r6 += d0_; r4 += d1_; } // t595 mm8 a=r3 b=r0 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r3 += d0_; r2 += d1_; } // t596 mm8 a=r7 b=r6 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r5 += d0_; r1 += d1_; } // t597 mm8 a=r2 b=r4 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r4 += d0_; r0 += d1_; } // t598 mm8 a=r7 b=r7 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r2 += d0_; r6 += d1_; } // t599 mm8 a=r7 b=r2 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r5 += d0_; r4 += d1_; } // t600 mm8 a=r7 b=r7 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r7 += d0_; r5 += d1_; } // t601 mm8 a=r2 b=r7 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r7 += d0_; r1 += d1_; } // t602 mm8 a=r0 b=r1 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r7 += d0_; r4 += d1_; } // t603 mm8 a=r0 b=r0 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r2 += d0_; r1 += d1_; } // t604 mm8 a=r5 b=r5 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r4 += d0_; r2 += d1_; } // t605 mm8 a=r2 b=r0 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r1 += d0_; r7 += d1_; } // t606 mm8 a=r7 b=r5 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r4 += d0_; r5 += d1_; } // t607 mm8 a=r0 b=r2 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r6 += d0_; r4 += d1_; } // t608 mm8 a=r7 b=r6 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r7 += d0_; r1 += d1_; } // t609 mm8 a=r1 b=r5 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r1 += d0_; r6 += d1_; } // t610 mm8 a=r3 b=r3 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r5 += d0_; r7 += d1_; } // t611 mm8 a=r2 b=r2 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r1 += d0_; r2 += d1_; } // t612 mm8 a=r1 b=r1 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r3 += d0_; r4 += d1_; } // t613 mm8 a=r6 b=r3 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t614 mm8 a=r3 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r7 += d0_; r6 += d1_; } // t615 mm8 a=r6 b=r2 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r1 += d0_; r7 += d1_; } // t616 mm8 a=r2 b=r3 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r3 += d0_; r2 += d1_; } // t617 mm8 a=r5 b=r4 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r5 += d0_; r0 += d1_; } // t618 mm8 a=r3 b=r6 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r0 += d0_; r7 += d1_; } // t619 mm8 a=r5 b=r0 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r6 += d0_; r4 += d1_; } // t620 mm8 a=r3 b=r5 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r7 += d0_; r2 += d1_; } // t621 mm8 a=r4 b=r2 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r4 += d0_; r3 += d1_; } // t622 mm8 a=r7 b=r0 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r2 += d0_; r1 += d1_; } // t623 mm8 a=r4 b=r6 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r1 += d0_; r0 += d1_; } // t624 mm8 a=r6 b=r2 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r7 += d0_; r5 += d1_; } // t625 mm8 a=r6 b=r2 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r5 += d0_; r6 += d1_; } // t626 mm8 a=r5 b=r2 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r2 += d0_; r3 += d1_; } // t627 mm8 a=r4 b=r0 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r0 += d1_; } // t628 mm8 a=r1 b=r5 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r7 += d0_; r3 += d1_; } // t629 mm8 a=r4 b=r5 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r6 += d0_; r4 += d1_; } // t630 mm8 a=r4 b=r5 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r7 += d0_; r4 += d1_; } // t631 mm8 a=r6 b=r4 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r7 += d0_; r1 += d1_; } // t632 mm8 a=r2 b=r0 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t633 mm8 a=r2 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t634 mm8 a=r1 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r2 += d0_; r4 += d1_; } // t635 mm8 a=r6 b=r7 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r4 += d0_; r1 += d1_; } // t636 mm8 a=r2 b=r5 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t637 mm8 a=r5 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r5 += d0_; r1 += d1_; } // t638 mm8 a=r1 b=r4 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r5 += d0_; r1 += d1_; } // t639 mm8 a=r1 b=r7 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r4 += d0_; r0 += d1_; } // t640 mm8 a=r0 b=r4 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r5 += d0_; r4 += d1_; } // t641 mm8 a=r5 b=r6 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r1 += d0_; r6 += d1_; } // t642 mm8 a=r7 b=r5 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r5 += d0_; r6 += d1_; } // t643 mm8 a=r3 b=r5 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r2 += d0_; r5 += d1_; } // t644 mm8 a=r1 b=r6 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r4 += d0_; r1 += d1_; } // t645 mm8 a=r0 b=r7 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r4 += d0_; r3 += d1_; } // t646 mm8 a=r5 b=r6 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r6 += d0_; r4 += d1_; } // t647 mm8 a=r6 b=r4 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r6 += d0_; r2 += d1_; } // t648 mm8 a=r3 b=r1 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r6 += d0_; r7 += d1_; } // t649 mm8 a=r7 b=r2 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r3 += d0_; r5 += d1_; } // t650 mm8 a=r7 b=r7 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r3 += d0_; r4 += d1_; } // t651 mm8 a=r6 b=r3 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r3 += d0_; r2 += d1_; } // t652 mm8 a=r1 b=r6 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r0 += d0_; r5 += d1_; } // t653 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r7 += d0_; r4 += d1_; } // t654 mm8 a=r1 b=r6 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r3 += d0_; r1 += d1_; } // t655 mm8 a=r7 b=r5 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r1 += d0_; r6 += d1_; } // t656 mm8 a=r6 b=r5 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r7 += d0_; r2 += d1_; } // t657 mm8 a=r7 b=r4 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r6 += d0_; r1 += d1_; } // t658 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r3 += d0_; r0 += d1_; } // t659 mm8 a=r0 b=r7 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r4 += d0_; r5 += d1_; } // t660 mm8 a=r6 b=r1 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r1 += d0_; r4 += d1_; } // t661 mm8 a=r3 b=r7 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r5 += d0_; r3 += d1_; } // t662 mm8 a=r2 b=r3 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r0 += d0_; r4 += d1_; } // t663 mm8 a=r0 b=r3 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r1 += d0_; r3 += d1_; } // t664 mm8 a=r2 b=r0 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r1 += d0_; r0 += d1_; } // t665 mm8 a=r1 b=r1 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r2 += d0_; r7 += d1_; } // t666 mm8 a=r7 b=r1 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r3 += d0_; r6 += d1_; } // t667 mm8 a=r2 b=r0 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r3 += d0_; r6 += d1_; } // t668 mm8 a=r3 b=r1 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r4 += d0_; r3 += d1_; } // t669 mm8 a=r1 b=r4 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r5 += d0_; r0 += d1_; } // t670 mm8 a=r7 b=r7 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r0 += d0_; r3 += d1_; } // t671 mm8 a=r0 b=r3 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r4 += d0_; r3 += d1_; } // t672 mm8 a=r0 b=r1 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r0 += d0_; r2 += d1_; } // t673 mm8 a=r6 b=r2 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r0 += d0_; r5 += d1_; } // t674 mm8 a=r3 b=r3 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r3 += d0_; r6 += d1_; } // t675 mm8 a=r5 b=r6 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r1 += d0_; r0 += d1_; } // t676 mm8 a=r4 b=r3 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t677 mm8 a=r0 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r7 += d0_; r1 += d1_; } // t678 mm8 a=r4 b=r7 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r1 += d0_; r4 += d1_; } // t679 mm8 a=r4 b=r7 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r5 += d0_; r2 += d1_; } // t680 mm8 a=r5 b=r1 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r4 += d0_; r2 += d1_; } // t681 mm8 a=r4 b=r2 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t682 mm8 a=r3 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r4 += d0_; r7 += d1_; } // t683 mm8 a=r0 b=r5 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r3 += d0_; r7 += d1_; } // t684 mm8 a=r6 b=r2 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r0 += d0_; r5 += d1_; } // t685 mm8 a=r7 b=r3 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r1 += d0_; r7 += d1_; } // t686 mm8 a=r4 b=r1 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r3 += d0_; r0 += d1_; } // t687 mm8 a=r0 b=r5 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r4 += d0_; r1 += d1_; } // t688 mm8 a=r3 b=r6 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r7 += d0_; r1 += d1_; } // t689 mm8 a=r3 b=r1 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r0 += d0_; r1 += d1_; } // t690 mm8 a=r2 b=r6 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r0 += d0_; r6 += d1_; } // t691 mm8 a=r2 b=r6 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r0 += d0_; r7 += d1_; } // t692 mm8 a=r1 b=r4 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t693 mm8 a=r7 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r7 += d0_; r3 += d1_; } // t694 mm8 a=r3 b=r5 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r2 += d0_; r4 += d1_; } // t695 mm8 a=r3 b=r2 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r3 += d0_; r0 += d1_; } // t696 mm8 a=r2 b=r4 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r3 += d0_; r4 += d1_; } // t697 mm8 a=r2 b=r3 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r4 += d0_; r7 += d1_; } // t698 mm8 a=r3 b=r0 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r4 += d0_; r2 += d1_; } // t699 mm8 a=r2 b=r1 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r1 += d0_; r3 += d1_; } // t700 mm8 a=r0 b=r1 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r1 += d0_; r3 += d1_; } // t701 mm8 a=r5 b=r7 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r4 += d0_; r6 += d1_; } // t702 mm8 a=r5 b=r5 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r1 += d0_; r2 += d1_; } // t703 mm8 a=r2 b=r2 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r7 += d0_; r2 += d1_; } // t704 mm8 a=r6 b=r5 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r7 += d0_; r3 += d1_; } // t705 mm8 a=r5 b=r4 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r4 += d0_; r1 += d1_; } // t706 mm8 a=r6 b=r0 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r7 += d0_; r6 += d1_; } // t707 mm8 a=r6 b=r4 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r7 += d0_; r4 += d1_; } // t708 mm8 a=r2 b=r0 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r7 += d0_; r5 += d1_; } // t709 mm8 a=r7 b=r0 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r5 += d0_; r6 += d1_; } // t710 mm8 a=r4 b=r2 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r6 += d0_; r1 += d1_; } // t711 mm8 a=r4 b=r7 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r2 += d1_; } // t712 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r5 += d0_; r1 += d1_; } // t713 mm8 a=r5 b=r2 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r5 += d0_; r1 += d1_; } // t714 mm8 a=r0 b=r0 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t715 mm8 a=r1 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r5 += d0_; r6 += d1_; } // t716 mm8 a=r3 b=r3 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r7 += d0_; r1 += d1_; } // t717 mm8 a=r3 b=r3 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r2 += d0_; r7 += d1_; } // t718 mm8 a=r7 b=r5 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r3 += d0_; r0 += d1_; } // t719 mm8 a=r4 b=r7 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r5 += d0_; r3 += d1_; } // t720 mm8 a=r4 b=r7 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r0 += d0_; r3 += d1_; } // t721 mm8 a=r6 b=r4 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r7 += d0_; r6 += d1_; } // t722 mm8 a=r7 b=r5 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r7 += d0_; r2 += d1_; } // t723 mm8 a=r5 b=r0 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r4 += d0_; r0 += d1_; } // t724 mm8 a=r7 b=r4 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r3 += d0_; r5 += d1_; } // t725 mm8 a=r6 b=r1 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r5 += d0_; r1 += d1_; } // t726 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r4 += d0_; r6 += d1_; } // t727 mm8 a=r0 b=r6 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t728 mm8 a=r5 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r2 += d0_; r3 += d1_; } // t729 mm8 a=r5 b=r6 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r4 += d0_; r7 += d1_; } // t730 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r6 += d0_; r7 += d1_; } // t731 mm8 a=r5 b=r6 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r0 += d0_; r6 += d1_; } // t732 mm8 a=r5 b=r1 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t733 mm8 a=r3 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r0 += d0_; r3 += d1_; } // t734 mm8 a=r4 b=r3 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r4 += d0_; r3 += d1_; } // t735 mm8 a=r7 b=r4 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r3 += d0_; r0 += d1_; } // t736 mm8 a=r4 b=r2 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r1 += d0_; r2 += d1_; } // t737 mm8 a=r4 b=r7 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r1 += d0_; r2 += d1_; } // t738 mm8 a=r6 b=r2 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r6 += d0_; r4 += d1_; } // t739 mm8 a=r7 b=r5 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r2 += d0_; r7 += d1_; } // t740 mm8 a=r4 b=r3 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t741 mm8 a=r7 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r4 += d0_; r1 += d1_; } // t742 mm8 a=r6 b=r5 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r6 += d0_; r4 += d1_; } // t743 mm8 a=r5 b=r3 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r3 += d0_; r6 += d1_; } // t744 mm8 a=r7 b=r4 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r0 += d0_; r4 += d1_; } // t745 mm8 a=r7 b=r0 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r1 += d0_; r3 += d1_; } // t746 mm8 a=r2 b=r5 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r1 += d0_; r0 += d1_; } // t747 mm8 a=r0 b=r3 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r5 += d0_; r7 += d1_; } // t748 mm8 a=r1 b=r6 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r7 += d0_; r5 += d1_; } // t749 mm8 a=r2 b=r5 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r7 += d0_; r0 += d1_; } // t750 mm8 a=r4 b=r3 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r5 += d0_; r3 += d1_; } // t751 mm8 a=r6 b=r6 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r5 += d0_; r3 += d1_; } // t752 mm8 a=r5 b=r0 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r6 += d0_; r0 += d1_; } // t753 mm8 a=r6 b=r7 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r5 += d0_; r2 += d1_; } // t754 mm8 a=r2 b=r4 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r4 += d0_; r0 += d1_; } // t755 mm8 a=r5 b=r6 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r5 += d0_; r4 += d1_; } // t756 mm8 a=r7 b=r3 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r3 += d0_; r5 += d1_; } // t757 mm8 a=r6 b=r6 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r4 += d0_; r2 += d1_; } // t758 mm8 a=r6 b=r2 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r0 += d1_; } // t759 mm8 a=r2 b=r7 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r1 += d0_; r0 += d1_; } // t760 mm8 a=r7 b=r7 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r2 += d0_; r4 += d1_; } // t761 mm8 a=r3 b=r3 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r5 += d0_; r6 += d1_; } // t762 mm8 a=r5 b=r2 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r6 += d0_; r0 += d1_; } // t763 mm8 a=r1 b=r1 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r0 += d0_; r2 += d1_; } // t764 mm8 a=r1 b=r2 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r3 += d1_; } // t765 mm8 a=r2 b=r4 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r6 += d0_; r0 += d1_; } // t766 mm8 a=r7 b=r6 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r5 += d0_; r7 += d1_; } // t767 mm8 a=r7 b=r5 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r7 += d0_; r3 += d1_; } // t768 mm8 a=r5 b=r6 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r1 += d0_; r5 += d1_; } // t769 mm8 a=r4 b=r5 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r2 += d0_; r6 += d1_; } // t770 mm8 a=r7 b=r5 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r3 += d0_; r6 += d1_; } // t771 mm8 a=r2 b=r5 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r1 += d0_; r6 += d1_; } // t772 mm8 a=r0 b=r4 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r1 += d0_; r3 += d1_; } // t773 mm8 a=r2 b=r5 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r2 += d0_; r0 += d1_; } // t774 mm8 a=r4 b=r0 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t775 mm8 a=r4 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r1 += d0_; r0 += d1_; } // t776 mm8 a=r5 b=r7 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r2 += d0_; r0 += d1_; } // t777 mm8 a=r1 b=r1 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r5 += d0_; r7 += d1_; } // t778 mm8 a=r1 b=r2 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r0 += d0_; r7 += d1_; } // t779 mm8 a=r7 b=r5 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r0 += d0_; r3 += d1_; } // t780 mm8 a=r3 b=r7 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r7 += d0_; r3 += d1_; } // t781 mm8 a=r6 b=r7 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r6 += d0_; r4 += d1_; } // t782 mm8 a=r6 b=r4 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r6 += d0_; r0 += d1_; } // t783 mm8 a=r6 b=r5 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r6 += d1_; } // t784 mm8 a=r1 b=r5 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r5 += d0_; r3 += d1_; } // t785 mm8 a=r7 b=r2 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r5 += d0_; r2 += d1_; } // t786 mm8 a=r2 b=r4 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r4 += d0_; r6 += d1_; } // t787 mm8 a=r7 b=r2 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r7 += d0_; r6 += d1_; } // t788 mm8 a=r0 b=r0 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r3 += d0_; r5 += d1_; } // t789 mm8 a=r1 b=r4 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r2 += d0_; r3 += d1_; } // t790 mm8 a=r3 b=r1 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r7 += d0_; r4 += d1_; } // t791 mm8 a=r3 b=r5 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r6 += d0_; r4 += d1_; } // t792 mm8 a=r7 b=r1 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r1 += d0_; r0 += d1_; } // t793 mm8 a=r7 b=r2 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r3 += d0_; r0 += d1_; } // t794 mm8 a=r2 b=r4 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r2 += d1_; } // t795 mm8 a=r1 b=r7 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r3 += d0_; r6 += d1_; } // t796 mm8 a=r2 b=r4 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r2 += d0_; r7 += d1_; } // t797 mm8 a=r6 b=r2 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r2 += d0_; r1 += d1_; } // t798 mm8 a=r1 b=r3 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r4 += d0_; r2 += d1_; } // t799 mm8 a=r1 b=r3 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r6 += d0_; r4 += d1_; } // t800 mm8 a=r0 b=r4 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r1 += d0_; r7 += d1_; } // t801 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r6 += d0_; r0 += d1_; } // t802 mm8 a=r4 b=r7 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r7 += d0_; r5 += d1_; } // t803 mm8 a=r3 b=r2 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r7 += d0_; r2 += d1_; } // t804 mm8 a=r0 b=r4 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r5 += d0_; r2 += d1_; } // t805 mm8 a=r4 b=r6 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r2 += d0_; r7 += d1_; } // t806 mm8 a=r0 b=r5 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r2 += d0_; r3 += d1_; } // t807 mm8 a=r5 b=r0 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r7 += d0_; r4 += d1_; } // t808 mm8 a=r2 b=r4 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r4 += d0_; r6 += d1_; } // t809 mm8 a=r1 b=r6 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r5 += d0_; r4 += d1_; } // t810 mm8 a=r1 b=r2 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t811 mm8 a=r2 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r6 += d0_; r1 += d1_; } // t812 mm8 a=r0 b=r6 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r2 += d0_; r3 += d1_; } // t813 mm8 a=r2 b=r2 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r2 += d0_; r6 += d1_; } // t814 mm8 a=r3 b=r4 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r5 += d0_; r7 += d1_; } // t815 mm8 a=r4 b=r5 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r1 += d0_; r7 += d1_; } // t816 mm8 a=r3 b=r0 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r5 += d0_; r7 += d1_; } // t817 mm8 a=r3 b=r4 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r5 += d0_; r2 += d1_; } // t818 mm8 a=r2 b=r2 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r7 += d0_; r4 += d1_; } // t819 mm8 a=r1 b=r2 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r2 += d0_; r6 += d1_; } // t820 mm8 a=r3 b=r6 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r5 += d0_; r1 += d1_; } // t821 mm8 a=r7 b=r3 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r4 += d0_; r3 += d1_; } // t822 mm8 a=r1 b=r5 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r2 += d0_; r5 += d1_; } // t823 mm8 a=r1 b=r4 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r0 += d0_; r5 += d1_; } // t824 mm8 a=r7 b=r7 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r1 += d0_; r2 += d1_; } // t825 mm8 a=r1 b=r1 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r7 += d0_; r1 += d1_; } // t826 mm8 a=r0 b=r2 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r6 += d0_; r7 += d1_; } // t827 mm8 a=r3 b=r1 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r2 += d0_; r4 += d1_; } // t828 mm8 a=r7 b=r4 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r0 += d0_; r5 += d1_; } // t829 mm8 a=r5 b=r2 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r4 += d0_; r0 += d1_; } // t830 mm8 a=r6 b=r3 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r5 += d1_; } // t831 mm8 a=r2 b=r7 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r6 += d0_; r1 += d1_; } // t832 mm8 a=r1 b=r6 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r5 += d0_; r2 += d1_; } // t833 mm8 a=r0 b=r6 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r3 += d0_; r6 += d1_; } // t834 mm8 a=r2 b=r4 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r5 += d0_; r0 += d1_; } // t835 mm8 a=r0 b=r7 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t836 mm8 a=r3 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r6 += d0_; r4 += d1_; } // t837 mm8 a=r4 b=r7 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r4 += d0_; r3 += d1_; } // t838 mm8 a=r5 b=r4 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r7 += d0_; r3 += d1_; } // t839 mm8 a=r0 b=r6 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r5 += d0_; r3 += d1_; } // t840 mm8 a=r1 b=r6 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r4 += d0_; r2 += d1_; } // t841 mm8 a=r4 b=r0 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r3 += d0_; r7 += d1_; } // t842 mm8 a=r3 b=r5 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r2 += d0_; r4 += d1_; } // t843 mm8 a=r2 b=r2 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r2 += d0_; r1 += d1_; } // t844 mm8 a=r3 b=r3 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t845 mm8 a=r1 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r2 += d0_; r3 += d1_; } // t846 mm8 a=r1 b=r4 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t847 mm8 a=r4 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r1 += d0_; r0 += d1_; } // t848 mm8 a=r6 b=r7 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r5 += d0_; r4 += d1_; } // t849 mm8 a=r6 b=r6 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r6 += d0_; r3 += d1_; } // t850 mm8 a=r6 b=r0 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r0 += d0_; r3 += d1_; } // t851 mm8 a=r0 b=r5 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r2 += d0_; r1 += d1_; } // t852 mm8 a=r2 b=r2 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r6 += d0_; r7 += d1_; } // t853 mm8 a=r6 b=r5 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r5 += d0_; r7 += d1_; } // t854 mm8 a=r1 b=r4 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r0 += d0_; r1 += d1_; } // t855 mm8 a=r1 b=r7 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r1 += d0_; r7 += d1_; } // t856 mm8 a=r2 b=r3 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r4 += d0_; r0 += d1_; } // t857 mm8 a=r6 b=r5 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r7 += d1_; } // t858 mm8 a=r0 b=r5 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t859 mm8 a=r2 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r4 += d0_; r7 += d1_; } // t860 mm8 a=r7 b=r2 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r7 += d0_; r4 += d1_; } // t861 mm8 a=r3 b=r3 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r4 += d0_; r5 += d1_; } // t862 mm8 a=r0 b=r1 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r4 += d0_; r1 += d1_; } // t863 mm8 a=r0 b=r0 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r2 += d0_; r1 += d1_; } // t864 mm8 a=r5 b=r5 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t865 mm8 a=r3 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r1 += d0_; r5 += d1_; } // t866 mm8 a=r6 b=r6 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t867 mm8 a=r7 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t868 mm8 a=r4 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r2 += d0_; r7 += d1_; } // t869 mm8 a=r3 b=r6 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r6 += d0_; r5 += d1_; } // t870 mm8 a=r0 b=r1 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r3 += d0_; r1 += d1_; } // t871 mm8 a=r5 b=r1 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r1 += d0_; r4 += d1_; } // t872 mm8 a=r0 b=r6 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r2 += d0_; r3 += d1_; } // t873 mm8 a=r2 b=r1 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r5 += d0_; r6 += d1_; } // t874 mm8 a=r5 b=r1 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r2 += d0_; r5 += d1_; } // t875 mm8 a=r1 b=r7 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r7 += d0_; r1 += d1_; } // t876 mm8 a=r7 b=r6 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r6 += d0_; r1 += d1_; } // t877 mm8 a=r6 b=r2 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r7 += d0_; r4 += d1_; } // t878 mm8 a=r5 b=r0 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r7 += d0_; r2 += d1_; } // t879 mm8 a=r4 b=r1 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r7 += d0_; r1 += d1_; } // t880 mm8 a=r3 b=r3 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r0 += d0_; r5 += d1_; } // t881 mm8 a=r2 b=r7 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r1 += d0_; r4 += d1_; } // t882 mm8 a=r7 b=r5 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r1 += d0_; r2 += d1_; } // t883 mm8 a=r2 b=r3 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r5 += d0_; r2 += d1_; } // t884 mm8 a=r2 b=r7 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r4 += d0_; r0 += d1_; } // t885 mm8 a=r3 b=r5 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r5 += d0_; r3 += d1_; } // t886 mm8 a=r5 b=r4 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r4 += d0_; r7 += d1_; } // t887 mm8 a=r1 b=r1 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r7 += d0_; r3 += d1_; } // t888 mm8 a=r6 b=r2 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r4 += d0_; r5 += d1_; } // t889 mm8 a=r4 b=r6 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r6 += d0_; r2 += d1_; } // t890 mm8 a=r0 b=r5 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r2 += d0_; r0 += d1_; } // t891 mm8 a=r0 b=r7 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r7 += d0_; r4 += d1_; } // t892 mm8 a=r5 b=r6 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r0 += d0_; r1 += d1_; } // t893 mm8 a=r2 b=r1 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r3 += d0_; r7 += d1_; } // t894 mm8 a=r0 b=r7 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r5 += d0_; r0 += d1_; } // t895 mm8 a=r2 b=r4 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r2 += d0_; r5 += d1_; } // t896 mm8 a=r0 b=r2 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r6 += d0_; r0 += d1_; } // t897 mm8 a=r7 b=r4 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r3 += d0_; r0 += d1_; } // t898 mm8 a=r1 b=r3 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r1 += d0_; r3 += d1_; } // t899 mm8 a=r5 b=r1 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r7 += d0_; r0 += d1_; } // t900 mm8 a=r7 b=r7 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r5 += d1_; } // t901 mm8 a=r4 b=r6 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r5 += d0_; r6 += d1_; } // t902 mm8 a=r2 b=r1 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r1 += d0_; r6 += d1_; } // t903 mm8 a=r0 b=r3 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r6 += d1_; } // t904 mm8 a=r6 b=r7 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r6 += d0_; r1 += d1_; } // t905 mm8 a=r0 b=r4 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r2 += d0_; r3 += d1_; } // t906 mm8 a=r3 b=r7 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r0 += d0_; r1 += d1_; } // t907 mm8 a=r2 b=r4 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r4 += d0_; r3 += d1_; } // t908 mm8 a=r7 b=r1 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r7 += d0_; r5 += d1_; } // t909 mm8 a=r4 b=r5 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r4 += d0_; r5 += d1_; } // t910 mm8 a=r3 b=r7 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r3 += d0_; r5 += d1_; } // t911 mm8 a=r2 b=r6 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r4 += d0_; r5 += d1_; } // t912 mm8 a=r6 b=r1 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r2 += d0_; r6 += d1_; } // t913 mm8 a=r2 b=r3 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r5 += d0_; r4 += d1_; } // t914 mm8 a=r7 b=r4 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r3 += d0_; r5 += d1_; } // t915 mm8 a=r4 b=r3 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r3 += d0_; r6 += d1_; } // t916 mm8 a=r4 b=r2 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r6 += d0_; r7 += d1_; } // t917 mm8 a=r5 b=r5 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r3 += d0_; r7 += d1_; } // t918 mm8 a=r7 b=r2 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r6 += d0_; r7 += d1_; } // t919 mm8 a=r3 b=r4 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r2 += d0_; r6 += d1_; } // t920 mm8 a=r6 b=r5 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r0 += d0_; r2 += d1_; } // t921 mm8 a=r6 b=r5 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r4 += d0_; r7 += d1_; } // t922 mm8 a=r1 b=r0 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r6 += d0_; r7 += d1_; } // t923 mm8 a=r6 b=r3 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r7 += d0_; r6 += d1_; } // t924 mm8 a=r2 b=r0 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r3 += d0_; r4 += d1_; } // t925 mm8 a=r6 b=r6 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r2 += d0_; r0 += d1_; } // t926 mm8 a=r5 b=r0 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t927 mm8 a=r1 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r7 += d0_; r5 += d1_; } // t928 mm8 a=r5 b=r7 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r7 += d0_; r5 += d1_; } // t929 mm8 a=r1 b=r0 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t930 mm8 a=r7 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t931 mm8 a=r7 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r6 += d0_; r4 += d1_; } // t932 mm8 a=r1 b=r3 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r0 += d0_; r3 += d1_; } // t933 mm8 a=r1 b=r3 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r6 += d0_; r0 += d1_; } // t934 mm8 a=r6 b=r0 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r6 += d0_; r7 += d1_; } // t935 mm8 a=r5 b=r0 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r1 += d0_; r6 += d1_; } // t936 mm8 a=r3 b=r3 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r7 += d0_; r5 += d1_; } // t937 mm8 a=r0 b=r6 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r1 += d0_; r3 += d1_; } // t938 mm8 a=r1 b=r5 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r3 += d0_; r0 += d1_; } // t939 mm8 a=r7 b=r0 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r4 += d0_; r5 += d1_; } // t940 mm8 a=r1 b=r2 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r2 += d0_; r7 += d1_; } // t941 mm8 a=r1 b=r6 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r1 += d0_; r3 += d1_; } // t942 mm8 a=r0 b=r6 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r1 += d0_; r4 += d1_; } // t943 mm8 a=r4 b=r0 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r2 += d0_; r1 += d1_; } // t944 mm8 a=r6 b=r5 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r4 += d0_; r2 += d1_; } // t945 mm8 a=r3 b=r7 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t946 mm8 a=r2 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r5 += d0_; r3 += d1_; } // t947 mm8 a=r7 b=r2 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t948 mm8 a=r6 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r0 += d0_; r6 += d1_; } // t949 mm8 a=r6 b=r6 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t950 mm8 a=r7 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t951 mm8 a=r2 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r1 += d0_; r2 += d1_; } // t952 mm8 a=r3 b=r2 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r0 += d0_; r5 += d1_; } // t953 mm8 a=r4 b=r4 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r2 += d1_; } // t954 mm8 a=r2 b=r4 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r0 += d0_; r7 += d1_; } // t955 mm8 a=r0 b=r7 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t956 mm8 a=r7 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t957 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r7 += d0_; r4 += d1_; } // t958 mm8 a=r6 b=r5 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r1 += d0_; r5 += d1_; } // t959 mm8 a=r4 b=r7 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r4 += d0_; r2 += d1_; } // t960 mm8 a=r1 b=r1 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r6 += d0_; r3 += d1_; } // t961 mm8 a=r1 b=r5 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r0 += d0_; r2 += d1_; } // t962 mm8 a=r6 b=r4 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r4 += d0_; r7 += d1_; } // t963 mm8 a=r7 b=r1 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r1 += d0_; r0 += d1_; } // t964 mm8 a=r2 b=r5 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t965 mm8 a=r3 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r6 += d0_; r0 += d1_; } // t966 mm8 a=r7 b=r3 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r7 += d0_; r1 += d1_; } // t967 mm8 a=r0 b=r0 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r5 += d0_; r7 += d1_; } // t968 mm8 a=r5 b=r7 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r6 += d0_; r1 += d1_; } // t969 mm8 a=r4 b=r4 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r0 += d0_; r7 += d1_; } // t970 mm8 a=r4 b=r3 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r7 += d0_; r3 += d1_; } // t971 mm8 a=r3 b=r4 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r4 += d0_; r7 += d1_; } // t972 mm8 a=r1 b=r7 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r1 += d0_; r7 += d1_; } // t973 mm8 a=r4 b=r1 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r3 += d0_; r2 += d1_; } // t974 mm8 a=r5 b=r6 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r6 += d0_; r4 += d1_; } // t975 mm8 a=r1 b=r2 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r7 += d0_; r6 += d1_; } // t976 mm8 a=r0 b=r6 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r4 += d0_; r0 += d1_; } // t977 mm8 a=r2 b=r0 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r2 += d0_; r6 += d1_; } // t978 mm8 a=r0 b=r2 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r3 += d0_; r6 += d1_; } // t979 mm8 a=r4 b=r0 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r2 += d0_; r7 += d1_; } // t980 mm8 a=r3 b=r1 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r2 += d0_; r1 += d1_; } // t981 mm8 a=r2 b=r0 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t982 mm8 a=r1 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r7 += d0_; r4 += d1_; } // t983 mm8 a=r2 b=r1 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r7 += d0_; r1 += d1_; } // t984 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r1 += d0_; r5 += d1_; } // t985 mm8 a=r4 b=r3 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r5 += d1_; } // t986 mm8 a=r2 b=r7 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r7 += d0_; r3 += d1_; } // t987 mm8 a=r5 b=r1 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r3 += d0_; r7 += d1_; } // t988 mm8 a=r6 b=r0 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r1 += d0_; r3 += d1_; } // t989 mm8 a=r2 b=r3 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r4 += d0_; r3 += d1_; } // t990 mm8 a=r1 b=r5 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r1 += d0_; r0 += d1_; } // t991 mm8 a=r6 b=r0 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r0 += d0_; r4 += d1_; } // t992 mm8 a=r2 b=r1 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r1 += d0_; r7 += d1_; } // t993 mm8 a=r6 b=r0 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r6 += d0_; r2 += d1_; } // t994 mm8 a=r6 b=r0 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r1 += d0_; r3 += d1_; } // t995 mm8 a=r0 b=r3 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r6 += d0_; r0 += d1_; } // t996 mm8 a=r7 b=r0 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r0 += d0_; r6 += d1_; } // t997 mm8 a=r5 b=r5 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r1 += d0_; r7 += d1_; } // t998 mm8 a=r3 b=r4 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r4 += d0_; r7 += d1_; } // t999 mm8 a=r7 b=r2 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r5 += d0_; r3 += d1_; } // t1000 mm8 a=r5 b=r3 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r0 += d0_; r5 += d1_; } // t1001 mm8 a=r5 b=r6 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r1 += d0_; r4 += d1_; } // t1002 mm8 a=r6 b=r2 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r1 += d0_; r5 += d1_; } // t1003 mm8 a=r4 b=r2 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r4 += d0_; r5 += d1_; } // t1004 mm8 a=r7 b=r3 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r4 += d0_; r1 += d1_; } // t1005 mm8 a=r1 b=r7 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r2 += d0_; r7 += d1_; } // t1006 mm8 a=r4 b=r2 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r4 += d0_; r5 += d1_; } // t1007 mm8 a=r0 b=r2 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r5 += d0_; r7 += d1_; } // t1008 mm8 a=r3 b=r5 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r6 += d0_; r7 += d1_; } // t1009 mm8 a=r0 b=r7 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r2 += d0_; r5 += d1_; } // t1010 mm8 a=r6 b=r7 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r5 += d0_; r2 += d1_; } // t1011 mm8 a=r7 b=r5 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r1 += d0_; r5 += d1_; } // t1012 mm8 a=r3 b=r5 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r6 += d0_; r1 += d1_; } // t1013 mm8 a=r6 b=r6 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r2 += d0_; r3 += d1_; } // t1014 mm8 a=r7 b=r3 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r2 += d0_; r6 += d1_; } // t1015 mm8 a=r7 b=r6 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r1 += d0_; r6 += d1_; } // t1016 mm8 a=r4 b=r0 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r6 += d0_; r3 += d1_; } // t1017 mm8 a=r5 b=r2 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r2 += d0_; r6 += d1_; } // t1018 mm8 a=r7 b=r6 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r5 += d0_; r4 += d1_; } // t1019 mm8 a=r0 b=r7 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r5 += d0_; r4 += d1_; } // t1020 mm8 a=r5 b=r6 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r3 += d0_; r4 += d1_; } // t1021 mm8 a=r7 b=r6 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r2 += d0_; r6 += d1_; } // t1022 mm8 a=r3 b=r5 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r6 += d0_; r7 += d1_; } // t1023 mm8 a=r0 b=r3 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r2 += d0_; r1 += d1_; } // t1024 mm8 a=r2 b=r6 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r2 += d0_; r0 += d1_; } // t1025 mm8 a=r4 b=r7 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r2 += d0_; r4 += d1_; } // t1026 mm8 a=r0 b=r1 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r5 += d0_; r0 += d1_; } // t1027 mm8 a=r1 b=r5 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r3 += d0_; r6 += d1_; } // t1028 mm8 a=r7 b=r6 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r0 += d0_; r7 += d1_; } // t1029 mm8 a=r2 b=r1 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r0 += d0_; r2 += d1_; } // t1030 mm8 a=r1 b=r7 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r0 += d0_; r3 += d1_; } // t1031 mm8 a=r3 b=r7 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r7 += d0_; r2 += d1_; } // t1032 mm8 a=r2 b=r1 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r5 += d0_; r6 += d1_; } // t1033 mm8 a=r2 b=r7 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r4 += d0_; r3 += d1_; } // t1034 mm8 a=r4 b=r3 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r5 += d0_; r6 += d1_; } // t1035 mm8 a=r3 b=r0 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r7 += d0_; r5 += d1_; } // t1036 mm8 a=r3 b=r0 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r4 += d0_; r2 += d1_; } // t1037 mm8 a=r7 b=r1 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r1 += d0_; r3 += d1_; } // t1038 mm8 a=r2 b=r1 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t1039 mm8 a=r5 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r3 += d0_; r7 += d1_; } // t1040 mm8 a=r0 b=r7 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r7 += d0_; r1 += d1_; } // t1041 mm8 a=r2 b=r2 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r2 += d0_; r4 += d1_; } // t1042 mm8 a=r6 b=r6 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r3 += d0_; r6 += d1_; } // t1043 mm8 a=r6 b=r7 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r7 += d0_; r1 += d1_; } // t1044 mm8 a=r4 b=r2 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r1 += d0_; r6 += d1_; } // t1045 mm8 a=r0 b=r0 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r3 += d0_; r2 += d1_; } // t1046 mm8 a=r7 b=r1 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r5 += d0_; r6 += d1_; } // t1047 mm8 a=r5 b=r7 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r6 += d0_; r0 += d1_; } // t1048 mm8 a=r2 b=r3 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r6 += d0_; r7 += d1_; } // t1049 mm8 a=r5 b=r1 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r0 += d1_; } // t1050 mm8 a=r0 b=r5 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r4 += d0_; r2 += d1_; } // t1051 mm8 a=r7 b=r7 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t1052 mm8 a=r2 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r2 += d0_; r0 += d1_; } // t1053 mm8 a=r6 b=r0 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r4 += d0_; r5 += d1_; } // t1054 mm8 a=r4 b=r2 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r7 += d0_; r2 += d1_; } // t1055 mm8 a=r5 b=r3 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r5 += d0_; r6 += d1_; } // t1056 mm8 a=r5 b=r1 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r5 += d0_; r4 += d1_; } // t1057 mm8 a=r1 b=r5 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r4 += d0_; r3 += d1_; } // t1058 mm8 a=r0 b=r5 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r4 += d0_; r5 += d1_; } // t1059 mm8 a=r5 b=r6 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r5 += d0_; r0 += d1_; } // t1060 mm8 a=r7 b=r3 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r6 += d0_; r1 += d1_; } // t1061 mm8 a=r5 b=r6 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r2 += d0_; r0 += d1_; } // t1062 mm8 a=r3 b=r0 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r7 += d0_; r0 += d1_; } // t1063 mm8 a=r1 b=r1 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r2 += d0_; r1 += d1_; } // t1064 mm8 a=r5 b=r4 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r4 += d0_; r5 += d1_; } // t1065 mm8 a=r3 b=r1 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r4 += d0_; r5 += d1_; } // t1066 mm8 a=r7 b=r7 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r6 += d0_; r1 += d1_; } // t1067 mm8 a=r0 b=r5 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r4 += d0_; r6 += d1_; } // t1068 mm8 a=r4 b=r6 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r6 += d0_; r5 += d1_; } // t1069 mm8 a=r2 b=r0 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r2 += d0_; r1 += d1_; } // t1070 mm8 a=r2 b=r3 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r1 += d0_; r7 += d1_; } // t1071 mm8 a=r2 b=r6 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r2 += d0_; r4 += d1_; } // t1072 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r2 += d0_; r0 += d1_; } // t1073 mm8 a=r3 b=r6 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r7 += d0_; r1 += d1_; } // t1074 mm8 a=r0 b=r0 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r7 += d0_; r1 += d1_; } // t1075 mm8 a=r4 b=r4 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r3 += d0_; r6 += d1_; } // t1076 mm8 a=r1 b=r7 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t1077 mm8 a=r7 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r1 += d0_; r3 += d1_; } // t1078 mm8 a=r3 b=r7 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r7 += d0_; r4 += d1_; } // t1079 mm8 a=r6 b=r6 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r1 += d0_; r4 += d1_; } // t1080 mm8 a=r4 b=r5 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r4 += d0_; r5 += d1_; } // t1081 mm8 a=r1 b=r5 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r3 += d0_; r6 += d1_; } // t1082 mm8 a=r5 b=r4 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r6 += d0_; r7 += d1_; } // t1083 mm8 a=r2 b=r2 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r2 += d0_; r1 += d1_; } // t1084 mm8 a=r2 b=r0 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r0 += d0_; r4 += d1_; } // t1085 mm8 a=r3 b=r3 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r3 += d0_; r2 += d1_; } // t1086 mm8 a=r3 b=r1 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r6 += d0_; r5 += d1_; } // t1087 mm8 a=r1 b=r3 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r5 += d0_; r7 += d1_; } // t1088 mm8 a=r3 b=r5 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r2 += d0_; r3 += d1_; } // t1089 mm8 a=r0 b=r0 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r7 += d0_; r2 += d1_; } // t1090 mm8 a=r4 b=r6 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r6 += d0_; r4 += d1_; } // t1091 mm8 a=r4 b=r3 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r1 += d0_; r6 += d1_; } // t1092 mm8 a=r2 b=r6 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r0 += d0_; r4 += d1_; } // t1093 mm8 a=r5 b=r7 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r2 += d0_; r6 += d1_; } // t1094 mm8 a=r7 b=r3 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r4 += d0_; r6 += d1_; } // t1095 mm8 a=r0 b=r2 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r6 += d0_; r1 += d1_; } // t1096 mm8 a=r3 b=r3 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r5 += d0_; r4 += d1_; } // t1097 mm8 a=r6 b=r2 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r1 += d0_; r6 += d1_; } // t1098 mm8 a=r3 b=r6 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r4 += d0_; r0 += d1_; } // t1099 mm8 a=r2 b=r5 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r1 += d0_; r0 += d1_; } // t1100 mm8 a=r1 b=r4 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r5 += d0_; r7 += d1_; } // t1101 mm8 a=r0 b=r6 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r6 += d0_; r5 += d1_; } // t1102 mm8 a=r0 b=r5 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t1103 mm8 a=r7 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r7 += d0_; r4 += d1_; } // t1104 mm8 a=r4 b=r2 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r2 += d0_; r0 += d1_; } // t1105 mm8 a=r3 b=r4 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r7 += d0_; r0 += d1_; } // t1106 mm8 a=r2 b=r1 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r6 += d0_; r5 += d1_; } // t1107 mm8 a=r3 b=r0 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r4 += d0_; r2 += d1_; } // t1108 mm8 a=r3 b=r5 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r4 += d0_; r3 += d1_; } // t1109 mm8 a=r6 b=r2 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r4 += d0_; r7 += d1_; } // t1110 mm8 a=r2 b=r4 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r2 += d0_; r1 += d1_; } // t1111 mm8 a=r0 b=r7 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r3 += d0_; r0 += d1_; } // t1112 mm8 a=r1 b=r0 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r5 += d0_; r6 += d1_; } // t1113 mm8 a=r4 b=r6 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r0 += d0_; r6 += d1_; } // t1114 mm8 a=r0 b=r5 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r3 += d0_; r4 += d1_; } // t1115 mm8 a=r4 b=r5 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r7 += d0_; r5 += d1_; } // t1116 mm8 a=r4 b=r6 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r0 += d0_; r7 += d1_; } // t1117 mm8 a=r0 b=r6 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r5 += d0_; r6 += d1_; } // t1118 mm8 a=r7 b=r6 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r0 += d0_; r5 += d1_; } // t1119 mm8 a=r2 b=r7 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r0 += d0_; r5 += d1_; } // t1120 mm8 a=r7 b=r5 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r6 += d0_; r7 += d1_; } // t1121 mm8 a=r4 b=r0 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r3 += d0_; r2 += d1_; } // t1122 mm8 a=r0 b=r5 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t1123 mm8 a=r0 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r4 += d0_; r0 += d1_; } // t1124 mm8 a=r0 b=r1 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r2 += d0_; r0 += d1_; } // t1125 mm8 a=r6 b=r6 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r0 += d0_; r3 += d1_; } // t1126 mm8 a=r0 b=r0 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r7 += d0_; r1 += d1_; } // t1127 mm8 a=r1 b=r1 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r7 += d0_; r3 += d1_; } // t1128 mm8 a=r4 b=r1 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r6 += d0_; r2 += d1_; } // t1129 mm8 a=r0 b=r0 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r7 += d0_; r5 += d1_; } // t1130 mm8 a=r0 b=r4 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r3 += d0_; r0 += d1_; } // t1131 mm8 a=r3 b=r0 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r5 += d0_; r7 += d1_; } // t1132 mm8 a=r0 b=r6 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r2 += d0_; r1 += d1_; } // t1133 mm8 a=r6 b=r0 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r1 += d0_; r4 += d1_; } // t1134 mm8 a=r7 b=r3 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r6 += d0_; r4 += d1_; } // t1135 mm8 a=r6 b=r7 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r7 += d0_; r2 += d1_; } // t1136 mm8 a=r7 b=r4 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r5 += d0_; r7 += d1_; } // t1137 mm8 a=r4 b=r0 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r3 += d0_; r4 += d1_; } // t1138 mm8 a=r6 b=r4 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r1 += d0_; r7 += d1_; } // t1139 mm8 a=r6 b=r2 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r6 += d0_; r3 += d1_; } // t1140 mm8 a=r1 b=r2 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r6 += d1_; } // t1141 mm8 a=r0 b=r5 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r2 += d0_; r5 += d1_; } // t1142 mm8 a=r4 b=r5 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r6 += d0_; r1 += d1_; } // t1143 mm8 a=r5 b=r6 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r7 += d0_; r3 += d1_; } // t1144 mm8 a=r6 b=r4 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r2 += d0_; r7 += d1_; } // t1145 mm8 a=r5 b=r1 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r5 += d0_; r3 += d1_; } // t1146 mm8 a=r6 b=r5 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r4 += d0_; r6 += d1_; } // t1147 mm8 a=r7 b=r1 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r4 += d0_; r6 += d1_; } // t1148 mm8 a=r1 b=r6 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r5 += d0_; r7 += d1_; } // t1149 mm8 a=r5 b=r5 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r5 += d0_; r1 += d1_; } // t1150 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r7 += d0_; r5 += d1_; } // t1151 mm8 a=r7 b=r5 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r5 += d0_; r3 += d1_; } // t1152 mm8 a=r0 b=r6 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r0 += d0_; r3 += d1_; } // t1153 mm8 a=r2 b=r5 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r7 += d0_; r0 += d1_; } // t1154 mm8 a=r5 b=r3 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r0 += d0_; r4 += d1_; } // t1155 mm8 a=r0 b=r6 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r4 += d0_; r7 += d1_; } // t1156 mm8 a=r2 b=r6 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r1 += d0_; r2 += d1_; } // t1157 mm8 a=r6 b=r7 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r3 += d0_; r7 += d1_; } // t1158 mm8 a=r2 b=r3 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t1159 mm8 a=r1 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r2 += d0_; r7 += d1_; } // t1160 mm8 a=r5 b=r0 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r3 += d0_; r4 += d1_; } // t1161 mm8 a=r7 b=r0 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r0 += d0_; r4 += d1_; } // t1162 mm8 a=r5 b=r3 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r2 += d0_; r1 += d1_; } // t1163 mm8 a=r4 b=r3 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r2 += d0_; r0 += d1_; } // t1164 mm8 a=r3 b=r4 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r7 += d0_; r1 += d1_; } // t1165 mm8 a=r1 b=r2 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r7 += d0_; r4 += d1_; } // t1166 mm8 a=r4 b=r7 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r5 += d1_; } // t1167 mm8 a=r2 b=r7 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r4 += d0_; r0 += d1_; } // t1168 mm8 a=r2 b=r6 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r3 += d0_; r7 += d1_; } // t1169 mm8 a=r0 b=r4 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r3 += d0_; r4 += d1_; } // t1170 mm8 a=r2 b=r1 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r7 += d1_; } // t1171 mm8 a=r1 b=r7 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r0 += d0_; r4 += d1_; } // t1172 mm8 a=r0 b=r1 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t1173 mm8 a=r6 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r2 += d0_; r6 += d1_; } // t1174 mm8 a=r0 b=r1 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r3 += d0_; r1 += d1_; } // t1175 mm8 a=r0 b=r0 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r5 += d0_; r6 += d1_; } // t1176 mm8 a=r4 b=r5 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r5 += d0_; r2 += d1_; } // t1177 mm8 a=r7 b=r2 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r7 += d0_; r0 += d1_; } // t1178 mm8 a=r1 b=r5 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t1179 mm8 a=r0 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r5 += d0_; r2 += d1_; } // t1180 mm8 a=r7 b=r3 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r0 += d0_; r7 += d1_; } // t1181 mm8 a=r7 b=r3 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r7 += d0_; r3 += d1_; } // t1182 mm8 a=r1 b=r2 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t1183 mm8 a=r2 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r2 += d0_; r7 += d1_; } // t1184 mm8 a=r7 b=r0 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r4 += d0_; r5 += d1_; } // t1185 mm8 a=r5 b=r2 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r2 += d0_; r3 += d1_; } // t1186 mm8 a=r1 b=r0 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r2 += d0_; r7 += d1_; } // t1187 mm8 a=r5 b=r4 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r4 += d0_; r6 += d1_; } // t1188 mm8 a=r3 b=r0 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t1189 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r2 += d0_; r7 += d1_; } // t1190 mm8 a=r5 b=r0 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r5 += d0_; r1 += d1_; } // t1191 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r2 += d0_; r1 += d1_; } // t1192 mm8 a=r0 b=r2 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r5 += d0_; r1 += d1_; } // t1193 mm8 a=r3 b=r7 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r1 += d0_; r2 += d1_; } // t1194 mm8 a=r7 b=r2 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r0 += d0_; r4 += d1_; } // t1195 mm8 a=r1 b=r4 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r5 += d0_; r0 += d1_; } // t1196 mm8 a=r4 b=r6 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r3 += d0_; r4 += d1_; } // t1197 mm8 a=r5 b=r6 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r2 += d0_; r5 += d1_; } // t1198 mm8 a=r0 b=r6 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r7 += d0_; r2 += d1_; } // t1199 mm8 a=r5 b=r1 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r1 += d0_; r2 += d1_; } // t1200 mm8 a=r1 b=r3 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r2 += d0_; r0 += d1_; } // t1201 mm8 a=r6 b=r6 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r4 += d0_; r7 += d1_; } // t1202 mm8 a=r2 b=r4 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r4 += d0_; r2 += d1_; } // t1203 mm8 a=r0 b=r5 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r7 += d0_; r4 += d1_; } // t1204 mm8 a=r0 b=r2 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r7 += d0_; r3 += d1_; } // t1205 mm8 a=r4 b=r6 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r4 += d0_; r6 += d1_; } // t1206 mm8 a=r4 b=r6 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r7 += d0_; r6 += d1_; } // t1207 mm8 a=r3 b=r7 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r2 += d0_; r4 += d1_; } // t1208 mm8 a=r6 b=r1 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r2 += d0_; r4 += d1_; } // t1209 mm8 a=r5 b=r5 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r3 += d0_; r0 += d1_; } // t1210 mm8 a=r0 b=r0 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r2 += d0_; r3 += d1_; } // t1211 mm8 a=r7 b=r5 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r7 += d0_; r1 += d1_; } // t1212 mm8 a=r2 b=r6 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r0 += d0_; r1 += d1_; } // t1213 mm8 a=r6 b=r2 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r3 += d0_; r5 += d1_; } // t1214 mm8 a=r4 b=r2 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r1 += d0_; r7 += d1_; } // t1215 mm8 a=r7 b=r2 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r7 += d0_; r0 += d1_; } // t1216 mm8 a=r3 b=r3 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r3 += d0_; r5 += d1_; } // t1217 mm8 a=r5 b=r5 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r3 += d0_; r6 += d1_; } // t1218 mm8 a=r0 b=r7 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r0 += d0_; r4 += d1_; } // t1219 mm8 a=r2 b=r4 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r7 += d0_; r4 += d1_; } // t1220 mm8 a=r1 b=r7 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r3 += d0_; r2 += d1_; } // t1221 mm8 a=r5 b=r5 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r2 += d0_; r0 += d1_; } // t1222 mm8 a=r1 b=r3 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r7 += d0_; r5 += d1_; } // t1223 mm8 a=r0 b=r2 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r6 += d0_; r3 += d1_; } // t1224 mm8 a=r1 b=r1 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r6 += d0_; r3 += d1_; } // t1225 mm8 a=r4 b=r1 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r4 += d0_; r5 += d1_; } // t1226 mm8 a=r5 b=r6 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r1 += d0_; r6 += d1_; } // t1227 mm8 a=r1 b=r7 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r4 += d0_; r5 += d1_; } // t1228 mm8 a=r4 b=r2 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r0 += d0_; r6 += d1_; } // t1229 mm8 a=r5 b=r3 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r2 += d0_; r7 += d1_; } // t1230 mm8 a=r6 b=r2 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r5 += d0_; r3 += d1_; } // t1231 mm8 a=r4 b=r3 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r4 += d1_; } // t1232 mm8 a=r6 b=r7 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r4 += d0_; r2 += d1_; } // t1233 mm8 a=r0 b=r5 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r7 += d0_; r1 += d1_; } // t1234 mm8 a=r5 b=r4 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r3 += d0_; r5 += d1_; } // t1235 mm8 a=r5 b=r7 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r1 += d0_; r3 += d1_; } // t1236 mm8 a=r0 b=r7 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r4 += d0_; r1 += d1_; } // t1237 mm8 a=r7 b=r2 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r5 += d0_; r0 += d1_; } // t1238 mm8 a=r7 b=r0 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r3 += d0_; r2 += d1_; } // t1239 mm8 a=r3 b=r2 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r2 += d0_; r4 += d1_; } // t1240 mm8 a=r0 b=r5 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r3 += d0_; r7 += d1_; } // t1241 mm8 a=r5 b=r0 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r7 += d0_; r4 += d1_; } // t1242 mm8 a=r5 b=r7 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r6 += d0_; r7 += d1_; } // t1243 mm8 a=r6 b=r3 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t1244 mm8 a=r1 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r5 += d0_; r1 += d1_; } // t1245 mm8 a=r0 b=r6 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r1 += d0_; r4 += d1_; } // t1246 mm8 a=r5 b=r5 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r7 += d0_; r1 += d1_; } // t1247 mm8 a=r1 b=r6 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r2 += d0_; r7 += d1_; } // t1248 mm8 a=r4 b=r0 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r1 += d0_; r7 += d1_; } // t1249 mm8 a=r7 b=r6 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r6 += d0_; r1 += d1_; } // t1250 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r1 += d0_; r0 += d1_; } // t1251 mm8 a=r3 b=r7 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r2 += d0_; r7 += d1_; } // t1252 mm8 a=r5 b=r5 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r6 += d0_; r2 += d1_; } // t1253 mm8 a=r5 b=r5 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r3 += d0_; r5 += d1_; } // t1254 mm8 a=r0 b=r6 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t1255 mm8 a=r1 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r5 += d0_; r6 += d1_; } // t1256 mm8 a=r1 b=r3 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r7 += d0_; r3 += d1_; } // t1257 mm8 a=r0 b=r7 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r4 += d0_; r3 += d1_; } // t1258 mm8 a=r5 b=r4 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r5 += d0_; r2 += d1_; } // t1259 mm8 a=r0 b=r3 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r5 += d0_; r7 += d1_; } // t1260 mm8 a=r1 b=r5 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r4 += d0_; r2 += d1_; } // t1261 mm8 a=r6 b=r7 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r5 += d0_; r1 += d1_; } // t1262 mm8 a=r2 b=r5 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r4 += d0_; r1 += d1_; } // t1263 mm8 a=r3 b=r7 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r6 += d0_; r7 += d1_; } // t1264 mm8 a=r1 b=r6 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r3 += d0_; r6 += d1_; } // t1265 mm8 a=r2 b=r3 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r3 += d1_; } // t1266 mm8 a=r4 b=r6 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r7 += d0_; r3 += d1_; } // t1267 mm8 a=r6 b=r6 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r5 += d0_; r7 += d1_; } // t1268 mm8 a=r4 b=r5 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r7 += d0_; r4 += d1_; } // t1269 mm8 a=r5 b=r7 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r7 += d0_; r3 += d1_; } // t1270 mm8 a=r7 b=r3 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r6 += d0_; r0 += d1_; } // t1271 mm8 a=r7 b=r0 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r1 += d0_; r3 += d1_; } // t1272 mm8 a=r1 b=r3 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r7 += d0_; r5 += d1_; } // t1273 mm8 a=r3 b=r2 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r6 += d0_; r3 += d1_; } // t1274 mm8 a=r2 b=r0 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r0 += d0_; r5 += d1_; } // t1275 mm8 a=r7 b=r3 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r5 += d0_; r2 += d1_; } // t1276 mm8 a=r4 b=r3 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r2 += d0_; r4 += d1_; } // t1277 mm8 a=r5 b=r4 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r1 += d0_; r6 += d1_; } // t1278 mm8 a=r6 b=r3 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r7 += d0_; r1 += d1_; } // t1279 mm8 a=r0 b=r1 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r4 += d0_; r2 += d1_; } // t1280 mm8 a=r4 b=r1 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r2 += d0_; r4 += d1_; } // t1281 mm8 a=r6 b=r3 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r2 += d0_; r5 += d1_; } // t1282 mm8 a=r1 b=r7 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r7 += d0_; r1 += d1_; } // t1283 mm8 a=r6 b=r2 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r7 += d0_; r2 += d1_; } // t1284 mm8 a=r1 b=r6 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r1 += d0_; r4 += d1_; } // t1285 mm8 a=r3 b=r5 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r2 += d0_; r4 += d1_; } // t1286 mm8 a=r3 b=r1 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r5 += d0_; r3 += d1_; } // t1287 mm8 a=r4 b=r1 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r6 += d0_; r0 += d1_; } // t1288 mm8 a=r3 b=r6 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r2 += d0_; r5 += d1_; } // t1289 mm8 a=r6 b=r2 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r1 += d0_; r5 += d1_; } // t1290 mm8 a=r6 b=r5 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r7 += d0_; r6 += d1_; } // t1291 mm8 a=r5 b=r6 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r1 += d0_; r6 += d1_; } // t1292 mm8 a=r6 b=r6 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r0 += d0_; r4 += d1_; } // t1293 mm8 a=r6 b=r2 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r2 += d0_; r3 += d1_; } // t1294 mm8 a=r0 b=r5 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r2 += d0_; r6 += d1_; } // t1295 mm8 a=r5 b=r0 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r3 += d0_; r5 += d1_; } // t1296 mm8 a=r1 b=r0 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r6 += d0_; r1 += d1_; } // t1297 mm8 a=r0 b=r0 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r0 += d0_; r4 += d1_; } // t1298 mm8 a=r2 b=r6 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r7 += d0_; r4 += d1_; } // t1299 mm8 a=r7 b=r0 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r4 += d0_; r5 += d1_; } // t1300 mm8 a=r4 b=r1 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r2 += d0_; r5 += d1_; } // t1301 mm8 a=r4 b=r5 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r7 += d0_; r2 += d1_; } // t1302 mm8 a=r0 b=r4 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r7 += d0_; r1 += d1_; } // t1303 mm8 a=r4 b=r2 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r3 += d0_; r7 += d1_; } // t1304 mm8 a=r4 b=r7 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r2 += d0_; r4 += d1_; } // t1305 mm8 a=r3 b=r2 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t1306 mm8 a=r3 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r6 += d0_; r7 += d1_; } // t1307 mm8 a=r0 b=r2 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r7 += d0_; r1 += d1_; } // t1308 mm8 a=r2 b=r5 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r5 += d0_; r4 += d1_; } // t1309 mm8 a=r5 b=r7 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r4 += d0_; r7 += d1_; } // t1310 mm8 a=r4 b=r5 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r1 += d0_; r4 += d1_; } // t1311 mm8 a=r4 b=r7 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t1312 mm8 a=r3 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r7 += d0_; r0 += d1_; } // t1313 mm8 a=r5 b=r1 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r0 += d0_; r2 += d1_; } // t1314 mm8 a=r5 b=r2 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r3 += d0_; r7 += d1_; } // t1315 mm8 a=r1 b=r0 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r6 += d0_; r7 += d1_; } // t1316 mm8 a=r7 b=r3 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r0 += d0_; r3 += d1_; } // t1317 mm8 a=r2 b=r5 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r5 += d0_; r4 += d1_; } // t1318 mm8 a=r2 b=r3 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r0 += d0_; r5 += d1_; } // t1319 mm8 a=r7 b=r5 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r4 += d0_; r6 += d1_; } // t1320 mm8 a=r0 b=r3 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r6 += d0_; r1 += d1_; } // t1321 mm8 a=r1 b=r0 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r6 += d0_; r3 += d1_; } // t1322 mm8 a=r4 b=r0 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r0 += d0_; r3 += d1_; } // t1323 mm8 a=r5 b=r3 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r4 += d0_; r6 += d1_; } // t1324 mm8 a=r3 b=r0 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r1 += d0_; r0 += d1_; } // t1325 mm8 a=r6 b=r4 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r0 += d1_; } // t1326 mm8 a=r4 b=r6 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r0 += d0_; r4 += d1_; } // t1327 mm8 a=r1 b=r3 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r7 += d0_; r6 += d1_; } // t1328 mm8 a=r7 b=r3 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r6 += d0_; r2 += d1_; } // t1329 mm8 a=r3 b=r7 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r5 += d0_; r3 += d1_; } // t1330 mm8 a=r3 b=r6 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r2 += d0_; r3 += d1_; } // t1331 mm8 a=r0 b=r7 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r5 += d0_; r4 += d1_; } // t1332 mm8 a=r0 b=r7 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r5 += d0_; r1 += d1_; } // t1333 mm8 a=r3 b=r3 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r0 += d0_; r3 += d1_; } // t1334 mm8 a=r7 b=r5 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r7 += d0_; r6 += d1_; } // t1335 mm8 a=r5 b=r7 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r1 += d0_; r2 += d1_; } // t1336 mm8 a=r5 b=r5 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r6 += d0_; r7 += d1_; } // t1337 mm8 a=r6 b=r0 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r7 += d0_; r6 += d1_; } // t1338 mm8 a=r6 b=r4 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r6 += d0_; r0 += d1_; } // t1339 mm8 a=r4 b=r6 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r7 += d0_; r0 += d1_; } // t1340 mm8 a=r6 b=r1 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r1 += d0_; r4 += d1_; } // t1341 mm8 a=r0 b=r6 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r0 += d0_; r1 += d1_; } // t1342 mm8 a=r4 b=r6 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r2 += d0_; r1 += d1_; } // t1343 mm8 a=r5 b=r5 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r3 += d0_; r1 += d1_; } // t1344 mm8 a=r2 b=r6 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r4 += d0_; r2 += d1_; } // t1345 mm8 a=r7 b=r3 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r0 += d0_; r2 += d1_; } // t1346 mm8 a=r5 b=r0 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r4 += d0_; r2 += d1_; } // t1347 mm8 a=r2 b=r3 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r2 += d0_; r1 += d1_; } // t1348 mm8 a=r4 b=r3 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r3 += d0_; r1 += d1_; } // t1349 mm8 a=r0 b=r1 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r0 += d0_; r4 += d1_; } // t1350 mm8 a=r1 b=r6 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r2 += d1_; } // t1351 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r5 += d0_; r0 += d1_; } // t1352 mm8 a=r5 b=r4 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t1353 mm8 a=r2 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r7 += d0_; r0 += d1_; } // t1354 mm8 a=r7 b=r4 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r3 += d0_; r0 += d1_; } // t1355 mm8 a=r7 b=r4 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r5 += d0_; r3 += d1_; } // t1356 mm8 a=r4 b=r6 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r6 += d1_; } // t1357 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r7 += d0_; r1 += d1_; } // t1358 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t1359 mm8 a=r7 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t1360 mm8 a=r0 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r7 += d0_; r4 += d1_; } // t1361 mm8 a=r0 b=r5 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r4 += d0_; r0 += d1_; } // t1362 mm8 a=r0 b=r2 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r5 += d0_; r6 += d1_; } // t1363 mm8 a=r4 b=r4 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r0 += d0_; r2 += d1_; } // t1364 mm8 a=r7 b=r3 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r3 += d0_; r4 += d1_; } // t1365 mm8 a=r2 b=r4 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r1 += d0_; r5 += d1_; } // t1366 mm8 a=r4 b=r3 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r6 += d0_; r2 += d1_; } // t1367 mm8 a=r3 b=r3 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r5 += d0_; r6 += d1_; } // t1368 mm8 a=r4 b=r0 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r2 += d0_; r7 += d1_; } // t1369 mm8 a=r5 b=r3 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t1370 mm8 a=r0 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r0 += d0_; r3 += d1_; } // t1371 mm8 a=r0 b=r5 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r7 += d0_; r2 += d1_; } // t1372 mm8 a=r5 b=r6 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r3 += d0_; r2 += d1_; } // t1373 mm8 a=r7 b=r1 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t1374 mm8 a=r4 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r2 += d0_; r5 += d1_; } // t1375 mm8 a=r5 b=r2 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r5 += d0_; r2 += d1_; } // t1376 mm8 a=r5 b=r5 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r7 += d0_; r5 += d1_; } // t1377 mm8 a=r7 b=r3 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r2 += d0_; r6 += d1_; } // t1378 mm8 a=r4 b=r1 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r7 += d0_; r2 += d1_; } // t1379 mm8 a=r4 b=r0 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r3 += d0_; r6 += d1_; } // t1380 mm8 a=r0 b=r7 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r0 += d0_; r7 += d1_; } // t1381 mm8 a=r2 b=r0 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r4 += d0_; r7 += d1_; } // t1382 mm8 a=r7 b=r5 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r1 += d1_; } // t1383 mm8 a=r1 b=r5 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r2 += d0_; r6 += d1_; } // t1384 mm8 a=r5 b=r2 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r5 += d0_; r1 += d1_; } // t1385 mm8 a=r0 b=r7 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r0 += d0_; r1 += d1_; } // t1386 mm8 a=r7 b=r0 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r0 += d0_; r2 += d1_; } // t1387 mm8 a=r4 b=r3 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r7 += d0_; r5 += d1_; } // t1388 mm8 a=r4 b=r6 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r3 += d0_; r2 += d1_; } // t1389 mm8 a=r4 b=r2 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r0 += d0_; r3 += d1_; } // t1390 mm8 a=r5 b=r1 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r4 += d0_; r0 += d1_; } // t1391 mm8 a=r6 b=r1 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r0 += d0_; r6 += d1_; } // t1392 mm8 a=r5 b=r7 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r7 += d0_; r4 += d1_; } // t1393 mm8 a=r3 b=r4 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r3 += d0_; r5 += d1_; } // t1394 mm8 a=r4 b=r7 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r6 += d0_; r5 += d1_; } // t1395 mm8 a=r7 b=r1 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r6 += d0_; r1 += d1_; } // t1396 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t1397 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r4 += d0_; r3 += d1_; } // t1398 mm8 a=r3 b=r0 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t1399 mm8 a=r3 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r2 += d0_; r5 += d1_; } // t1400 mm8 a=r6 b=r4 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r3 += d0_; r5 += d1_; } // t1401 mm8 a=r0 b=r5 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r6 += d0_; r2 += d1_; } // t1402 mm8 a=r7 b=r5 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r4 += d0_; r7 += d1_; } // t1403 mm8 a=r0 b=r0 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r6 += d0_; r2 += d1_; } // t1404 mm8 a=r3 b=r6 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r2 += d0_; r3 += d1_; } // t1405 mm8 a=r0 b=r0 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r4 += d0_; r5 += d1_; } // t1406 mm8 a=r5 b=r3 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r3 += d0_; r1 += d1_; } // t1407 mm8 a=r7 b=r3 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r0 += d0_; r1 += d1_; } // t1408 mm8 a=r4 b=r7 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r6 += d0_; r2 += d1_; } // t1409 mm8 a=r4 b=r2 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r6 += d0_; r3 += d1_; } // t1410 mm8 a=r6 b=r0 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r4 += d1_; } // t1411 mm8 a=r1 b=r7 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r3 += d0_; r2 += d1_; } // t1412 mm8 a=r4 b=r3 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r4 += d0_; r2 += d1_; } // t1413 mm8 a=r4 b=r6 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r2 += d0_; r5 += d1_; } // t1414 mm8 a=r0 b=r7 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r7 += d0_; r5 += d1_; } // t1415 mm8 a=r3 b=r6 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r4 += d0_; r7 += d1_; } // t1416 mm8 a=r3 b=r2 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r0 += d0_; r2 += d1_; } // t1417 mm8 a=r2 b=r7 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r2 += d0_; r7 += d1_; } // t1418 mm8 a=r0 b=r6 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t1419 mm8 a=r0 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r5 += d0_; r1 += d1_; } // t1420 mm8 a=r1 b=r3 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r3 += d0_; r2 += d1_; } // t1421 mm8 a=r4 b=r0 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r6 += d0_; r2 += d1_; } // t1422 mm8 a=r6 b=r4 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r3 += d0_; r1 += d1_; } // t1423 mm8 a=r4 b=r6 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r2 += d0_; r5 += d1_; } // t1424 mm8 a=r6 b=r3 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r0 += d0_; r1 += d1_; } // t1425 mm8 a=r7 b=r3 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r5 += d0_; r4 += d1_; } // t1426 mm8 a=r0 b=r3 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r3 += d0_; r6 += d1_; } // t1427 mm8 a=r5 b=r6 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r5 += d0_; r7 += d1_; } // t1428 mm8 a=r0 b=r3 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r3 += d0_; r7 += d1_; } // t1429 mm8 a=r7 b=r1 c=r3 c2=r7 - } - uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((ulong)hi << 32) | (ulong)lo; -} - -#if IGNEUM_EXCHANGE != 0 -// Reports the sub-group size this device uses for a work-group of IGNEUM_GROUP items. host.c runs it only when the -// per-kernel query (clGetKernelSubGroupInfoKHR on igneum_hash) is unavailable; that query is preferred because a -// compiler may pick a different wave width per kernel (RDNA: wave32 or wave64). See WAVEFRONT.md. -IGNEUM_KERNEL_HASH void igneum_probe_subgroup(__global uint* out) { - if (get_local_id(0) == 0u) { out[0] = get_sub_group_size(); out[1] = get_num_sub_groups(); } -} -#endif diff --git a/proto-cuda/packs-ca4/mx8_mm1430/kernel.cu b/proto-cuda/packs-ca4/mx8_mm1430/kernel.cu deleted file mode 100644 index 1d87d3130..000000000 --- a/proto-cuda/packs-ca4/mx8_mm1430/kernel.cu +++ /dev/null @@ -1,1595 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// Bit-exact twin of the Metal kernel for the same seed (see proto-cuda/CHECKLIST.md and program.metal). -// Compiled ahead of time by nvcc together with proto-cuda/host.cu. No NVRTC. -#include -#include -#include "program.h" -#include "memhard.h" - -__device__ __forceinline__ uint32_t splitmix32(uint32_t x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -// n is a literal in 1..31 at every call site, so both shift amounts are in 1..31. -__device__ __forceinline__ uint32_t rotl_imm(uint32_t x, uint32_t n) { return (x << n) | (x >> (32u - n)); } -// n is masked to 0..31; the second shift amount is masked too, so n == 0 gives x. -__device__ __forceinline__ uint32_t rotr_var(uint32_t x, uint32_t n) { n &= 31u; return (x >> n) | (x << ((32u - n) & 31u)); } -__device__ __forceinline__ uint32_t ds_elem(uint32_t i, uint32_t d0, uint32_t d1) { - uint32_t x = i ^ d0; - x *= 0x9E3779B1u; x ^= x >> 15; - x += d1; - x *= 0x85EBCA77u; x ^= x >> 13; - x *= 0xC2B2AE3Du; x ^= x >> 16; - return x; -} - -// Memory-hard dataset (MEMHARD.md). One thread per cache segment; one thread per 64-byte dataset item. -// The core functions (mh_cache_segment, mh_item) are in memhard.h and are also compiled for the host. -__global__ void igneum_cache_fill(uint32_t* cache, uint32_t nSegments) { - uint32_t seg = blockIdx.x * blockDim.x + threadIdx.x; - if (seg < nSegments) mh_cache_segment(cache, seg); -} -__global__ void igneum_build(uint32_t* ds, const uint32_t* cache, uint32_t nItems) { - uint32_t t = blockIdx.x * blockDim.x + threadIdx.x; - if (t < nItems) { - uint32_t s[16]; - mh_item(cache, t, s); - uint32_t* d = ds + (size_t)t * 16u; - for (uint32_t i = 0u; i < 16u; ++i) d[i] = s[i]; - } -} - -// One hash per thread. blockDim.x is a multiple of 32; lane = threadIdx.x & 31 and every -// __shfl_xor_sync stays inside the lane's own warp, exactly like simd_shuffle_xor inside a -// 32-wide Metal SIMD group. Control flow is uniform, so the full 0xffffffff member mask is valid. -__global__ void igneum_hash(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask) { - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t nonce = baseNonce + gid; - uint32_t r0, r1, r2, r3, r4, r5, r6, r7; - { uint32_t x = nonce ^ 0x67a9a7beu; x += 0x9e3779b9u; x = splitmix32(x); r0 = x ^ 0x1a155b25u; } // SEEDW[0], 0x9e3779b9u * 1u, SEEDW[1] - { uint32_t x = nonce ^ 0x1a155b25u; x += 0x3c6ef372u; x = splitmix32(x); r1 = x ^ 0xfddfb732u; } // SEEDW[1], 0x9e3779b9u * 2u, SEEDW[2] - { uint32_t x = nonce ^ 0xfddfb732u; x += 0xdaa66d2bu; x = splitmix32(x); r2 = x ^ 0x4b5af2e8u; } // SEEDW[2], 0x9e3779b9u * 3u, SEEDW[3] - { uint32_t x = nonce ^ 0x4b5af2e8u; x += 0x78dde6e4u; x = splitmix32(x); r3 = x ^ 0xc55caf33u; } // SEEDW[3], 0x9e3779b9u * 4u, SEEDW[4] - { uint32_t x = nonce ^ 0xc55caf33u; x += 0x1715609du; x = splitmix32(x); r4 = x ^ 0xa27c13b7u; } // SEEDW[4], 0x9e3779b9u * 5u, SEEDW[5] - { uint32_t x = nonce ^ 0xa27c13b7u; x += 0xb54cda56u; x = splitmix32(x); r5 = x ^ 0x06628a48u; } // SEEDW[5], 0x9e3779b9u * 6u, SEEDW[6] - { uint32_t x = nonce ^ 0x06628a48u; x += 0x5384540fu; x = splitmix32(x); r6 = x ^ 0x03852469u; } // SEEDW[6], 0x9e3779b9u * 7u, SEEDW[7] - { uint32_t x = nonce ^ 0x03852469u; x += 0xf1bbcdc8u; x = splitmix32(x); r7 = x ^ 0x67a9a7beu; } // SEEDW[7], 0x9e3779b9u * 8u, SEEDW[0] - - for (uint32_t it = 0u; it < 8u; ++it) { - uint32_t sel = r0; - r2 = r3 * r4 + r2; // 0 mad - r2 = r1 * r1 + r2; // 1 mad - r2 = r3 * r2 + r2; // 2 mad - r3 = r3 ^ r5; // 3 xor - r7 = r7 ^ ds[r2 & mask]; // 4 load - r5 = r5 ^ ds[r7 & mask]; // 5 load - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r4, 8); // 6 shfl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r3, 8); // 7 shfl - r1 = __umulhi(r1, r5); // 8 mulhi - r6 = rotr_var(r6, r3); // 9 rotr - r3 = r3 | r4; // 10 or - r4 = r4 ^ ds[r3 & mask]; // 11 load - r0 = __umulhi(r0, r4); // 12 mulhi - r5 = r5 + r1 + ((((sel >> 30u) & 1u) != 0u) ? 0xd3177981u : 0xc7934706u); // 13 add - r0 = r0 ^ ds[r4 & mask]; // 14 load - r2 = r2 - r4; // 15 sub - r2 = r2 ^ ds[r0 & mask]; // 16 load - r7 = r7 ^ ds[r2 & mask]; // 17 load - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r3, 4); // 18 shfl - r5 = r5 * r0; // 19 mul - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r4, 2); // 20 shfl - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r4, 16); // 21 shfl - r6 = __umulhi(r6, r2); // 22 mulhi - r6 = r6 ^ ds[r1 & mask]; // 23 load - r5 = r5 * r0; // 24 mul - r5 = rotl_imm(r5, 19u); // 25 rotl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r6, 2); // 26 shfl - r0 = r0 ^ r5; // 27 xor - r0 = r0 ^ r4; // 28 xor - r3 = r3 - r0; // 29 sub - r5 = r5 * r1; // 30 mul - r7 = r7 ^ ds[r2 & mask]; // 31 load - r1 = r1 ^ ds[r0 & mask]; // 32 load - r5 = r5 ^ r6; // 33 xor - r5 = r5 ^ ds[r1 & mask]; // 34 load - r0 = __umulhi(r0, r5); // 35 mulhi - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r2, 4); // 36 shfl - r7 = r7 ^ ds[r0 & mask]; // 37 load - r3 = r3 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0x230c005cu : 0x75ba2fadu); // 38 add - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r5, 4); // 39 shfl - r2 = r2 ^ r5; // 40 xor - r3 = r6 * r3 + r3; // 41 mad - r6 = r6 - r7; // 42 sub - r7 = r7 ^ r0; // 43 xor - r1 = r1 ^ ds[r7 & mask]; // 44 load - r2 = r2 * r3; // 45 mul - r1 = __umulhi(r1, r5); // 46 mulhi - r4 = r4 - r3; // 47 sub - r2 = rotr_var(r2, r6); // 48 rotr - r3 = r3 ^ ds[r5 & mask]; // 49 load - r1 = r1 + r5 + ((((sel >> 7u) & 1u) != 0u) ? 0x1907970cu : 0x81b8bc2cu); // 50 add - r0 = r0 * r2; // 51 mul - r0 = r0 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x699fd448u : 0x4f92b968u); // 52 add - r1 = r1 + r0 + ((((sel >> 12u) & 1u) != 0u) ? 0x77b1520du : 0x2bb965afu); // 53 add - r7 = rotl_imm(r7, 14u); // 54 rotl - r3 = r3 + r7 + ((((sel >> 1u) & 1u) != 0u) ? 0xa54c55a0u : 0x7b0fe07au); // 55 add - r6 = r6 ^ ds[r7 & mask]; // 56 load - r1 = rotr_var(r1, r5); // 57 rotr - r5 = r5 ^ ds[r4 & mask]; // 58 load - r6 = r6 ^ ds[r2 & mask]; // 59 load - r3 = r5 * r0 + r3; // 60 mad - r5 = r5 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0xad7493e7u : 0xaf9dd72du); // 61 add - r4 = r4 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x1e07c3d9u : 0x89841d87u); // 62 add - r5 = rotl_imm(r5, 19u); // 63 rotl - // int8 tile block (Counter ASIC 4.0 research, experimental): 1430 mma.m8n8k16 u8 tiles per iteration, both outputs consumed - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t0 mm8 a=r6 b=r5 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t1 mm8 a=r1 b=r1 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t2 mm8 a=r2 b=r5 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t3 mm8 a=r1 b=r5 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t4 mm8 a=r2 b=r0 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t5 mm8 a=r2 b=r6 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t6 mm8 a=r1 b=r4 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t7 mm8 a=r4 b=r6 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t8 mm8 a=r1 b=r1 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t9 mm8 a=r3 b=r7 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t10 mm8 a=r3 b=r0 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t11 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t12 mm8 a=r3 b=r1 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t13 mm8 a=r1 b=r5 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t14 mm8 a=r3 b=r3 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t15 mm8 a=r1 b=r0 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t16 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t17 mm8 a=r0 b=r5 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t18 mm8 a=r1 b=r1 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t19 mm8 a=r3 b=r3 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t20 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t21 mm8 a=r0 b=r3 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t22 mm8 a=r1 b=r7 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t23 mm8 a=r1 b=r2 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t24 mm8 a=r3 b=r4 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t25 mm8 a=r5 b=r0 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t26 mm8 a=r1 b=r0 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t27 mm8 a=r3 b=r2 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t28 mm8 a=r2 b=r4 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t29 mm8 a=r3 b=r4 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t30 mm8 a=r6 b=r5 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t31 mm8 a=r1 b=r7 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t32 mm8 a=r7 b=r6 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t33 mm8 a=r7 b=r1 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t34 mm8 a=r1 b=r0 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t35 mm8 a=r2 b=r4 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t36 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t37 mm8 a=r1 b=r7 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t38 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t39 mm8 a=r6 b=r2 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t40 mm8 a=r6 b=r1 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t41 mm8 a=r6 b=r0 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t42 mm8 a=r7 b=r4 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t43 mm8 a=r6 b=r1 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t44 mm8 a=r0 b=r6 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t45 mm8 a=r6 b=r5 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t46 mm8 a=r6 b=r6 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t47 mm8 a=r7 b=r3 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t48 mm8 a=r7 b=r7 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t49 mm8 a=r4 b=r5 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t50 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t51 mm8 a=r3 b=r4 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t52 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t53 mm8 a=r7 b=r7 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t54 mm8 a=r7 b=r6 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t55 mm8 a=r1 b=r7 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t56 mm8 a=r4 b=r0 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t57 mm8 a=r4 b=r3 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t58 mm8 a=r2 b=r3 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t59 mm8 a=r6 b=r7 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t60 mm8 a=r6 b=r0 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t61 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t62 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t63 mm8 a=r1 b=r6 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t64 mm8 a=r0 b=r5 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t65 mm8 a=r0 b=r1 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t66 mm8 a=r3 b=r5 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t67 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t68 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t69 mm8 a=r6 b=r7 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t70 mm8 a=r1 b=r7 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t71 mm8 a=r3 b=r0 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t72 mm8 a=r4 b=r0 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t73 mm8 a=r4 b=r1 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t74 mm8 a=r5 b=r1 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t75 mm8 a=r4 b=r4 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t76 mm8 a=r5 b=r6 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t77 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t78 mm8 a=r2 b=r5 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t79 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t80 mm8 a=r6 b=r7 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t81 mm8 a=r4 b=r3 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t82 mm8 a=r6 b=r0 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t83 mm8 a=r2 b=r2 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t84 mm8 a=r0 b=r6 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t85 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t86 mm8 a=r3 b=r4 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t87 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t88 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t89 mm8 a=r4 b=r2 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t90 mm8 a=r6 b=r5 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t91 mm8 a=r6 b=r5 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t92 mm8 a=r2 b=r6 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t93 mm8 a=r2 b=r2 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t94 mm8 a=r2 b=r4 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t95 mm8 a=r5 b=r4 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t96 mm8 a=r0 b=r6 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t97 mm8 a=r0 b=r5 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t98 mm8 a=r2 b=r7 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t99 mm8 a=r1 b=r4 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t100 mm8 a=r2 b=r4 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t101 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t102 mm8 a=r5 b=r7 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t103 mm8 a=r4 b=r7 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t104 mm8 a=r2 b=r0 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t105 mm8 a=r6 b=r1 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t106 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t107 mm8 a=r2 b=r1 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t108 mm8 a=r4 b=r6 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t109 mm8 a=r2 b=r7 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t110 mm8 a=r1 b=r0 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t111 mm8 a=r6 b=r5 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t112 mm8 a=r5 b=r7 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t113 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t114 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t115 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t116 mm8 a=r7 b=r2 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t117 mm8 a=r7 b=r6 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t118 mm8 a=r0 b=r1 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t119 mm8 a=r4 b=r3 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t120 mm8 a=r5 b=r3 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t121 mm8 a=r2 b=r7 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t122 mm8 a=r0 b=r1 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t123 mm8 a=r5 b=r2 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t124 mm8 a=r5 b=r1 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t125 mm8 a=r7 b=r2 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t126 mm8 a=r6 b=r3 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t127 mm8 a=r6 b=r7 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t128 mm8 a=r6 b=r7 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t129 mm8 a=r3 b=r2 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t130 mm8 a=r0 b=r4 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t131 mm8 a=r3 b=r5 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t132 mm8 a=r0 b=r7 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t133 mm8 a=r2 b=r6 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t134 mm8 a=r5 b=r6 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t135 mm8 a=r0 b=r2 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t136 mm8 a=r3 b=r1 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t137 mm8 a=r7 b=r0 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t138 mm8 a=r2 b=r2 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t139 mm8 a=r0 b=r4 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t140 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t141 mm8 a=r5 b=r5 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t142 mm8 a=r2 b=r2 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t143 mm8 a=r4 b=r3 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t144 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t145 mm8 a=r7 b=r2 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t146 mm8 a=r2 b=r7 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t147 mm8 a=r3 b=r1 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t148 mm8 a=r2 b=r1 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t149 mm8 a=r4 b=r6 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t150 mm8 a=r3 b=r6 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t151 mm8 a=r1 b=r4 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t152 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t153 mm8 a=r0 b=r3 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t154 mm8 a=r7 b=r3 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t155 mm8 a=r1 b=r3 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t156 mm8 a=r6 b=r3 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t157 mm8 a=r6 b=r4 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t158 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t159 mm8 a=r1 b=r0 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t160 mm8 a=r0 b=r1 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t161 mm8 a=r4 b=r0 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t162 mm8 a=r6 b=r6 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t163 mm8 a=r5 b=r3 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t164 mm8 a=r0 b=r7 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t165 mm8 a=r6 b=r5 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t166 mm8 a=r6 b=r5 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t167 mm8 a=r4 b=r5 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t168 mm8 a=r3 b=r6 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t169 mm8 a=r6 b=r2 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t170 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t171 mm8 a=r6 b=r3 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t172 mm8 a=r7 b=r3 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t173 mm8 a=r0 b=r0 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t174 mm8 a=r6 b=r6 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t175 mm8 a=r1 b=r6 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t176 mm8 a=r2 b=r4 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t177 mm8 a=r1 b=r7 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t178 mm8 a=r4 b=r2 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t179 mm8 a=r5 b=r2 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t180 mm8 a=r0 b=r4 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t181 mm8 a=r3 b=r5 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t182 mm8 a=r1 b=r1 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t183 mm8 a=r3 b=r1 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t184 mm8 a=r7 b=r4 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t185 mm8 a=r5 b=r0 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t186 mm8 a=r2 b=r5 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t187 mm8 a=r3 b=r0 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t188 mm8 a=r1 b=r4 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t189 mm8 a=r7 b=r2 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t190 mm8 a=r1 b=r5 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t191 mm8 a=r2 b=r1 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t192 mm8 a=r5 b=r0 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t193 mm8 a=r2 b=r1 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t194 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t195 mm8 a=r2 b=r7 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t196 mm8 a=r5 b=r6 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t197 mm8 a=r5 b=r3 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t198 mm8 a=r1 b=r0 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t199 mm8 a=r2 b=r5 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t200 mm8 a=r5 b=r7 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t201 mm8 a=r7 b=r2 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t202 mm8 a=r2 b=r5 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t203 mm8 a=r0 b=r5 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t204 mm8 a=r4 b=r7 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t205 mm8 a=r3 b=r7 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t206 mm8 a=r0 b=r6 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t207 mm8 a=r7 b=r7 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t208 mm8 a=r0 b=r1 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t209 mm8 a=r4 b=r4 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t210 mm8 a=r7 b=r1 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t211 mm8 a=r1 b=r5 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t212 mm8 a=r2 b=r4 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t213 mm8 a=r2 b=r2 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t214 mm8 a=r4 b=r6 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t215 mm8 a=r7 b=r3 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t216 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t217 mm8 a=r5 b=r7 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t218 mm8 a=r7 b=r1 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t219 mm8 a=r2 b=r1 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t220 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t221 mm8 a=r7 b=r1 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t222 mm8 a=r0 b=r6 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t223 mm8 a=r2 b=r2 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t224 mm8 a=r1 b=r1 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t225 mm8 a=r2 b=r7 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t226 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t227 mm8 a=r3 b=r5 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t228 mm8 a=r7 b=r1 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t229 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t230 mm8 a=r0 b=r3 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t231 mm8 a=r1 b=r3 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t232 mm8 a=r5 b=r4 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t233 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t234 mm8 a=r2 b=r0 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t235 mm8 a=r3 b=r4 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t236 mm8 a=r1 b=r7 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t237 mm8 a=r3 b=r5 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t238 mm8 a=r2 b=r0 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t239 mm8 a=r5 b=r7 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t240 mm8 a=r2 b=r7 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t241 mm8 a=r0 b=r1 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t242 mm8 a=r0 b=r2 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t243 mm8 a=r4 b=r4 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t244 mm8 a=r7 b=r6 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t245 mm8 a=r4 b=r4 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t246 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t247 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t248 mm8 a=r7 b=r5 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t249 mm8 a=r4 b=r4 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t250 mm8 a=r0 b=r6 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t251 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t252 mm8 a=r0 b=r2 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t253 mm8 a=r5 b=r5 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t254 mm8 a=r6 b=r7 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t255 mm8 a=r5 b=r0 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t256 mm8 a=r2 b=r1 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t257 mm8 a=r4 b=r3 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t258 mm8 a=r4 b=r1 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t259 mm8 a=r5 b=r5 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t260 mm8 a=r3 b=r2 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t261 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t262 mm8 a=r6 b=r4 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t263 mm8 a=r6 b=r4 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t264 mm8 a=r7 b=r2 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t265 mm8 a=r6 b=r1 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t266 mm8 a=r0 b=r0 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t267 mm8 a=r6 b=r3 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t268 mm8 a=r1 b=r0 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t269 mm8 a=r7 b=r7 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t270 mm8 a=r6 b=r2 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t271 mm8 a=r0 b=r3 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t272 mm8 a=r6 b=r0 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t273 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t274 mm8 a=r1 b=r0 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t275 mm8 a=r1 b=r3 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t276 mm8 a=r1 b=r5 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t277 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t278 mm8 a=r0 b=r0 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t279 mm8 a=r2 b=r4 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t280 mm8 a=r7 b=r2 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t281 mm8 a=r1 b=r0 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t282 mm8 a=r0 b=r1 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t283 mm8 a=r0 b=r2 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t284 mm8 a=r1 b=r5 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t285 mm8 a=r4 b=r7 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t286 mm8 a=r1 b=r0 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t287 mm8 a=r7 b=r1 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t288 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t289 mm8 a=r0 b=r5 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t290 mm8 a=r3 b=r2 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t291 mm8 a=r7 b=r4 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t292 mm8 a=r6 b=r7 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t293 mm8 a=r5 b=r5 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t294 mm8 a=r5 b=r7 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t295 mm8 a=r0 b=r7 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t296 mm8 a=r6 b=r1 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t297 mm8 a=r4 b=r5 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t298 mm8 a=r6 b=r0 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t299 mm8 a=r6 b=r0 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t300 mm8 a=r6 b=r4 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t301 mm8 a=r6 b=r4 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t302 mm8 a=r7 b=r6 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t303 mm8 a=r3 b=r2 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t304 mm8 a=r6 b=r5 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t305 mm8 a=r4 b=r5 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t306 mm8 a=r3 b=r1 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t307 mm8 a=r4 b=r3 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t308 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t309 mm8 a=r2 b=r1 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t310 mm8 a=r6 b=r1 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t311 mm8 a=r4 b=r6 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t312 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t313 mm8 a=r2 b=r6 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t314 mm8 a=r0 b=r3 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t315 mm8 a=r7 b=r1 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t316 mm8 a=r2 b=r4 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t317 mm8 a=r1 b=r6 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t318 mm8 a=r2 b=r1 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t319 mm8 a=r0 b=r2 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t320 mm8 a=r5 b=r2 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t321 mm8 a=r7 b=r4 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t322 mm8 a=r2 b=r4 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t323 mm8 a=r4 b=r3 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t324 mm8 a=r5 b=r7 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t325 mm8 a=r6 b=r2 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t326 mm8 a=r6 b=r1 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t327 mm8 a=r1 b=r5 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t328 mm8 a=r0 b=r1 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t329 mm8 a=r0 b=r3 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t330 mm8 a=r6 b=r7 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t331 mm8 a=r4 b=r4 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t332 mm8 a=r4 b=r4 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t333 mm8 a=r4 b=r6 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t334 mm8 a=r2 b=r2 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t335 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t336 mm8 a=r4 b=r0 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t337 mm8 a=r1 b=r4 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t338 mm8 a=r3 b=r7 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t339 mm8 a=r5 b=r1 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t340 mm8 a=r5 b=r0 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t341 mm8 a=r6 b=r6 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t342 mm8 a=r4 b=r7 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t343 mm8 a=r0 b=r4 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t344 mm8 a=r2 b=r5 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t345 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t346 mm8 a=r6 b=r0 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t347 mm8 a=r5 b=r3 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t348 mm8 a=r0 b=r4 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t349 mm8 a=r6 b=r1 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t350 mm8 a=r1 b=r5 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t351 mm8 a=r6 b=r4 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t352 mm8 a=r4 b=r1 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t353 mm8 a=r7 b=r3 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t354 mm8 a=r4 b=r1 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t355 mm8 a=r4 b=r5 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t356 mm8 a=r7 b=r6 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t357 mm8 a=r7 b=r3 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t358 mm8 a=r4 b=r7 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t359 mm8 a=r2 b=r7 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t360 mm8 a=r4 b=r4 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t361 mm8 a=r2 b=r2 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t362 mm8 a=r2 b=r7 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t363 mm8 a=r7 b=r6 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t364 mm8 a=r6 b=r4 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t365 mm8 a=r6 b=r6 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t366 mm8 a=r7 b=r7 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t367 mm8 a=r4 b=r1 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t368 mm8 a=r7 b=r0 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t369 mm8 a=r1 b=r0 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t370 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t371 mm8 a=r5 b=r4 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t372 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t373 mm8 a=r4 b=r6 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t374 mm8 a=r1 b=r5 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t375 mm8 a=r6 b=r5 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t376 mm8 a=r3 b=r6 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t377 mm8 a=r6 b=r4 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t378 mm8 a=r6 b=r3 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t379 mm8 a=r3 b=r0 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t380 mm8 a=r2 b=r0 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t381 mm8 a=r3 b=r5 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t382 mm8 a=r5 b=r2 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t383 mm8 a=r1 b=r2 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t384 mm8 a=r3 b=r1 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t385 mm8 a=r6 b=r0 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t386 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t387 mm8 a=r3 b=r7 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t388 mm8 a=r3 b=r0 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t389 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t390 mm8 a=r5 b=r4 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t391 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t392 mm8 a=r2 b=r3 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t393 mm8 a=r3 b=r6 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t394 mm8 a=r0 b=r7 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t395 mm8 a=r0 b=r7 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t396 mm8 a=r7 b=r7 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t397 mm8 a=r5 b=r7 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t398 mm8 a=r6 b=r5 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t399 mm8 a=r6 b=r3 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t400 mm8 a=r2 b=r7 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t401 mm8 a=r7 b=r6 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t402 mm8 a=r0 b=r0 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t403 mm8 a=r7 b=r4 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t404 mm8 a=r1 b=r1 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t405 mm8 a=r6 b=r6 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t406 mm8 a=r6 b=r1 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t407 mm8 a=r5 b=r7 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t408 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t409 mm8 a=r3 b=r4 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t410 mm8 a=r0 b=r5 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t411 mm8 a=r2 b=r0 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t412 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t413 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t414 mm8 a=r7 b=r1 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t415 mm8 a=r4 b=r7 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t416 mm8 a=r5 b=r4 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t417 mm8 a=r6 b=r6 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t418 mm8 a=r1 b=r3 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t419 mm8 a=r5 b=r0 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t420 mm8 a=r4 b=r2 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t421 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t422 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t423 mm8 a=r1 b=r7 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t424 mm8 a=r5 b=r1 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t425 mm8 a=r2 b=r7 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t426 mm8 a=r1 b=r6 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t427 mm8 a=r6 b=r6 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t428 mm8 a=r6 b=r2 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t429 mm8 a=r3 b=r3 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t430 mm8 a=r7 b=r7 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t431 mm8 a=r6 b=r4 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t432 mm8 a=r3 b=r2 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t433 mm8 a=r3 b=r0 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t434 mm8 a=r0 b=r7 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t435 mm8 a=r7 b=r5 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t436 mm8 a=r6 b=r4 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t437 mm8 a=r0 b=r7 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t438 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t439 mm8 a=r4 b=r7 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t440 mm8 a=r3 b=r5 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t441 mm8 a=r5 b=r1 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t442 mm8 a=r1 b=r2 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t443 mm8 a=r1 b=r7 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t444 mm8 a=r3 b=r7 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t445 mm8 a=r7 b=r5 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t446 mm8 a=r3 b=r1 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t447 mm8 a=r6 b=r7 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t448 mm8 a=r5 b=r2 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t449 mm8 a=r7 b=r1 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t450 mm8 a=r7 b=r5 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t451 mm8 a=r0 b=r3 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t452 mm8 a=r2 b=r4 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t453 mm8 a=r5 b=r7 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t454 mm8 a=r7 b=r7 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t455 mm8 a=r7 b=r1 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t456 mm8 a=r6 b=r5 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t457 mm8 a=r7 b=r7 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t458 mm8 a=r5 b=r1 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t459 mm8 a=r6 b=r7 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t460 mm8 a=r7 b=r7 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t461 mm8 a=r0 b=r6 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t462 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t463 mm8 a=r2 b=r0 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t464 mm8 a=r0 b=r3 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t465 mm8 a=r1 b=r7 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t466 mm8 a=r1 b=r2 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t467 mm8 a=r1 b=r0 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t468 mm8 a=r4 b=r1 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t469 mm8 a=r2 b=r0 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t470 mm8 a=r3 b=r1 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t471 mm8 a=r2 b=r2 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t472 mm8 a=r1 b=r7 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t473 mm8 a=r2 b=r7 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t474 mm8 a=r5 b=r3 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t475 mm8 a=r7 b=r6 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t476 mm8 a=r3 b=r0 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t477 mm8 a=r3 b=r1 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t478 mm8 a=r0 b=r6 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t479 mm8 a=r7 b=r6 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t480 mm8 a=r6 b=r0 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t481 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t482 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t483 mm8 a=r7 b=r0 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t484 mm8 a=r3 b=r7 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t485 mm8 a=r6 b=r4 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t486 mm8 a=r6 b=r5 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t487 mm8 a=r2 b=r5 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t488 mm8 a=r1 b=r2 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t489 mm8 a=r5 b=r2 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t490 mm8 a=r7 b=r1 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t491 mm8 a=r0 b=r0 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t492 mm8 a=r3 b=r1 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t493 mm8 a=r6 b=r1 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t494 mm8 a=r5 b=r6 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t495 mm8 a=r2 b=r2 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t496 mm8 a=r3 b=r5 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t497 mm8 a=r3 b=r2 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t498 mm8 a=r3 b=r1 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t499 mm8 a=r1 b=r6 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t500 mm8 a=r7 b=r6 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t501 mm8 a=r4 b=r4 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t502 mm8 a=r5 b=r7 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t503 mm8 a=r7 b=r5 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t504 mm8 a=r2 b=r0 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t505 mm8 a=r4 b=r0 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t506 mm8 a=r0 b=r2 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t507 mm8 a=r5 b=r6 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t508 mm8 a=r7 b=r3 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t509 mm8 a=r4 b=r4 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t510 mm8 a=r0 b=r6 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t511 mm8 a=r2 b=r4 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t512 mm8 a=r2 b=r7 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t513 mm8 a=r0 b=r3 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t514 mm8 a=r6 b=r7 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t515 mm8 a=r5 b=r2 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t516 mm8 a=r0 b=r2 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t517 mm8 a=r6 b=r7 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t518 mm8 a=r1 b=r4 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t519 mm8 a=r6 b=r0 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t520 mm8 a=r1 b=r2 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t521 mm8 a=r0 b=r4 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t522 mm8 a=r7 b=r2 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t523 mm8 a=r5 b=r3 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t524 mm8 a=r0 b=r4 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t525 mm8 a=r1 b=r5 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t526 mm8 a=r7 b=r1 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t527 mm8 a=r5 b=r1 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t528 mm8 a=r4 b=r1 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t529 mm8 a=r4 b=r2 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t530 mm8 a=r0 b=r2 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t531 mm8 a=r7 b=r0 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t532 mm8 a=r7 b=r2 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t533 mm8 a=r3 b=r2 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t534 mm8 a=r4 b=r3 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t535 mm8 a=r0 b=r6 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t536 mm8 a=r3 b=r1 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t537 mm8 a=r1 b=r6 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t538 mm8 a=r6 b=r6 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t539 mm8 a=r4 b=r4 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t540 mm8 a=r1 b=r4 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t541 mm8 a=r5 b=r3 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t542 mm8 a=r7 b=r7 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t543 mm8 a=r4 b=r2 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t544 mm8 a=r5 b=r7 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t545 mm8 a=r5 b=r2 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t546 mm8 a=r7 b=r1 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t547 mm8 a=r3 b=r5 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t548 mm8 a=r6 b=r3 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t549 mm8 a=r2 b=r2 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t550 mm8 a=r5 b=r5 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t551 mm8 a=r7 b=r6 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t552 mm8 a=r6 b=r7 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t553 mm8 a=r6 b=r3 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t554 mm8 a=r4 b=r2 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t555 mm8 a=r2 b=r7 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t556 mm8 a=r3 b=r6 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t557 mm8 a=r7 b=r7 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t558 mm8 a=r5 b=r0 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t559 mm8 a=r4 b=r1 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t560 mm8 a=r5 b=r4 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t561 mm8 a=r6 b=r1 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t562 mm8 a=r1 b=r2 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t563 mm8 a=r5 b=r3 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t564 mm8 a=r4 b=r3 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t565 mm8 a=r6 b=r0 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t566 mm8 a=r1 b=r6 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t567 mm8 a=r3 b=r6 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t568 mm8 a=r6 b=r5 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t569 mm8 a=r3 b=r7 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t570 mm8 a=r5 b=r1 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t571 mm8 a=r6 b=r0 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t572 mm8 a=r5 b=r0 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t573 mm8 a=r3 b=r0 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t574 mm8 a=r2 b=r2 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t575 mm8 a=r3 b=r1 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t576 mm8 a=r2 b=r3 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t577 mm8 a=r6 b=r3 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t578 mm8 a=r5 b=r6 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t579 mm8 a=r4 b=r7 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t580 mm8 a=r7 b=r1 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t581 mm8 a=r1 b=r6 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t582 mm8 a=r2 b=r1 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t583 mm8 a=r5 b=r0 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t584 mm8 a=r1 b=r7 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t585 mm8 a=r0 b=r4 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t586 mm8 a=r4 b=r4 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t587 mm8 a=r0 b=r4 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t588 mm8 a=r5 b=r5 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t589 mm8 a=r3 b=r6 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t590 mm8 a=r3 b=r2 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t591 mm8 a=r5 b=r1 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t592 mm8 a=r5 b=r3 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t593 mm8 a=r5 b=r3 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t594 mm8 a=r6 b=r7 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t595 mm8 a=r3 b=r0 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t596 mm8 a=r7 b=r6 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t597 mm8 a=r2 b=r4 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t598 mm8 a=r7 b=r7 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t599 mm8 a=r7 b=r2 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t600 mm8 a=r7 b=r7 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t601 mm8 a=r2 b=r7 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t602 mm8 a=r0 b=r1 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t603 mm8 a=r0 b=r0 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t604 mm8 a=r5 b=r5 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t605 mm8 a=r2 b=r0 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t606 mm8 a=r7 b=r5 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t607 mm8 a=r0 b=r2 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t608 mm8 a=r7 b=r6 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t609 mm8 a=r1 b=r5 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t610 mm8 a=r3 b=r3 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t611 mm8 a=r2 b=r2 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t612 mm8 a=r1 b=r1 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t613 mm8 a=r6 b=r3 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t614 mm8 a=r3 b=r4 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t615 mm8 a=r6 b=r2 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t616 mm8 a=r2 b=r3 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t617 mm8 a=r5 b=r4 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t618 mm8 a=r3 b=r6 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t619 mm8 a=r5 b=r0 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t620 mm8 a=r3 b=r5 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t621 mm8 a=r4 b=r2 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t622 mm8 a=r7 b=r0 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t623 mm8 a=r4 b=r6 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t624 mm8 a=r6 b=r2 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t625 mm8 a=r6 b=r2 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t626 mm8 a=r5 b=r2 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t627 mm8 a=r4 b=r0 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t628 mm8 a=r1 b=r5 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t629 mm8 a=r4 b=r5 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t630 mm8 a=r4 b=r5 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t631 mm8 a=r6 b=r4 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t632 mm8 a=r2 b=r0 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t633 mm8 a=r2 b=r1 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t634 mm8 a=r1 b=r1 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t635 mm8 a=r6 b=r7 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t636 mm8 a=r2 b=r5 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t637 mm8 a=r5 b=r7 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t638 mm8 a=r1 b=r4 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t639 mm8 a=r1 b=r7 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t640 mm8 a=r0 b=r4 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t641 mm8 a=r5 b=r6 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t642 mm8 a=r7 b=r5 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t643 mm8 a=r3 b=r5 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t644 mm8 a=r1 b=r6 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t645 mm8 a=r0 b=r7 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t646 mm8 a=r5 b=r6 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t647 mm8 a=r6 b=r4 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t648 mm8 a=r3 b=r1 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t649 mm8 a=r7 b=r2 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t650 mm8 a=r7 b=r7 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t651 mm8 a=r6 b=r3 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t652 mm8 a=r1 b=r6 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t653 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t654 mm8 a=r1 b=r6 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t655 mm8 a=r7 b=r5 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t656 mm8 a=r6 b=r5 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t657 mm8 a=r7 b=r4 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t658 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t659 mm8 a=r0 b=r7 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t660 mm8 a=r6 b=r1 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t661 mm8 a=r3 b=r7 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t662 mm8 a=r2 b=r3 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t663 mm8 a=r0 b=r3 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t664 mm8 a=r2 b=r0 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t665 mm8 a=r1 b=r1 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t666 mm8 a=r7 b=r1 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t667 mm8 a=r2 b=r0 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t668 mm8 a=r3 b=r1 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t669 mm8 a=r1 b=r4 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t670 mm8 a=r7 b=r7 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t671 mm8 a=r0 b=r3 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t672 mm8 a=r0 b=r1 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t673 mm8 a=r6 b=r2 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t674 mm8 a=r3 b=r3 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t675 mm8 a=r5 b=r6 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t676 mm8 a=r4 b=r3 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t677 mm8 a=r0 b=r1 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t678 mm8 a=r4 b=r7 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t679 mm8 a=r4 b=r7 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t680 mm8 a=r5 b=r1 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t681 mm8 a=r4 b=r2 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t682 mm8 a=r3 b=r6 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t683 mm8 a=r0 b=r5 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t684 mm8 a=r6 b=r2 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t685 mm8 a=r7 b=r3 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t686 mm8 a=r4 b=r1 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t687 mm8 a=r0 b=r5 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t688 mm8 a=r3 b=r6 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t689 mm8 a=r3 b=r1 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t690 mm8 a=r2 b=r6 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t691 mm8 a=r2 b=r6 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t692 mm8 a=r1 b=r4 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t693 mm8 a=r7 b=r1 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t694 mm8 a=r3 b=r5 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t695 mm8 a=r3 b=r2 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t696 mm8 a=r2 b=r4 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t697 mm8 a=r2 b=r3 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t698 mm8 a=r3 b=r0 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t699 mm8 a=r2 b=r1 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t700 mm8 a=r0 b=r1 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t701 mm8 a=r5 b=r7 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t702 mm8 a=r5 b=r5 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t703 mm8 a=r2 b=r2 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t704 mm8 a=r6 b=r5 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t705 mm8 a=r5 b=r4 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t706 mm8 a=r6 b=r0 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t707 mm8 a=r6 b=r4 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t708 mm8 a=r2 b=r0 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t709 mm8 a=r7 b=r0 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t710 mm8 a=r4 b=r2 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t711 mm8 a=r4 b=r7 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t712 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t713 mm8 a=r5 b=r2 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t714 mm8 a=r0 b=r0 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t715 mm8 a=r1 b=r7 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t716 mm8 a=r3 b=r3 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t717 mm8 a=r3 b=r3 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t718 mm8 a=r7 b=r5 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t719 mm8 a=r4 b=r7 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t720 mm8 a=r4 b=r7 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t721 mm8 a=r6 b=r4 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t722 mm8 a=r7 b=r5 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t723 mm8 a=r5 b=r0 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t724 mm8 a=r7 b=r4 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t725 mm8 a=r6 b=r1 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t726 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t727 mm8 a=r0 b=r6 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t728 mm8 a=r5 b=r7 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t729 mm8 a=r5 b=r6 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t730 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t731 mm8 a=r5 b=r6 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t732 mm8 a=r5 b=r1 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t733 mm8 a=r3 b=r4 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t734 mm8 a=r4 b=r3 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t735 mm8 a=r7 b=r4 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t736 mm8 a=r4 b=r2 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t737 mm8 a=r4 b=r7 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t738 mm8 a=r6 b=r2 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t739 mm8 a=r7 b=r5 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t740 mm8 a=r4 b=r3 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t741 mm8 a=r7 b=r0 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t742 mm8 a=r6 b=r5 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t743 mm8 a=r5 b=r3 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t744 mm8 a=r7 b=r4 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t745 mm8 a=r7 b=r0 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t746 mm8 a=r2 b=r5 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t747 mm8 a=r0 b=r3 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t748 mm8 a=r1 b=r6 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t749 mm8 a=r2 b=r5 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t750 mm8 a=r4 b=r3 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t751 mm8 a=r6 b=r6 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t752 mm8 a=r5 b=r0 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t753 mm8 a=r6 b=r7 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t754 mm8 a=r2 b=r4 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t755 mm8 a=r5 b=r6 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t756 mm8 a=r7 b=r3 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t757 mm8 a=r6 b=r6 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t758 mm8 a=r6 b=r2 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t759 mm8 a=r2 b=r7 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t760 mm8 a=r7 b=r7 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t761 mm8 a=r3 b=r3 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t762 mm8 a=r5 b=r2 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t763 mm8 a=r1 b=r1 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t764 mm8 a=r1 b=r2 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t765 mm8 a=r2 b=r4 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t766 mm8 a=r7 b=r6 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t767 mm8 a=r7 b=r5 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t768 mm8 a=r5 b=r6 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t769 mm8 a=r4 b=r5 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t770 mm8 a=r7 b=r5 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t771 mm8 a=r2 b=r5 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t772 mm8 a=r0 b=r4 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t773 mm8 a=r2 b=r5 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t774 mm8 a=r4 b=r0 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t775 mm8 a=r4 b=r6 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t776 mm8 a=r5 b=r7 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t777 mm8 a=r1 b=r1 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t778 mm8 a=r1 b=r2 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t779 mm8 a=r7 b=r5 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t780 mm8 a=r3 b=r7 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t781 mm8 a=r6 b=r7 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t782 mm8 a=r6 b=r4 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t783 mm8 a=r6 b=r5 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t784 mm8 a=r1 b=r5 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t785 mm8 a=r7 b=r2 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t786 mm8 a=r2 b=r4 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t787 mm8 a=r7 b=r2 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t788 mm8 a=r0 b=r0 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t789 mm8 a=r1 b=r4 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t790 mm8 a=r3 b=r1 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t791 mm8 a=r3 b=r5 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t792 mm8 a=r7 b=r1 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t793 mm8 a=r7 b=r2 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t794 mm8 a=r2 b=r4 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t795 mm8 a=r1 b=r7 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t796 mm8 a=r2 b=r4 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t797 mm8 a=r6 b=r2 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t798 mm8 a=r1 b=r3 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t799 mm8 a=r1 b=r3 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t800 mm8 a=r0 b=r4 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t801 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t802 mm8 a=r4 b=r7 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t803 mm8 a=r3 b=r2 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t804 mm8 a=r0 b=r4 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t805 mm8 a=r4 b=r6 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t806 mm8 a=r0 b=r5 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t807 mm8 a=r5 b=r0 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t808 mm8 a=r2 b=r4 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t809 mm8 a=r1 b=r6 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t810 mm8 a=r1 b=r2 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t811 mm8 a=r2 b=r7 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t812 mm8 a=r0 b=r6 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t813 mm8 a=r2 b=r2 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t814 mm8 a=r3 b=r4 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t815 mm8 a=r4 b=r5 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t816 mm8 a=r3 b=r0 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t817 mm8 a=r3 b=r4 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t818 mm8 a=r2 b=r2 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t819 mm8 a=r1 b=r2 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t820 mm8 a=r3 b=r6 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t821 mm8 a=r7 b=r3 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t822 mm8 a=r1 b=r5 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t823 mm8 a=r1 b=r4 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t824 mm8 a=r7 b=r7 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t825 mm8 a=r1 b=r1 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t826 mm8 a=r0 b=r2 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t827 mm8 a=r3 b=r1 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t828 mm8 a=r7 b=r4 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t829 mm8 a=r5 b=r2 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t830 mm8 a=r6 b=r3 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t831 mm8 a=r2 b=r7 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t832 mm8 a=r1 b=r6 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t833 mm8 a=r0 b=r6 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t834 mm8 a=r2 b=r4 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t835 mm8 a=r0 b=r7 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t836 mm8 a=r3 b=r0 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t837 mm8 a=r4 b=r7 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t838 mm8 a=r5 b=r4 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t839 mm8 a=r0 b=r6 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t840 mm8 a=r1 b=r6 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t841 mm8 a=r4 b=r0 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t842 mm8 a=r3 b=r5 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t843 mm8 a=r2 b=r2 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t844 mm8 a=r3 b=r3 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t845 mm8 a=r1 b=r7 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t846 mm8 a=r1 b=r4 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t847 mm8 a=r4 b=r0 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t848 mm8 a=r6 b=r7 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t849 mm8 a=r6 b=r6 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t850 mm8 a=r6 b=r0 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t851 mm8 a=r0 b=r5 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t852 mm8 a=r2 b=r2 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t853 mm8 a=r6 b=r5 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t854 mm8 a=r1 b=r4 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t855 mm8 a=r1 b=r7 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t856 mm8 a=r2 b=r3 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t857 mm8 a=r6 b=r5 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t858 mm8 a=r0 b=r5 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t859 mm8 a=r2 b=r6 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t860 mm8 a=r7 b=r2 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t861 mm8 a=r3 b=r3 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t862 mm8 a=r0 b=r1 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t863 mm8 a=r0 b=r0 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t864 mm8 a=r5 b=r5 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t865 mm8 a=r3 b=r7 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t866 mm8 a=r6 b=r6 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t867 mm8 a=r7 b=r3 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t868 mm8 a=r4 b=r4 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t869 mm8 a=r3 b=r6 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t870 mm8 a=r0 b=r1 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t871 mm8 a=r5 b=r1 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t872 mm8 a=r0 b=r6 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t873 mm8 a=r2 b=r1 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t874 mm8 a=r5 b=r1 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t875 mm8 a=r1 b=r7 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t876 mm8 a=r7 b=r6 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t877 mm8 a=r6 b=r2 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t878 mm8 a=r5 b=r0 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t879 mm8 a=r4 b=r1 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t880 mm8 a=r3 b=r3 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t881 mm8 a=r2 b=r7 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t882 mm8 a=r7 b=r5 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t883 mm8 a=r2 b=r3 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t884 mm8 a=r2 b=r7 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t885 mm8 a=r3 b=r5 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t886 mm8 a=r5 b=r4 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t887 mm8 a=r1 b=r1 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t888 mm8 a=r6 b=r2 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t889 mm8 a=r4 b=r6 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t890 mm8 a=r0 b=r5 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t891 mm8 a=r0 b=r7 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t892 mm8 a=r5 b=r6 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t893 mm8 a=r2 b=r1 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t894 mm8 a=r0 b=r7 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t895 mm8 a=r2 b=r4 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t896 mm8 a=r0 b=r2 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t897 mm8 a=r7 b=r4 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t898 mm8 a=r1 b=r3 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t899 mm8 a=r5 b=r1 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t900 mm8 a=r7 b=r7 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t901 mm8 a=r4 b=r6 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t902 mm8 a=r2 b=r1 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t903 mm8 a=r0 b=r3 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t904 mm8 a=r6 b=r7 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t905 mm8 a=r0 b=r4 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t906 mm8 a=r3 b=r7 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t907 mm8 a=r2 b=r4 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t908 mm8 a=r7 b=r1 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t909 mm8 a=r4 b=r5 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t910 mm8 a=r3 b=r7 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t911 mm8 a=r2 b=r6 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t912 mm8 a=r6 b=r1 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t913 mm8 a=r2 b=r3 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t914 mm8 a=r7 b=r4 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t915 mm8 a=r4 b=r3 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t916 mm8 a=r4 b=r2 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t917 mm8 a=r5 b=r5 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t918 mm8 a=r7 b=r2 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t919 mm8 a=r3 b=r4 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t920 mm8 a=r6 b=r5 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t921 mm8 a=r6 b=r5 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t922 mm8 a=r1 b=r0 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t923 mm8 a=r6 b=r3 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t924 mm8 a=r2 b=r0 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t925 mm8 a=r6 b=r6 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t926 mm8 a=r5 b=r0 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t927 mm8 a=r1 b=r1 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t928 mm8 a=r5 b=r7 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t929 mm8 a=r1 b=r0 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t930 mm8 a=r7 b=r7 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t931 mm8 a=r7 b=r0 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t932 mm8 a=r1 b=r3 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t933 mm8 a=r1 b=r3 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t934 mm8 a=r6 b=r0 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t935 mm8 a=r5 b=r0 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t936 mm8 a=r3 b=r3 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t937 mm8 a=r0 b=r6 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t938 mm8 a=r1 b=r5 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t939 mm8 a=r7 b=r0 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t940 mm8 a=r1 b=r2 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t941 mm8 a=r1 b=r6 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t942 mm8 a=r0 b=r6 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t943 mm8 a=r4 b=r0 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t944 mm8 a=r6 b=r5 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t945 mm8 a=r3 b=r7 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t946 mm8 a=r2 b=r6 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t947 mm8 a=r7 b=r2 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t948 mm8 a=r6 b=r4 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t949 mm8 a=r6 b=r6 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t950 mm8 a=r7 b=r6 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t951 mm8 a=r2 b=r7 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t952 mm8 a=r3 b=r2 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t953 mm8 a=r4 b=r4 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t954 mm8 a=r2 b=r4 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t955 mm8 a=r0 b=r7 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t956 mm8 a=r7 b=r1 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t957 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t958 mm8 a=r6 b=r5 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t959 mm8 a=r4 b=r7 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t960 mm8 a=r1 b=r1 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t961 mm8 a=r1 b=r5 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t962 mm8 a=r6 b=r4 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t963 mm8 a=r7 b=r1 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t964 mm8 a=r2 b=r5 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t965 mm8 a=r3 b=r7 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t966 mm8 a=r7 b=r3 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t967 mm8 a=r0 b=r0 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t968 mm8 a=r5 b=r7 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t969 mm8 a=r4 b=r4 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t970 mm8 a=r4 b=r3 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t971 mm8 a=r3 b=r4 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t972 mm8 a=r1 b=r7 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t973 mm8 a=r4 b=r1 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t974 mm8 a=r5 b=r6 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t975 mm8 a=r1 b=r2 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t976 mm8 a=r0 b=r6 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t977 mm8 a=r2 b=r0 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t978 mm8 a=r0 b=r2 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t979 mm8 a=r4 b=r0 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t980 mm8 a=r3 b=r1 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t981 mm8 a=r2 b=r0 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t982 mm8 a=r1 b=r6 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t983 mm8 a=r2 b=r1 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t984 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t985 mm8 a=r4 b=r3 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t986 mm8 a=r2 b=r7 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t987 mm8 a=r5 b=r1 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t988 mm8 a=r6 b=r0 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t989 mm8 a=r2 b=r3 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t990 mm8 a=r1 b=r5 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t991 mm8 a=r6 b=r0 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t992 mm8 a=r2 b=r1 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t993 mm8 a=r6 b=r0 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t994 mm8 a=r6 b=r0 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t995 mm8 a=r0 b=r3 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t996 mm8 a=r7 b=r0 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t997 mm8 a=r5 b=r5 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t998 mm8 a=r3 b=r4 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t999 mm8 a=r7 b=r2 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t1000 mm8 a=r5 b=r3 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t1001 mm8 a=r5 b=r6 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t1002 mm8 a=r6 b=r2 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t1003 mm8 a=r4 b=r2 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t1004 mm8 a=r7 b=r3 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t1005 mm8 a=r1 b=r7 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t1006 mm8 a=r4 b=r2 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t1007 mm8 a=r0 b=r2 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t1008 mm8 a=r3 b=r5 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t1009 mm8 a=r0 b=r7 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t1010 mm8 a=r6 b=r7 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t1011 mm8 a=r7 b=r5 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t1012 mm8 a=r3 b=r5 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t1013 mm8 a=r6 b=r6 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t1014 mm8 a=r7 b=r3 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t1015 mm8 a=r7 b=r6 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t1016 mm8 a=r4 b=r0 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t1017 mm8 a=r5 b=r2 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t1018 mm8 a=r7 b=r6 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t1019 mm8 a=r0 b=r7 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t1020 mm8 a=r5 b=r6 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t1021 mm8 a=r7 b=r6 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t1022 mm8 a=r3 b=r5 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t1023 mm8 a=r0 b=r3 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t1024 mm8 a=r2 b=r6 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t1025 mm8 a=r4 b=r7 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t1026 mm8 a=r0 b=r1 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t1027 mm8 a=r1 b=r5 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t1028 mm8 a=r7 b=r6 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t1029 mm8 a=r2 b=r1 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t1030 mm8 a=r1 b=r7 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t1031 mm8 a=r3 b=r7 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t1032 mm8 a=r2 b=r1 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t1033 mm8 a=r2 b=r7 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t1034 mm8 a=r4 b=r3 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t1035 mm8 a=r3 b=r0 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t1036 mm8 a=r3 b=r0 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t1037 mm8 a=r7 b=r1 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t1038 mm8 a=r2 b=r1 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t1039 mm8 a=r5 b=r1 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t1040 mm8 a=r0 b=r7 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t1041 mm8 a=r2 b=r2 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t1042 mm8 a=r6 b=r6 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t1043 mm8 a=r6 b=r7 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t1044 mm8 a=r4 b=r2 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t1045 mm8 a=r0 b=r0 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t1046 mm8 a=r7 b=r1 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t1047 mm8 a=r5 b=r7 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t1048 mm8 a=r2 b=r3 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t1049 mm8 a=r5 b=r1 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t1050 mm8 a=r0 b=r5 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t1051 mm8 a=r7 b=r7 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t1052 mm8 a=r2 b=r3 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t1053 mm8 a=r6 b=r0 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t1054 mm8 a=r4 b=r2 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t1055 mm8 a=r5 b=r3 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t1056 mm8 a=r5 b=r1 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t1057 mm8 a=r1 b=r5 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t1058 mm8 a=r0 b=r5 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t1059 mm8 a=r5 b=r6 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t1060 mm8 a=r7 b=r3 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t1061 mm8 a=r5 b=r6 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t1062 mm8 a=r3 b=r0 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t1063 mm8 a=r1 b=r1 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t1064 mm8 a=r5 b=r4 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t1065 mm8 a=r3 b=r1 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t1066 mm8 a=r7 b=r7 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t1067 mm8 a=r0 b=r5 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t1068 mm8 a=r4 b=r6 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t1069 mm8 a=r2 b=r0 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t1070 mm8 a=r2 b=r3 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t1071 mm8 a=r2 b=r6 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t1072 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t1073 mm8 a=r3 b=r6 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t1074 mm8 a=r0 b=r0 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t1075 mm8 a=r4 b=r4 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t1076 mm8 a=r1 b=r7 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t1077 mm8 a=r7 b=r7 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t1078 mm8 a=r3 b=r7 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t1079 mm8 a=r6 b=r6 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t1080 mm8 a=r4 b=r5 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t1081 mm8 a=r1 b=r5 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t1082 mm8 a=r5 b=r4 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t1083 mm8 a=r2 b=r2 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t1084 mm8 a=r2 b=r0 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t1085 mm8 a=r3 b=r3 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t1086 mm8 a=r3 b=r1 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t1087 mm8 a=r1 b=r3 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t1088 mm8 a=r3 b=r5 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t1089 mm8 a=r0 b=r0 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t1090 mm8 a=r4 b=r6 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t1091 mm8 a=r4 b=r3 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t1092 mm8 a=r2 b=r6 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t1093 mm8 a=r5 b=r7 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t1094 mm8 a=r7 b=r3 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t1095 mm8 a=r0 b=r2 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t1096 mm8 a=r3 b=r3 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t1097 mm8 a=r6 b=r2 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t1098 mm8 a=r3 b=r6 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t1099 mm8 a=r2 b=r5 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t1100 mm8 a=r1 b=r4 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t1101 mm8 a=r0 b=r6 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t1102 mm8 a=r0 b=r5 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t1103 mm8 a=r7 b=r0 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t1104 mm8 a=r4 b=r2 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t1105 mm8 a=r3 b=r4 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t1106 mm8 a=r2 b=r1 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t1107 mm8 a=r3 b=r0 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t1108 mm8 a=r3 b=r5 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t1109 mm8 a=r6 b=r2 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t1110 mm8 a=r2 b=r4 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t1111 mm8 a=r0 b=r7 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t1112 mm8 a=r1 b=r0 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t1113 mm8 a=r4 b=r6 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t1114 mm8 a=r0 b=r5 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t1115 mm8 a=r4 b=r5 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t1116 mm8 a=r4 b=r6 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t1117 mm8 a=r0 b=r6 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t1118 mm8 a=r7 b=r6 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t1119 mm8 a=r2 b=r7 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t1120 mm8 a=r7 b=r5 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t1121 mm8 a=r4 b=r0 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t1122 mm8 a=r0 b=r5 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t1123 mm8 a=r0 b=r6 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t1124 mm8 a=r0 b=r1 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t1125 mm8 a=r6 b=r6 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t1126 mm8 a=r0 b=r0 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t1127 mm8 a=r1 b=r1 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t1128 mm8 a=r4 b=r1 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t1129 mm8 a=r0 b=r0 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t1130 mm8 a=r0 b=r4 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t1131 mm8 a=r3 b=r0 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t1132 mm8 a=r0 b=r6 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t1133 mm8 a=r6 b=r0 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t1134 mm8 a=r7 b=r3 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t1135 mm8 a=r6 b=r7 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t1136 mm8 a=r7 b=r4 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t1137 mm8 a=r4 b=r0 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t1138 mm8 a=r6 b=r4 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t1139 mm8 a=r6 b=r2 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t1140 mm8 a=r1 b=r2 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t1141 mm8 a=r0 b=r5 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t1142 mm8 a=r4 b=r5 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t1143 mm8 a=r5 b=r6 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t1144 mm8 a=r6 b=r4 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t1145 mm8 a=r5 b=r1 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t1146 mm8 a=r6 b=r5 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t1147 mm8 a=r7 b=r1 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t1148 mm8 a=r1 b=r6 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t1149 mm8 a=r5 b=r5 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t1150 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t1151 mm8 a=r7 b=r5 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t1152 mm8 a=r0 b=r6 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t1153 mm8 a=r2 b=r5 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t1154 mm8 a=r5 b=r3 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t1155 mm8 a=r0 b=r6 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t1156 mm8 a=r2 b=r6 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t1157 mm8 a=r6 b=r7 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t1158 mm8 a=r2 b=r3 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t1159 mm8 a=r1 b=r5 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t1160 mm8 a=r5 b=r0 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t1161 mm8 a=r7 b=r0 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t1162 mm8 a=r5 b=r3 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t1163 mm8 a=r4 b=r3 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t1164 mm8 a=r3 b=r4 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t1165 mm8 a=r1 b=r2 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t1166 mm8 a=r4 b=r7 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t1167 mm8 a=r2 b=r7 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t1168 mm8 a=r2 b=r6 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t1169 mm8 a=r0 b=r4 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t1170 mm8 a=r2 b=r1 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t1171 mm8 a=r1 b=r7 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t1172 mm8 a=r0 b=r1 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t1173 mm8 a=r6 b=r4 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t1174 mm8 a=r0 b=r1 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t1175 mm8 a=r0 b=r0 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t1176 mm8 a=r4 b=r5 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t1177 mm8 a=r7 b=r2 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t1178 mm8 a=r1 b=r5 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t1179 mm8 a=r0 b=r7 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t1180 mm8 a=r7 b=r3 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t1181 mm8 a=r7 b=r3 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t1182 mm8 a=r1 b=r2 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t1183 mm8 a=r2 b=r4 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t1184 mm8 a=r7 b=r0 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t1185 mm8 a=r5 b=r2 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t1186 mm8 a=r1 b=r0 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t1187 mm8 a=r5 b=r4 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t1188 mm8 a=r3 b=r0 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t1189 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t1190 mm8 a=r5 b=r0 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t1191 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t1192 mm8 a=r0 b=r2 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t1193 mm8 a=r3 b=r7 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t1194 mm8 a=r7 b=r2 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t1195 mm8 a=r1 b=r4 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t1196 mm8 a=r4 b=r6 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t1197 mm8 a=r5 b=r6 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t1198 mm8 a=r0 b=r6 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t1199 mm8 a=r5 b=r1 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t1200 mm8 a=r1 b=r3 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t1201 mm8 a=r6 b=r6 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t1202 mm8 a=r2 b=r4 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t1203 mm8 a=r0 b=r5 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t1204 mm8 a=r0 b=r2 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t1205 mm8 a=r4 b=r6 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t1206 mm8 a=r4 b=r6 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t1207 mm8 a=r3 b=r7 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t1208 mm8 a=r6 b=r1 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t1209 mm8 a=r5 b=r5 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t1210 mm8 a=r0 b=r0 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t1211 mm8 a=r7 b=r5 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t1212 mm8 a=r2 b=r6 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t1213 mm8 a=r6 b=r2 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t1214 mm8 a=r4 b=r2 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t1215 mm8 a=r7 b=r2 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t1216 mm8 a=r3 b=r3 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t1217 mm8 a=r5 b=r5 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t1218 mm8 a=r0 b=r7 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t1219 mm8 a=r2 b=r4 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t1220 mm8 a=r1 b=r7 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t1221 mm8 a=r5 b=r5 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t1222 mm8 a=r1 b=r3 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t1223 mm8 a=r0 b=r2 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t1224 mm8 a=r1 b=r1 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t1225 mm8 a=r4 b=r1 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t1226 mm8 a=r5 b=r6 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t1227 mm8 a=r1 b=r7 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t1228 mm8 a=r4 b=r2 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t1229 mm8 a=r5 b=r3 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t1230 mm8 a=r6 b=r2 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t1231 mm8 a=r4 b=r3 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t1232 mm8 a=r6 b=r7 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t1233 mm8 a=r0 b=r5 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t1234 mm8 a=r5 b=r4 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t1235 mm8 a=r5 b=r7 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t1236 mm8 a=r0 b=r7 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t1237 mm8 a=r7 b=r2 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t1238 mm8 a=r7 b=r0 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t1239 mm8 a=r3 b=r2 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t1240 mm8 a=r0 b=r5 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t1241 mm8 a=r5 b=r0 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t1242 mm8 a=r5 b=r7 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t1243 mm8 a=r6 b=r3 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t1244 mm8 a=r1 b=r5 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t1245 mm8 a=r0 b=r6 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t1246 mm8 a=r5 b=r5 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t1247 mm8 a=r1 b=r6 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t1248 mm8 a=r4 b=r0 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t1249 mm8 a=r7 b=r6 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t1250 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t1251 mm8 a=r3 b=r7 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t1252 mm8 a=r5 b=r5 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t1253 mm8 a=r5 b=r5 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t1254 mm8 a=r0 b=r6 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t1255 mm8 a=r1 b=r4 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t1256 mm8 a=r1 b=r3 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t1257 mm8 a=r0 b=r7 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t1258 mm8 a=r5 b=r4 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t1259 mm8 a=r0 b=r3 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t1260 mm8 a=r1 b=r5 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t1261 mm8 a=r6 b=r7 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t1262 mm8 a=r2 b=r5 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t1263 mm8 a=r3 b=r7 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t1264 mm8 a=r1 b=r6 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t1265 mm8 a=r2 b=r3 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t1266 mm8 a=r4 b=r6 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t1267 mm8 a=r6 b=r6 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t1268 mm8 a=r4 b=r5 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t1269 mm8 a=r5 b=r7 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t1270 mm8 a=r7 b=r3 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t1271 mm8 a=r7 b=r0 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t1272 mm8 a=r1 b=r3 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t1273 mm8 a=r3 b=r2 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t1274 mm8 a=r2 b=r0 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t1275 mm8 a=r7 b=r3 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t1276 mm8 a=r4 b=r3 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t1277 mm8 a=r5 b=r4 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t1278 mm8 a=r6 b=r3 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t1279 mm8 a=r0 b=r1 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t1280 mm8 a=r4 b=r1 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t1281 mm8 a=r6 b=r3 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t1282 mm8 a=r1 b=r7 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t1283 mm8 a=r6 b=r2 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t1284 mm8 a=r1 b=r6 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t1285 mm8 a=r3 b=r5 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t1286 mm8 a=r3 b=r1 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t1287 mm8 a=r4 b=r1 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t1288 mm8 a=r3 b=r6 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t1289 mm8 a=r6 b=r2 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t1290 mm8 a=r6 b=r5 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t1291 mm8 a=r5 b=r6 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t1292 mm8 a=r6 b=r6 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t1293 mm8 a=r6 b=r2 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t1294 mm8 a=r0 b=r5 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t1295 mm8 a=r5 b=r0 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t1296 mm8 a=r1 b=r0 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t1297 mm8 a=r0 b=r0 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t1298 mm8 a=r2 b=r6 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t1299 mm8 a=r7 b=r0 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t1300 mm8 a=r4 b=r1 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t1301 mm8 a=r4 b=r5 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t1302 mm8 a=r0 b=r4 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t1303 mm8 a=r4 b=r2 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t1304 mm8 a=r4 b=r7 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t1305 mm8 a=r3 b=r2 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t1306 mm8 a=r3 b=r0 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t1307 mm8 a=r0 b=r2 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t1308 mm8 a=r2 b=r5 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t1309 mm8 a=r5 b=r7 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t1310 mm8 a=r4 b=r5 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t1311 mm8 a=r4 b=r7 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t1312 mm8 a=r3 b=r7 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t1313 mm8 a=r5 b=r1 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t1314 mm8 a=r5 b=r2 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t1315 mm8 a=r1 b=r0 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t1316 mm8 a=r7 b=r3 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t1317 mm8 a=r2 b=r5 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t1318 mm8 a=r2 b=r3 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t1319 mm8 a=r7 b=r5 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t1320 mm8 a=r0 b=r3 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t1321 mm8 a=r1 b=r0 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t1322 mm8 a=r4 b=r0 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t1323 mm8 a=r5 b=r3 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t1324 mm8 a=r3 b=r0 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t1325 mm8 a=r6 b=r4 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t1326 mm8 a=r4 b=r6 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t1327 mm8 a=r1 b=r3 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t1328 mm8 a=r7 b=r3 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t1329 mm8 a=r3 b=r7 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t1330 mm8 a=r3 b=r6 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t1331 mm8 a=r0 b=r7 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t1332 mm8 a=r0 b=r7 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t1333 mm8 a=r3 b=r3 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t1334 mm8 a=r7 b=r5 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t1335 mm8 a=r5 b=r7 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t1336 mm8 a=r5 b=r5 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t1337 mm8 a=r6 b=r0 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t1338 mm8 a=r6 b=r4 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t1339 mm8 a=r4 b=r6 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t1340 mm8 a=r6 b=r1 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t1341 mm8 a=r0 b=r6 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t1342 mm8 a=r4 b=r6 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t1343 mm8 a=r5 b=r5 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t1344 mm8 a=r2 b=r6 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t1345 mm8 a=r7 b=r3 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t1346 mm8 a=r5 b=r0 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t1347 mm8 a=r2 b=r3 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t1348 mm8 a=r4 b=r3 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t1349 mm8 a=r0 b=r1 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t1350 mm8 a=r1 b=r6 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t1351 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t1352 mm8 a=r5 b=r4 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t1353 mm8 a=r2 b=r7 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t1354 mm8 a=r7 b=r4 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t1355 mm8 a=r7 b=r4 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t1356 mm8 a=r4 b=r6 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t1357 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t1358 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t1359 mm8 a=r7 b=r7 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t1360 mm8 a=r0 b=r4 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t1361 mm8 a=r0 b=r5 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t1362 mm8 a=r0 b=r2 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t1363 mm8 a=r4 b=r4 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t1364 mm8 a=r7 b=r3 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t1365 mm8 a=r2 b=r4 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t1366 mm8 a=r4 b=r3 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t1367 mm8 a=r3 b=r3 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t1368 mm8 a=r4 b=r0 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t1369 mm8 a=r5 b=r3 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t1370 mm8 a=r0 b=r1 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t1371 mm8 a=r0 b=r5 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t1372 mm8 a=r5 b=r6 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t1373 mm8 a=r7 b=r1 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t1374 mm8 a=r4 b=r6 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t1375 mm8 a=r5 b=r2 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t1376 mm8 a=r5 b=r5 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t1377 mm8 a=r7 b=r3 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t1378 mm8 a=r4 b=r1 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t1379 mm8 a=r4 b=r0 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t1380 mm8 a=r0 b=r7 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t1381 mm8 a=r2 b=r0 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t1382 mm8 a=r7 b=r5 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t1383 mm8 a=r1 b=r5 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t1384 mm8 a=r5 b=r2 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t1385 mm8 a=r0 b=r7 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t1386 mm8 a=r7 b=r0 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t1387 mm8 a=r4 b=r3 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t1388 mm8 a=r4 b=r6 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t1389 mm8 a=r4 b=r2 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t1390 mm8 a=r5 b=r1 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t1391 mm8 a=r6 b=r1 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t1392 mm8 a=r5 b=r7 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t1393 mm8 a=r3 b=r4 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t1394 mm8 a=r4 b=r7 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t1395 mm8 a=r7 b=r1 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t1396 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t1397 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t1398 mm8 a=r3 b=r0 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t1399 mm8 a=r3 b=r4 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t1400 mm8 a=r6 b=r4 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t1401 mm8 a=r0 b=r5 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t1402 mm8 a=r7 b=r5 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t1403 mm8 a=r0 b=r0 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t1404 mm8 a=r3 b=r6 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t1405 mm8 a=r0 b=r0 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t1406 mm8 a=r5 b=r3 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t1407 mm8 a=r7 b=r3 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t1408 mm8 a=r4 b=r7 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t1409 mm8 a=r4 b=r2 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t1410 mm8 a=r6 b=r0 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t1411 mm8 a=r1 b=r7 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t1412 mm8 a=r4 b=r3 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t1413 mm8 a=r4 b=r6 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t1414 mm8 a=r0 b=r7 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t1415 mm8 a=r3 b=r6 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t1416 mm8 a=r3 b=r2 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t1417 mm8 a=r2 b=r7 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t1418 mm8 a=r0 b=r6 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t1419 mm8 a=r0 b=r3 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t1420 mm8 a=r1 b=r3 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t1421 mm8 a=r4 b=r0 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t1422 mm8 a=r6 b=r4 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t1423 mm8 a=r4 b=r6 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t1424 mm8 a=r6 b=r3 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t1425 mm8 a=r7 b=r3 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t1426 mm8 a=r0 b=r3 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t1427 mm8 a=r5 b=r6 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t1428 mm8 a=r0 b=r3 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t1429 mm8 a=r7 b=r1 c=r3 c2=r7 - } - uint32_t lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint32_t hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((uint64_t)hi << 32) | (uint64_t)lo; -} - -// Host-side launch wrappers. Declared in program.h, called from host.cu. -cudaError_t igneum_launch_cache_fill(uint32_t* cache, uint32_t nSegments) { - if (nSegments == 0u) return cudaErrorInvalidValue; - uint32_t block = 256u; - uint32_t grid = (nSegments + block - 1u) / block; - igneum_cache_fill<<>>(cache, nSegments); - return cudaGetLastError(); -} - -cudaError_t igneum_launch_build(uint32_t* ds, const uint32_t* cache, uint32_t nItems) { - if (nItems == 0u) return cudaErrorInvalidValue; - uint32_t block = 256u; - uint32_t grid = (nItems + block - 1u) / block; - igneum_build<<>>(ds, cache, nItems); - return cudaGetLastError(); -} - -cudaError_t igneum_launch_hash(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask, - uint32_t nonces, uint32_t blockWarps) { - if (blockWarps == 0u || blockWarps > 32u) return cudaErrorInvalidValue; - uint32_t block = 32u * blockWarps; - if (nonces == 0u || (nonces % block) != 0u) return cudaErrorInvalidValue; - igneum_hash<<>>(ds, out, baseNonce, mask); - return cudaGetLastError(); -} - -cudaError_t igneum_hash_info(int* numRegs, int* blocksPerSM, uint32_t blockWarps) { - cudaFuncAttributes attr; - cudaError_t e = cudaFuncGetAttributes(&attr, igneum_hash); - if (e != cudaSuccess) return e; - *numRegs = attr.numRegs; - return cudaOccupancyMaxActiveBlocksPerMultiprocessor(blocksPerSM, igneum_hash, (int)(32u * blockWarps), 0); -} diff --git a/proto-cuda/packs-ca4/mx8_mm1430/kernel_bound.cl b/proto-cuda/packs-ca4/mx8_mm1430/kernel_bound.cl deleted file mode 100644 index 2da73957e..000000000 --- a/proto-cuda/packs-ca4/mx8_mm1430/kernel_bound.cl +++ /dev/null @@ -1,3243 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// OpenCL C twin of the Metal kernel for the same seed (see proto-opencl/README.md, WAVEFRONT.md and program.metal). -// Built from source at runtime by proto-opencl/host.c, which passes these defines: -// IGNEUM_GROUP work-group size of igneum_hash, a multiple of 32 (default 32: one work-group = one 32-lane unit) -// IGNEUM_EXCHANGE 0 = local-memory exchange with a barrier (any device, any wave width; the default) -// 1 = sub_group_shuffle_xor (cl_khr_subgroup_shuffle), only with IGNEUM_GROUP 32 and a sub-group size of exactly 32 -// 2 = intel_sub_group_shuffle_xor (cl_intel_subgroups), same condition -// The verification unit is always 32 lanes. A 64-wide hardware wave (AMD GCN/CDNA, RDNA in wave64) runs two units; -// the exchange masks are 1, 2, 4, 8, 16, so every partner lane lies inside the lane's own aligned run of 32. -#ifndef IGNEUM_GROUP -#define IGNEUM_GROUP 32 -#endif -#ifndef IGNEUM_EXCHANGE -#define IGNEUM_EXCHANGE 0 -#endif -#ifdef __OPENCL_VERSION__ -#define IGNEUM_KERNEL_HASH __kernel __attribute__((reqd_work_group_size(IGNEUM_GROUP, 1, 1))) -#define IGNEUM_LOCAL_WORDS(name, n) __local uint name[n] -#if IGNEUM_EXCHANGE == 1 -#ifdef cl_khr_subgroups -#pragma OPENCL EXTENSION cl_khr_subgroups : enable -#endif -#ifdef cl_khr_subgroup_shuffle -#pragma OPENCL EXTENSION cl_khr_subgroup_shuffle : enable -#endif -#elif IGNEUM_EXCHANGE == 2 -#pragma OPENCL EXTENSION cl_intel_subgroups : enable -#endif -#else -// Not an OpenCL compiler: proto-opencl/emu compiles this file as C++ and supplies the built-ins and these two macros. -#include "emu_opencl.h" -#endif - -#if IGNEUM_EXCHANGE == 1 -#define IGNEUM_SHFL_XOR(dst, a, m) dst = sub_group_shuffle_xor((a), (uint)(m)) -#define IGNEUM_BCAST0(dst, a) dst = sub_group_broadcast((a), 0u) -#define IGNEUM_SHFL_IDX(dst, a, i) dst = sub_group_shuffle((a), (uint)(i)) -#elif IGNEUM_EXCHANGE == 2 -#define IGNEUM_SHFL_XOR(dst, a, m) dst = intel_sub_group_shuffle_xor((a), (uint)(m)) -#define IGNEUM_BCAST0(dst, a) dst = sub_group_broadcast((a), 0u) -#define IGNEUM_SHFL_IDX(dst, a, i) dst = intel_sub_group_shuffle((a), (uint)(i)) -#else -// Local-memory exchange. Two buffers of IGNEUM_GROUP words alternate (xk counts exchanges), so one barrier per -// exchange is enough: a lane can only overwrite buffer b at exchange k+2 after passing barrier k+1, and every lane -// reaches barrier k+1 only after its read of buffer b at exchange k. The partner lid ^ m stays inside the lane's -// aligned run of 32 because m < 32. Control flow is uniform, so every work-item reaches every barrier. -#define IGNEUM_SHFL_XOR(dst, a, m) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid ^ (uint)(m))]; xk += 1u; } -#define IGNEUM_BCAST0(dst, a) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid & ~31u)]; xk += 1u; } -#define IGNEUM_SHFL_IDX(dst, a, i) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid & ~31u) + (uint)(i)]; xk += 1u; } -#endif - -// int8 tile reference (Counter ASIC 4.0 research): 12 index shuffles and 32 byte products per lane; the PTX mma.m8n8k16 u8 layout. -#define IGNEUM_MM8_REF(av, bv, d0, d1) { uint row_ = lane >> 2, col0_ = 2u * (lane & 3u); uint acc0_ = 0u, acc1_ = 0u; for (uint j_ = 0u; j_ < 4u; ++j_) { uint aw_, bw0_, bw1_; IGNEUM_SHFL_IDX(aw_, (av), 4u * row_ + j_); IGNEUM_SHFL_IDX(bw0_, (bv), 4u * col0_ + j_); IGNEUM_SHFL_IDX(bw1_, (bv), 4u * (col0_ + 1u) + j_); for (uint t_ = 0u; t_ < 4u; ++t_) { uint ab_ = (aw_ >> (8u * t_)) & 0xffu; acc0_ += ab_ * ((bw0_ >> (8u * t_)) & 0xffu); acc1_ += ab_ * ((bw1_ >> (8u * t_)) & 0xffu); } } d0 = acc0_; d1 = acc1_; } - -static inline uint splitmix32(uint x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -// n is a literal in 1..31 at every call site. OpenCL rotate() rotates left by n modulo 32. -static inline uint rotl_imm(uint x, uint n) { return rotate(x, n); } -// Right rotation by n modulo 32 as a left rotation by (32 - n) modulo 32; n == 0 gives x. -static inline uint rotr_var(uint x, uint n) { return rotate(x, (0u - n) & 31u); } -static inline uint ds_elem(uint i, uint d0, uint d1) { - uint x = i ^ d0; - x *= 0x9E3779B1u; x ^= x >> 15; - x += d1; - x *= 0x85EBCA77u; x ^= x >> 13; - x *= 0xC2B2AE3Du; x ^= x >> 16; - return x; -} - -// Memory-hard dataset core (MEMHARD.md). Cache: 2^26 words in 2^16 segments of 64 chained ChaCha12 lines. -// Item: 8 rounds of 8 x seed-parameterised mixer + one 64-byte cache read, then 8 x final mixer (class v3, mixer multiplier 8, -// docs/plans/mixer-x4.md: the round key of application j of round r is 0x9E3779B9 * (r * m + j + 1)). All parameters are literals. -#define MH_CACHE_LINE_MASK 0x003fffffu -#define MH_SEGMENT_LINES 64u -#define MH_QR(a, b, c, d, r1, r2, r3, r4) { a += b; d ^= a; d = mh_rotl(d, r1); c += d; b ^= c; b = mh_rotl(b, r2); a += b; d ^= a; d = mh_rotl(d, r3); c += d; b ^= c; b = mh_rotl(b, r4); } -static inline uint mh_rotl(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 at every call site - -// y = ChaCha12 core(x) + x -static inline void mh_chacha_block(const uint* x, uint* y) { - for (uint i = 0u; i < 16u; ++i) y[i] = x[i]; - for (uint r = 0u; r < 6u; ++r) { - MH_QR(y[0], y[4], y[8], y[12], 16u, 12u, 8u, 7u) MH_QR(y[1], y[5], y[9], y[13], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[6], y[10], y[14], 16u, 12u, 8u, 7u) MH_QR(y[3], y[7], y[11], y[15], 16u, 12u, 8u, 7u) - MH_QR(y[0], y[5], y[10], y[15], 16u, 12u, 8u, 7u) MH_QR(y[1], y[6], y[11], y[12], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[7], y[8], y[13], 16u, 12u, 8u, 7u) MH_QR(y[3], y[4], y[9], y[14], 16u, 12u, 8u, 7u) - } - for (uint i = 0u; i < 16u; ++i) y[i] += x[i]; -} - -// One cache segment: 64 chained lines written at cache[seg * 1024]. in_j = prev ^ (sigma || K || seg || j || tag), prev_0 = 0. -static inline void mh_cache_segment(__global uint* cache, uint seg) { - uint prev[16]; uint x[16]; uint y[16]; - for (uint i = 0u; i < 16u; ++i) prev[i] = 0u; - for (uint j = 0u; j < MH_SEGMENT_LINES; ++j) { - x[0] = 0x61707865u ^ prev[0]; x[1] = 0x3320646eu ^ prev[1]; x[2] = 0x79622d32u ^ prev[2]; x[3] = 0x6b206574u ^ prev[3]; - x[4] = 0x3067619fu ^ prev[4]; - x[5] = 0x3c269176u ^ prev[5]; - x[6] = 0x84a03b03u ^ prev[6]; - x[7] = 0xf8c63294u ^ prev[7]; - x[8] = 0xff977c5bu ^ prev[8]; - x[9] = 0xe60def3eu ^ prev[9]; - x[10] = 0x63630141u ^ prev[10]; - x[11] = 0xb8fbcb58u ^ prev[11]; - x[12] = seg ^ prev[12]; x[13] = j ^ prev[13]; x[14] = 0x49676e65u ^ prev[14]; x[15] = 0x756d4d48u ^ prev[15]; - mh_chacha_block(x, y); - __global uint* line = cache + ((seg * MH_SEGMENT_LINES + j) * 16u); - for (uint i = 0u; i < 16u; ++i) { line[i] = y[i]; prev[i] = y[i]; } - } -} - -// M_r: per word (s ^ (RC + rk)) * MUL, then a column round and a diagonal round with the seed-drawn rotations. -static inline void mh_mixer(uint* s, uint rk) { - s[0] = (s[0] ^ (0xbab68293u + rk)) * 0x42146205u; - s[1] = (s[1] ^ (0xcc162340u + rk)) * 0x52cbe0fbu; - s[2] = (s[2] ^ (0x6ce151ccu + rk)) * 0x7ecf4a03u; - s[3] = (s[3] ^ (0xe62b8997u + rk)) * 0x6728907fu; - s[4] = (s[4] ^ (0xc9c80297u + rk)) * 0xd81d9751u; - s[5] = (s[5] ^ (0xf74a1654u + rk)) * 0x132952c3u; - s[6] = (s[6] ^ (0x3d704af5u + rk)) * 0xf60de277u; - s[7] = (s[7] ^ (0x3cf522b7u + rk)) * 0x05358035u; - s[8] = (s[8] ^ (0x2b9cac04u + rk)) * 0xbaf6499du; - s[9] = (s[9] ^ (0xa880ac10u + rk)) * 0xe4db9667u; - s[10] = (s[10] ^ (0x13e5dd1du + rk)) * 0x3e98f45du; - s[11] = (s[11] ^ (0x6fc3e233u + rk)) * 0xd0004eddu; - s[12] = (s[12] ^ (0x2d83eeacu + rk)) * 0x2691630du; - s[13] = (s[13] ^ (0x9006e8bfu + rk)) * 0x9beb3bcfu; - s[14] = (s[14] ^ (0x2c4b5362u + rk)) * 0xab310379u; - s[15] = (s[15] ^ (0x31b49ee2u + rk)) * 0x99cfb423u; - MH_QR(s[0], s[4], s[8], s[12], 20u, 20u, 19u, 4u) MH_QR(s[1], s[5], s[9], s[13], 20u, 20u, 19u, 4u) - MH_QR(s[2], s[6], s[10], s[14], 20u, 20u, 19u, 4u) MH_QR(s[3], s[7], s[11], s[15], 20u, 20u, 19u, 4u) - MH_QR(s[0], s[5], s[10], s[15], 26u, 3u, 3u, 27u) MH_QR(s[1], s[6], s[11], s[12], 26u, 3u, 3u, 27u) - MH_QR(s[2], s[7], s[8], s[13], 26u, 3u, 3u, 27u) MH_QR(s[3], s[4], s[9], s[14], 26u, 3u, 3u, 27u) -} - -// Item t: 16 words. s = (K, t * MUL[i] + RC[i]); 8 rounds of 8 x mixer + cache line s[0] & mask; 8 x final mixer. -static inline void mh_item(__global const uint* cache, uint t, uint* s) { - s[0] = 0x3067619fu; - s[1] = 0x3c269176u; - s[2] = 0x84a03b03u; - s[3] = 0xf8c63294u; - s[4] = 0xff977c5bu; - s[5] = 0xe60def3eu; - s[6] = 0x63630141u; - s[7] = 0xb8fbcb58u; - s[8] = t * 0x42146205u + 0xbab68293u; - s[9] = t * 0x52cbe0fbu + 0xcc162340u; - s[10] = t * 0x7ecf4a03u + 0x6ce151ccu; - s[11] = t * 0x6728907fu + 0xe62b8997u; - s[12] = t * 0xd81d9751u + 0xc9c80297u; - s[13] = t * 0x132952c3u + 0xf74a1654u; - s[14] = t * 0xf60de277u + 0x3d704af5u; - s[15] = t * 0x05358035u + 0x3cf522b7u; - for (uint r = 0u; r < 8u; ++r) { - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (r * 8u + j + 1u)); - __global const uint* line = cache + ((s[0] & MH_CACHE_LINE_MASK) * 16u); - for (uint i = 0u; i < 16u; ++i) s[i] ^= line[i]; - } - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (64u + j + 1u)); -} -// dataset[w] without the dataset: derive item w >> 4 and take word w & 15. -static inline uint mh_word(__global const uint* cache, uint w) { uint s[16]; mh_item(cache, w >> 4u, s); return s[w & 15u]; } - -// Memory-hard dataset (MEMHARD.md). One work-item per cache segment; one work-item per 64-byte dataset item. -// The same constants as memhard.h in this pack (one emitter, three dialects). -__kernel void igneum_cache_fill(__global uint* cache, uint nSegments) { - uint seg = (uint)get_global_id(0); - if (seg < nSegments) mh_cache_segment(cache, seg); -} -__kernel void igneum_build(__global uint* ds, __global const uint* cache, uint nItems) { - uint t = (uint)get_global_id(0); - if (t < nItems) { - uint s[16]; - mh_item(cache, t, s); - __global uint* d = ds + ((ulong)t * 16u); - for (uint i = 0u; i < 16u; ++i) d[i] = s[i]; - } -} - -// One hash per work-item. IGNEUM_GROUP is a multiple of 32; lane = lid & 31 and every exchange stays inside the -// lane's own aligned run of 32 work-items, exactly like simd_shuffle_xor inside a 32-wide Metal SIMD group and -// __shfl_xor_sync inside a CUDA warp. Control flow is uniform (no branches at all). -IGNEUM_KERNEL_HASH void igneum_hash(__global const uint* ds, __global ulong* out, uint baseNonce, uint mask) { - uint gid = (uint)get_global_id(0); - uint lid = (uint)get_local_id(0); - uint nonce = baseNonce + gid; - uint r0, r1, r2, r3, r4, r5, r6, r7; -#if IGNEUM_EXCHANGE == 0 - IGNEUM_LOCAL_WORDS(xch, 2 * IGNEUM_GROUP); - uint xk = 0u; -#else - (void)lid; -#endif - uint lane = lid & 31u; - { uint x = nonce ^ 0x67a9a7beu; x += 0x9e3779b9u; x = splitmix32(x); r0 = x ^ 0x1a155b25u; } // SEEDW[0], 0x9e3779b9u * 1u, SEEDW[1] - { uint x = nonce ^ 0x1a155b25u; x += 0x3c6ef372u; x = splitmix32(x); r1 = x ^ 0xfddfb732u; } // SEEDW[1], 0x9e3779b9u * 2u, SEEDW[2] - { uint x = nonce ^ 0xfddfb732u; x += 0xdaa66d2bu; x = splitmix32(x); r2 = x ^ 0x4b5af2e8u; } // SEEDW[2], 0x9e3779b9u * 3u, SEEDW[3] - { uint x = nonce ^ 0x4b5af2e8u; x += 0x78dde6e4u; x = splitmix32(x); r3 = x ^ 0xc55caf33u; } // SEEDW[3], 0x9e3779b9u * 4u, SEEDW[4] - { uint x = nonce ^ 0xc55caf33u; x += 0x1715609du; x = splitmix32(x); r4 = x ^ 0xa27c13b7u; } // SEEDW[4], 0x9e3779b9u * 5u, SEEDW[5] - { uint x = nonce ^ 0xa27c13b7u; x += 0xb54cda56u; x = splitmix32(x); r5 = x ^ 0x06628a48u; } // SEEDW[5], 0x9e3779b9u * 6u, SEEDW[6] - { uint x = nonce ^ 0x06628a48u; x += 0x5384540fu; x = splitmix32(x); r6 = x ^ 0x03852469u; } // SEEDW[6], 0x9e3779b9u * 7u, SEEDW[7] - { uint x = nonce ^ 0x03852469u; x += 0xf1bbcdc8u; x = splitmix32(x); r7 = x ^ 0x67a9a7beu; } // SEEDW[7], 0x9e3779b9u * 8u, SEEDW[0] - - for (uint it = 0u; it < 8u; ++it) { - uint sel = r0; - r2 = r3 * r4 + r2; // 0 mad - r2 = r1 * r1 + r2; // 1 mad - r2 = r3 * r2 + r2; // 2 mad - r3 = r3 ^ r5; // 3 xor - r7 = r7 ^ ds[r2 & mask]; // 4 load - r5 = r5 ^ ds[r7 & mask]; // 5 load - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 8u); r1 = r1 ^ t_; } // 6 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 8u); r7 = r7 ^ t_; } // 7 shfl - r1 = mul_hi(r1, r5); // 8 mulhi - r6 = rotr_var(r6, r3); // 9 rotr - r3 = r3 | r4; // 10 or - r4 = r4 ^ ds[r3 & mask]; // 11 load - r0 = mul_hi(r0, r4); // 12 mulhi - r5 = r5 + r1 + ((((sel >> 30u) & 1u) != 0u) ? 0xd3177981u : 0xc7934706u); // 13 add - r0 = r0 ^ ds[r4 & mask]; // 14 load - r2 = r2 - r4; // 15 sub - r2 = r2 ^ ds[r0 & mask]; // 16 load - r7 = r7 ^ ds[r2 & mask]; // 17 load - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 4u); r7 = r7 ^ t_; } // 18 shfl - r5 = r5 * r0; // 19 mul - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 2u); r3 = r3 ^ t_; } // 20 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 16u); r2 = r2 ^ t_; } // 21 shfl - r6 = mul_hi(r6, r2); // 22 mulhi - r6 = r6 ^ ds[r1 & mask]; // 23 load - r5 = r5 * r0; // 24 mul - r5 = rotl_imm(r5, 19u); // 25 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 2u); r7 = r7 ^ t_; } // 26 shfl - r0 = r0 ^ r5; // 27 xor - r0 = r0 ^ r4; // 28 xor - r3 = r3 - r0; // 29 sub - r5 = r5 * r1; // 30 mul - r7 = r7 ^ ds[r2 & mask]; // 31 load - r1 = r1 ^ ds[r0 & mask]; // 32 load - r5 = r5 ^ r6; // 33 xor - r5 = r5 ^ ds[r1 & mask]; // 34 load - r0 = mul_hi(r0, r5); // 35 mulhi - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 4u); r5 = r5 ^ t_; } // 36 shfl - r7 = r7 ^ ds[r0 & mask]; // 37 load - r3 = r3 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0x230c005cu : 0x75ba2fadu); // 38 add - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 4u); r1 = r1 ^ t_; } // 39 shfl - r2 = r2 ^ r5; // 40 xor - r3 = r6 * r3 + r3; // 41 mad - r6 = r6 - r7; // 42 sub - r7 = r7 ^ r0; // 43 xor - r1 = r1 ^ ds[r7 & mask]; // 44 load - r2 = r2 * r3; // 45 mul - r1 = mul_hi(r1, r5); // 46 mulhi - r4 = r4 - r3; // 47 sub - r2 = rotr_var(r2, r6); // 48 rotr - r3 = r3 ^ ds[r5 & mask]; // 49 load - r1 = r1 + r5 + ((((sel >> 7u) & 1u) != 0u) ? 0x1907970cu : 0x81b8bc2cu); // 50 add - r0 = r0 * r2; // 51 mul - r0 = r0 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x699fd448u : 0x4f92b968u); // 52 add - r1 = r1 + r0 + ((((sel >> 12u) & 1u) != 0u) ? 0x77b1520du : 0x2bb965afu); // 53 add - r7 = rotl_imm(r7, 14u); // 54 rotl - r3 = r3 + r7 + ((((sel >> 1u) & 1u) != 0u) ? 0xa54c55a0u : 0x7b0fe07au); // 55 add - r6 = r6 ^ ds[r7 & mask]; // 56 load - r1 = rotr_var(r1, r5); // 57 rotr - r5 = r5 ^ ds[r4 & mask]; // 58 load - r6 = r6 ^ ds[r2 & mask]; // 59 load - r3 = r5 * r0 + r3; // 60 mad - r5 = r5 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0xad7493e7u : 0xaf9dd72du); // 61 add - r4 = r4 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x1e07c3d9u : 0x89841d87u); // 62 add - r5 = rotl_imm(r5, 19u); // 63 rotl - // int8 tile block (Counter ASIC 4.0 research, experimental): 1430 mma.m8n8k16 u8 tiles per iteration, both outputs consumed - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r2 += d0_; r3 += d1_; } // t0 mm8 a=r6 b=r5 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r4 += d0_; r5 += d1_; } // t1 mm8 a=r1 b=r1 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r0 += d0_; r7 += d1_; } // t2 mm8 a=r2 b=r5 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r3 += d1_; } // t3 mm8 a=r1 b=r5 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r2 += d0_; r6 += d1_; } // t4 mm8 a=r2 b=r0 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r7 += d0_; r3 += d1_; } // t5 mm8 a=r2 b=r6 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t6 mm8 a=r1 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r0 += d0_; r4 += d1_; } // t7 mm8 a=r4 b=r6 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r1 += d0_; r5 += d1_; } // t8 mm8 a=r1 b=r1 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r4 += d0_; r6 += d1_; } // t9 mm8 a=r3 b=r7 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r0 += d0_; r4 += d1_; } // t10 mm8 a=r3 b=r0 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r0 += d0_; r6 += d1_; } // t11 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r6 += d0_; r0 += d1_; } // t12 mm8 a=r3 b=r1 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r5 += d1_; } // t13 mm8 a=r1 b=r5 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r3 += d0_; r4 += d1_; } // t14 mm8 a=r3 b=r3 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r5 += d0_; r4 += d1_; } // t15 mm8 a=r1 b=r0 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r0 += d0_; r4 += d1_; } // t16 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r7 += d1_; } // t17 mm8 a=r0 b=r5 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r2 += d0_; r0 += d1_; } // t18 mm8 a=r1 b=r1 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r2 += d0_; r4 += d1_; } // t19 mm8 a=r3 b=r3 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r6 += d0_; r1 += d1_; } // t20 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r6 += d0_; r0 += d1_; } // t21 mm8 a=r0 b=r3 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r4 += d1_; } // t22 mm8 a=r1 b=r7 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r0 += d0_; r3 += d1_; } // t23 mm8 a=r1 b=r2 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r0 += d0_; r2 += d1_; } // t24 mm8 a=r3 b=r4 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r3 += d0_; r7 += d1_; } // t25 mm8 a=r5 b=r0 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t26 mm8 a=r1 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r0 += d0_; r5 += d1_; } // t27 mm8 a=r3 b=r2 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r5 += d1_; } // t28 mm8 a=r2 b=r4 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r1 += d0_; r5 += d1_; } // t29 mm8 a=r3 b=r4 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r6 += d0_; r3 += d1_; } // t30 mm8 a=r6 b=r5 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t31 mm8 a=r1 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r3 += d0_; r0 += d1_; } // t32 mm8 a=r7 b=r6 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r4 += d0_; r6 += d1_; } // t33 mm8 a=r7 b=r1 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r5 += d0_; r1 += d1_; } // t34 mm8 a=r1 b=r0 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t35 mm8 a=r2 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r7 += d0_; r1 += d1_; } // t36 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r0 += d0_; r2 += d1_; } // t37 mm8 a=r1 b=r7 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r2 += d0_; r4 += d1_; } // t38 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r2 += d0_; r5 += d1_; } // t39 mm8 a=r6 b=r2 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r4 += d0_; r1 += d1_; } // t40 mm8 a=r6 b=r1 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r6 += d0_; r2 += d1_; } // t41 mm8 a=r6 b=r0 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r5 += d0_; r1 += d1_; } // t42 mm8 a=r7 b=r4 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r6 += d0_; r0 += d1_; } // t43 mm8 a=r6 b=r1 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r4 += d0_; r1 += d1_; } // t44 mm8 a=r0 b=r6 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r6 += d0_; r5 += d1_; } // t45 mm8 a=r6 b=r5 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r1 += d0_; r0 += d1_; } // t46 mm8 a=r6 b=r6 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r3 += d0_; r1 += d1_; } // t47 mm8 a=r7 b=r3 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t48 mm8 a=r7 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r1 += d0_; r7 += d1_; } // t49 mm8 a=r4 b=r5 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r4 += d0_; r5 += d1_; } // t50 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r0 += d0_; r3 += d1_; } // t51 mm8 a=r3 b=r4 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r7 += d0_; r4 += d1_; } // t52 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r1 += d0_; r3 += d1_; } // t53 mm8 a=r7 b=r7 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t54 mm8 a=r7 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r3 += d0_; r7 += d1_; } // t55 mm8 a=r1 b=r7 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t56 mm8 a=r4 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r6 += d0_; r7 += d1_; } // t57 mm8 a=r4 b=r3 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r6 += d0_; r0 += d1_; } // t58 mm8 a=r2 b=r3 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r7 += d1_; } // t59 mm8 a=r6 b=r7 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t60 mm8 a=r6 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r2 += d0_; r6 += d1_; } // t61 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r6 += d1_; } // t62 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t63 mm8 a=r1 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r5 += d0_; r3 += d1_; } // t64 mm8 a=r0 b=r5 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r2 += d0_; r4 += d1_; } // t65 mm8 a=r0 b=r1 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r1 += d0_; r3 += d1_; } // t66 mm8 a=r3 b=r5 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r3 += d0_; r4 += d1_; } // t67 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r2 += d1_; } // t68 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r4 += d1_; } // t69 mm8 a=r6 b=r7 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r1 += d0_; r5 += d1_; } // t70 mm8 a=r1 b=r7 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r1 += d0_; r0 += d1_; } // t71 mm8 a=r3 b=r0 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r3 += d0_; r6 += d1_; } // t72 mm8 a=r4 b=r0 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r0 += d0_; r2 += d1_; } // t73 mm8 a=r4 b=r1 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r3 += d0_; r4 += d1_; } // t74 mm8 a=r5 b=r1 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r7 += d0_; r3 += d1_; } // t75 mm8 a=r4 b=r4 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r1 += d0_; r7 += d1_; } // t76 mm8 a=r5 b=r6 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r3 += d0_; r2 += d1_; } // t77 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r0 += d0_; r5 += d1_; } // t78 mm8 a=r2 b=r5 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t79 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r3 += d0_; r4 += d1_; } // t80 mm8 a=r6 b=r7 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r1 += d0_; r6 += d1_; } // t81 mm8 a=r4 b=r3 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r1 += d0_; r7 += d1_; } // t82 mm8 a=r6 b=r0 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r1 += d0_; r6 += d1_; } // t83 mm8 a=r2 b=r2 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t84 mm8 a=r0 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t85 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r2 += d0_; r3 += d1_; } // t86 mm8 a=r3 b=r4 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r3 += d0_; r7 += d1_; } // t87 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r5 += d0_; r3 += d1_; } // t88 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r2 += d0_; r1 += d1_; } // t89 mm8 a=r4 b=r2 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r3 += d0_; r6 += d1_; } // t90 mm8 a=r6 b=r5 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r5 += d0_; r4 += d1_; } // t91 mm8 a=r6 b=r5 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r2 += d0_; r3 += d1_; } // t92 mm8 a=r2 b=r6 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r5 += d0_; r7 += d1_; } // t93 mm8 a=r2 b=r2 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r7 += d0_; r3 += d1_; } // t94 mm8 a=r2 b=r4 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r1 += d0_; r6 += d1_; } // t95 mm8 a=r5 b=r4 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r3 += d0_; r1 += d1_; } // t96 mm8 a=r0 b=r6 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r2 += d1_; } // t97 mm8 a=r0 b=r5 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t98 mm8 a=r2 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r0 += d0_; r2 += d1_; } // t99 mm8 a=r1 b=r4 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r2 += d0_; r5 += d1_; } // t100 mm8 a=r2 b=r4 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r7 += d1_; } // t101 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t102 mm8 a=r5 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t103 mm8 a=r4 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r3 += d0_; r7 += d1_; } // t104 mm8 a=r2 b=r0 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r0 += d0_; r5 += d1_; } // t105 mm8 a=r6 b=r1 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r0 += d0_; r5 += d1_; } // t106 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r4 += d0_; r2 += d1_; } // t107 mm8 a=r2 b=r1 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t108 mm8 a=r4 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r5 += d0_; r3 += d1_; } // t109 mm8 a=r2 b=r7 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r0 += d0_; r2 += d1_; } // t110 mm8 a=r1 b=r0 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r4 += d0_; r6 += d1_; } // t111 mm8 a=r6 b=r5 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r4 += d0_; r2 += d1_; } // t112 mm8 a=r5 b=r7 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t113 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t114 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r1 += d0_; r4 += d1_; } // t115 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r5 += d0_; r0 += d1_; } // t116 mm8 a=r7 b=r2 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r7 += d0_; r6 += d1_; } // t117 mm8 a=r7 b=r6 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r2 += d0_; r1 += d1_; } // t118 mm8 a=r0 b=r1 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r5 += d0_; r3 += d1_; } // t119 mm8 a=r4 b=r3 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r7 += d0_; r2 += d1_; } // t120 mm8 a=r5 b=r3 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r1 += d1_; } // t121 mm8 a=r2 b=r7 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r3 += d0_; r0 += d1_; } // t122 mm8 a=r0 b=r1 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r3 += d0_; r5 += d1_; } // t123 mm8 a=r5 b=r2 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t124 mm8 a=r5 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r7 += d0_; r2 += d1_; } // t125 mm8 a=r7 b=r2 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r3 += d0_; r2 += d1_; } // t126 mm8 a=r6 b=r3 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r1 += d1_; } // t127 mm8 a=r6 b=r7 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r4 += d0_; r1 += d1_; } // t128 mm8 a=r6 b=r7 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r6 += d0_; r3 += d1_; } // t129 mm8 a=r3 b=r2 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r2 += d0_; r1 += d1_; } // t130 mm8 a=r0 b=r4 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r3 += d0_; r1 += d1_; } // t131 mm8 a=r3 b=r5 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r0 += d0_; r6 += d1_; } // t132 mm8 a=r0 b=r7 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r0 += d0_; r1 += d1_; } // t133 mm8 a=r2 b=r6 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r3 += d0_; r1 += d1_; } // t134 mm8 a=r5 b=r6 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r4 += d0_; r5 += d1_; } // t135 mm8 a=r0 b=r2 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r1 += d0_; r5 += d1_; } // t136 mm8 a=r3 b=r1 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r4 += d0_; r6 += d1_; } // t137 mm8 a=r7 b=r0 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r4 += d0_; r2 += d1_; } // t138 mm8 a=r2 b=r2 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r2 += d0_; r7 += d1_; } // t139 mm8 a=r0 b=r4 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r6 += d1_; } // t140 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r1 += d0_; r2 += d1_; } // t141 mm8 a=r5 b=r5 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r6 += d0_; r2 += d1_; } // t142 mm8 a=r2 b=r2 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r7 += d0_; r1 += d1_; } // t143 mm8 a=r4 b=r3 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r1 += d0_; r7 += d1_; } // t144 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r6 += d0_; r5 += d1_; } // t145 mm8 a=r7 b=r2 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r7 += d1_; } // t146 mm8 a=r2 b=r7 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r3 += d0_; r0 += d1_; } // t147 mm8 a=r3 b=r1 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r0 += d0_; r6 += d1_; } // t148 mm8 a=r2 b=r1 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r7 += d0_; r1 += d1_; } // t149 mm8 a=r4 b=r6 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r7 += d0_; r0 += d1_; } // t150 mm8 a=r3 b=r6 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r2 += d0_; r4 += d1_; } // t151 mm8 a=r1 b=r4 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r5 += d0_; r1 += d1_; } // t152 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r2 += d0_; r3 += d1_; } // t153 mm8 a=r0 b=r3 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r0 += d0_; r4 += d1_; } // t154 mm8 a=r7 b=r3 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r7 += d0_; r5 += d1_; } // t155 mm8 a=r1 b=r3 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r2 += d0_; r3 += d1_; } // t156 mm8 a=r6 b=r3 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r4 += d0_; r5 += d1_; } // t157 mm8 a=r6 b=r4 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t158 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r5 += d0_; r2 += d1_; } // t159 mm8 a=r1 b=r0 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r5 += d0_; r7 += d1_; } // t160 mm8 a=r0 b=r1 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r7 += d0_; r0 += d1_; } // t161 mm8 a=r4 b=r0 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r6 += d0_; r1 += d1_; } // t162 mm8 a=r6 b=r6 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r7 += d0_; r3 += d1_; } // t163 mm8 a=r5 b=r3 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t164 mm8 a=r0 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r1 += d0_; r5 += d1_; } // t165 mm8 a=r6 b=r5 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r4 += d0_; r0 += d1_; } // t166 mm8 a=r6 b=r5 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r3 += d0_; r7 += d1_; } // t167 mm8 a=r4 b=r5 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r0 += d0_; r7 += d1_; } // t168 mm8 a=r3 b=r6 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r4 += d0_; r3 += d1_; } // t169 mm8 a=r6 b=r2 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t170 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r2 += d0_; r3 += d1_; } // t171 mm8 a=r6 b=r3 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r5 += d0_; r2 += d1_; } // t172 mm8 a=r7 b=r3 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t173 mm8 a=r0 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r5 += d0_; r6 += d1_; } // t174 mm8 a=r6 b=r6 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r3 += d0_; r4 += d1_; } // t175 mm8 a=r1 b=r6 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r7 += d0_; r5 += d1_; } // t176 mm8 a=r2 b=r4 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r7 += d0_; r2 += d1_; } // t177 mm8 a=r1 b=r7 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r7 += d0_; r3 += d1_; } // t178 mm8 a=r4 b=r2 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r5 += d0_; r1 += d1_; } // t179 mm8 a=r5 b=r2 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r2 += d0_; r1 += d1_; } // t180 mm8 a=r0 b=r4 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r2 += d0_; r4 += d1_; } // t181 mm8 a=r3 b=r5 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r4 += d0_; r3 += d1_; } // t182 mm8 a=r1 b=r1 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r0 += d0_; r7 += d1_; } // t183 mm8 a=r3 b=r1 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r0 += d0_; r7 += d1_; } // t184 mm8 a=r7 b=r4 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r6 += d0_; r3 += d1_; } // t185 mm8 a=r5 b=r0 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r7 += d0_; r6 += d1_; } // t186 mm8 a=r2 b=r5 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r0 += d0_; r7 += d1_; } // t187 mm8 a=r3 b=r0 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r5 += d0_; r0 += d1_; } // t188 mm8 a=r1 b=r4 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r5 += d0_; r4 += d1_; } // t189 mm8 a=r7 b=r2 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r0 += d1_; } // t190 mm8 a=r1 b=r5 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t191 mm8 a=r2 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t192 mm8 a=r5 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r4 += d0_; r6 += d1_; } // t193 mm8 a=r2 b=r1 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r0 += d0_; r5 += d1_; } // t194 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r4 += d1_; } // t195 mm8 a=r2 b=r7 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r2 += d0_; r5 += d1_; } // t196 mm8 a=r5 b=r6 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r6 += d0_; r5 += d1_; } // t197 mm8 a=r5 b=r3 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r7 += d0_; r1 += d1_; } // t198 mm8 a=r1 b=r0 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t199 mm8 a=r2 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t200 mm8 a=r5 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r2 += d0_; r1 += d1_; } // t201 mm8 a=r7 b=r2 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r5 += d0_; r6 += d1_; } // t202 mm8 a=r2 b=r5 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r6 += d1_; } // t203 mm8 a=r0 b=r5 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r3 += d0_; r7 += d1_; } // t204 mm8 a=r4 b=r7 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r1 += d0_; r3 += d1_; } // t205 mm8 a=r3 b=r7 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r7 += d0_; r3 += d1_; } // t206 mm8 a=r0 b=r6 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r1 += d0_; r4 += d1_; } // t207 mm8 a=r7 b=r7 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r6 += d0_; r5 += d1_; } // t208 mm8 a=r0 b=r1 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r6 += d0_; r2 += d1_; } // t209 mm8 a=r4 b=r4 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t210 mm8 a=r7 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r5 += d0_; r6 += d1_; } // t211 mm8 a=r1 b=r5 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r5 += d0_; r3 += d1_; } // t212 mm8 a=r2 b=r4 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r3 += d0_; r5 += d1_; } // t213 mm8 a=r2 b=r2 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r5 += d0_; r7 += d1_; } // t214 mm8 a=r4 b=r6 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r2 += d0_; r6 += d1_; } // t215 mm8 a=r7 b=r3 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r3 += d0_; r4 += d1_; } // t216 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r0 += d0_; r6 += d1_; } // t217 mm8 a=r5 b=r7 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r0 += d1_; } // t218 mm8 a=r7 b=r1 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r3 += d0_; r1 += d1_; } // t219 mm8 a=r2 b=r1 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r4 += d0_; r5 += d1_; } // t220 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r2 += d0_; r1 += d1_; } // t221 mm8 a=r7 b=r1 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r2 += d0_; r1 += d1_; } // t222 mm8 a=r0 b=r6 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r7 += d0_; r0 += d1_; } // t223 mm8 a=r2 b=r2 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r6 += d0_; r2 += d1_; } // t224 mm8 a=r1 b=r1 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r4 += d1_; } // t225 mm8 a=r2 b=r7 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t226 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r5 += d0_; r6 += d1_; } // t227 mm8 a=r3 b=r5 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r3 += d0_; r1 += d1_; } // t228 mm8 a=r7 b=r1 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r1 += d0_; r4 += d1_; } // t229 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r5 += d0_; r6 += d1_; } // t230 mm8 a=r0 b=r3 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r4 += d0_; r0 += d1_; } // t231 mm8 a=r1 b=r3 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r0 += d0_; r1 += d1_; } // t232 mm8 a=r5 b=r4 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r0 += d0_; r4 += d1_; } // t233 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r2 += d0_; r1 += d1_; } // t234 mm8 a=r2 b=r0 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t235 mm8 a=r3 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r0 += d0_; r5 += d1_; } // t236 mm8 a=r1 b=r7 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r6 += d0_; r3 += d1_; } // t237 mm8 a=r3 b=r5 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r0 += d0_; r2 += d1_; } // t238 mm8 a=r2 b=r0 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r1 += d0_; r0 += d1_; } // t239 mm8 a=r5 b=r7 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r4 += d0_; r6 += d1_; } // t240 mm8 a=r2 b=r7 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r3 += d0_; r7 += d1_; } // t241 mm8 a=r0 b=r1 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r2 += d0_; r7 += d1_; } // t242 mm8 a=r0 b=r2 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r1 += d0_; r5 += d1_; } // t243 mm8 a=r4 b=r4 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r0 += d0_; r7 += d1_; } // t244 mm8 a=r7 b=r6 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r2 += d0_; r0 += d1_; } // t245 mm8 a=r4 b=r4 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t246 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r2 += d0_; r4 += d1_; } // t247 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r7 += d0_; r3 += d1_; } // t248 mm8 a=r7 b=r5 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r0 += d0_; r5 += d1_; } // t249 mm8 a=r4 b=r4 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r0 += d0_; r1 += d1_; } // t250 mm8 a=r0 b=r6 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t251 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r5 += d0_; r4 += d1_; } // t252 mm8 a=r0 b=r2 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r7 += d0_; r4 += d1_; } // t253 mm8 a=r5 b=r5 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t254 mm8 a=r6 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r0 += d0_; r3 += d1_; } // t255 mm8 a=r5 b=r0 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r3 += d0_; r4 += d1_; } // t256 mm8 a=r2 b=r1 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r6 += d0_; r0 += d1_; } // t257 mm8 a=r4 b=r3 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r7 += d0_; r3 += d1_; } // t258 mm8 a=r4 b=r1 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r3 += d0_; r0 += d1_; } // t259 mm8 a=r5 b=r5 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r3 += d0_; r0 += d1_; } // t260 mm8 a=r3 b=r2 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r3 += d0_; r2 += d1_; } // t261 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r2 += d0_; r5 += d1_; } // t262 mm8 a=r6 b=r4 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r2 += d0_; r1 += d1_; } // t263 mm8 a=r6 b=r4 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r1 += d0_; r4 += d1_; } // t264 mm8 a=r7 b=r2 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r3 += d0_; r1 += d1_; } // t265 mm8 a=r6 b=r1 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r7 += d0_; r0 += d1_; } // t266 mm8 a=r0 b=r0 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r3 += d0_; r4 += d1_; } // t267 mm8 a=r6 b=r3 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r6 += d0_; r5 += d1_; } // t268 mm8 a=r1 b=r0 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r7 += d0_; r3 += d1_; } // t269 mm8 a=r7 b=r7 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r7 += d0_; r0 += d1_; } // t270 mm8 a=r6 b=r2 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r5 += d0_; r3 += d1_; } // t271 mm8 a=r0 b=r3 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r0 += d0_; r3 += d1_; } // t272 mm8 a=r6 b=r0 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r7 += d1_; } // t273 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r7 += d0_; r6 += d1_; } // t274 mm8 a=r1 b=r0 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r1 += d0_; r4 += d1_; } // t275 mm8 a=r1 b=r3 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r6 += d0_; r2 += d1_; } // t276 mm8 a=r1 b=r5 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r4 += d0_; r7 += d1_; } // t277 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r6 += d0_; r7 += d1_; } // t278 mm8 a=r0 b=r0 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r2 += d0_; r4 += d1_; } // t279 mm8 a=r2 b=r4 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r3 += d0_; r5 += d1_; } // t280 mm8 a=r7 b=r2 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r4 += d0_; r2 += d1_; } // t281 mm8 a=r1 b=r0 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r5 += d0_; r1 += d1_; } // t282 mm8 a=r0 b=r1 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r7 += d0_; r2 += d1_; } // t283 mm8 a=r0 b=r2 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r0 += d0_; r2 += d1_; } // t284 mm8 a=r1 b=r5 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r5 += d0_; r0 += d1_; } // t285 mm8 a=r4 b=r7 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r2 += d0_; r0 += d1_; } // t286 mm8 a=r1 b=r0 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r7 += d0_; r4 += d1_; } // t287 mm8 a=r7 b=r1 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t288 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r5 += d0_; r4 += d1_; } // t289 mm8 a=r0 b=r5 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r6 += d0_; r7 += d1_; } // t290 mm8 a=r3 b=r2 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r7 += d0_; r1 += d1_; } // t291 mm8 a=r7 b=r4 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t292 mm8 a=r6 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r6 += d0_; r4 += d1_; } // t293 mm8 a=r5 b=r5 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r1 += d0_; r6 += d1_; } // t294 mm8 a=r5 b=r7 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r1 += d0_; r3 += d1_; } // t295 mm8 a=r0 b=r7 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r5 += d0_; r6 += d1_; } // t296 mm8 a=r6 b=r1 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r3 += d0_; r5 += d1_; } // t297 mm8 a=r4 b=r5 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r3 += d0_; r1 += d1_; } // t298 mm8 a=r6 b=r0 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r2 += d0_; r3 += d1_; } // t299 mm8 a=r6 b=r0 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r4 += d0_; r2 += d1_; } // t300 mm8 a=r6 b=r4 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r6 += d0_; r3 += d1_; } // t301 mm8 a=r6 b=r4 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r6 += d0_; r4 += d1_; } // t302 mm8 a=r7 b=r6 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r3 += d0_; r6 += d1_; } // t303 mm8 a=r3 b=r2 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r2 += d0_; r4 += d1_; } // t304 mm8 a=r6 b=r5 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r0 += d0_; r2 += d1_; } // t305 mm8 a=r4 b=r5 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r0 += d0_; r2 += d1_; } // t306 mm8 a=r3 b=r1 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r5 += d0_; r0 += d1_; } // t307 mm8 a=r4 b=r3 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r2 += d0_; r6 += d1_; } // t308 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r1 += d0_; r4 += d1_; } // t309 mm8 a=r2 b=r1 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r2 += d0_; r3 += d1_; } // t310 mm8 a=r6 b=r1 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r3 += d0_; r2 += d1_; } // t311 mm8 a=r4 b=r6 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t312 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r7 += d0_; r1 += d1_; } // t313 mm8 a=r2 b=r6 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r2 += d0_; r1 += d1_; } // t314 mm8 a=r0 b=r3 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r6 += d0_; r2 += d1_; } // t315 mm8 a=r7 b=r1 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r1 += d0_; r6 += d1_; } // t316 mm8 a=r2 b=r4 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r3 += d0_; r6 += d1_; } // t317 mm8 a=r1 b=r6 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r1 += d0_; r0 += d1_; } // t318 mm8 a=r2 b=r1 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r6 += d0_; r3 += d1_; } // t319 mm8 a=r0 b=r2 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r6 += d0_; r2 += d1_; } // t320 mm8 a=r5 b=r2 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t321 mm8 a=r7 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r7 += d0_; r1 += d1_; } // t322 mm8 a=r2 b=r4 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r2 += d0_; r3 += d1_; } // t323 mm8 a=r4 b=r3 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r2 += d0_; r0 += d1_; } // t324 mm8 a=r5 b=r7 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r7 += d0_; r2 += d1_; } // t325 mm8 a=r6 b=r2 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r7 += d0_; r6 += d1_; } // t326 mm8 a=r6 b=r1 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r3 += d0_; r0 += d1_; } // t327 mm8 a=r1 b=r5 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r1 += d0_; r6 += d1_; } // t328 mm8 a=r0 b=r1 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r7 += d0_; r1 += d1_; } // t329 mm8 a=r0 b=r3 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r4 += d0_; r0 += d1_; } // t330 mm8 a=r6 b=r7 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t331 mm8 a=r4 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r6 += d0_; r3 += d1_; } // t332 mm8 a=r4 b=r4 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r4 += d0_; r2 += d1_; } // t333 mm8 a=r4 b=r6 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r0 += d0_; r1 += d1_; } // t334 mm8 a=r2 b=r2 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r2 += d1_; } // t335 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r2 += d0_; r3 += d1_; } // t336 mm8 a=r4 b=r0 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r0 += d0_; r3 += d1_; } // t337 mm8 a=r1 b=r4 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r2 += d0_; r7 += d1_; } // t338 mm8 a=r3 b=r7 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r7 += d0_; r6 += d1_; } // t339 mm8 a=r5 b=r1 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r7 += d0_; r5 += d1_; } // t340 mm8 a=r5 b=r0 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r7 += d0_; r5 += d1_; } // t341 mm8 a=r6 b=r6 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r0 += d0_; r4 += d1_; } // t342 mm8 a=r4 b=r7 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r4 += d0_; r6 += d1_; } // t343 mm8 a=r0 b=r4 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r4 += d0_; r6 += d1_; } // t344 mm8 a=r2 b=r5 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t345 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t346 mm8 a=r6 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r3 += d0_; r5 += d1_; } // t347 mm8 a=r5 b=r3 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r4 += d0_; r5 += d1_; } // t348 mm8 a=r0 b=r4 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r4 += d0_; r7 += d1_; } // t349 mm8 a=r6 b=r1 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r3 += d0_; r5 += d1_; } // t350 mm8 a=r1 b=r5 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r4 += d0_; r5 += d1_; } // t351 mm8 a=r6 b=r4 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r6 += d0_; r0 += d1_; } // t352 mm8 a=r4 b=r1 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r1 += d0_; r4 += d1_; } // t353 mm8 a=r7 b=r3 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r1 += d0_; r5 += d1_; } // t354 mm8 a=r4 b=r1 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r0 += d0_; r1 += d1_; } // t355 mm8 a=r4 b=r5 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r0 += d0_; r4 += d1_; } // t356 mm8 a=r7 b=r6 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r7 += d0_; r5 += d1_; } // t357 mm8 a=r7 b=r3 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r5 += d0_; r7 += d1_; } // t358 mm8 a=r4 b=r7 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r3 += d0_; r5 += d1_; } // t359 mm8 a=r2 b=r7 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r0 += d0_; r2 += d1_; } // t360 mm8 a=r4 b=r4 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r5 += d0_; r6 += d1_; } // t361 mm8 a=r2 b=r2 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r3 += d0_; r0 += d1_; } // t362 mm8 a=r2 b=r7 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r5 += d0_; r0 += d1_; } // t363 mm8 a=r7 b=r6 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r0 += d0_; r5 += d1_; } // t364 mm8 a=r6 b=r4 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r5 += d0_; r4 += d1_; } // t365 mm8 a=r6 b=r6 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r2 += d0_; r5 += d1_; } // t366 mm8 a=r7 b=r7 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t367 mm8 a=r4 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r3 += d0_; r5 += d1_; } // t368 mm8 a=r7 b=r0 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r6 += d0_; r0 += d1_; } // t369 mm8 a=r1 b=r0 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r7 += d0_; r4 += d1_; } // t370 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r2 += d0_; r6 += d1_; } // t371 mm8 a=r5 b=r4 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r2 += d1_; } // t372 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r2 += d0_; r4 += d1_; } // t373 mm8 a=r4 b=r6 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t374 mm8 a=r1 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r6 += d0_; r2 += d1_; } // t375 mm8 a=r6 b=r5 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t376 mm8 a=r3 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r6 += d0_; r4 += d1_; } // t377 mm8 a=r6 b=r4 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r2 += d0_; r4 += d1_; } // t378 mm8 a=r6 b=r3 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t379 mm8 a=r3 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r6 += d0_; r2 += d1_; } // t380 mm8 a=r2 b=r0 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r1 += d0_; r2 += d1_; } // t381 mm8 a=r3 b=r5 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r2 += d0_; r4 += d1_; } // t382 mm8 a=r5 b=r2 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r0 += d0_; r2 += d1_; } // t383 mm8 a=r1 b=r2 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r1 += d0_; r0 += d1_; } // t384 mm8 a=r3 b=r1 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r7 += d0_; r5 += d1_; } // t385 mm8 a=r6 b=r0 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r1 += d0_; r7 += d1_; } // t386 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r2 += d0_; r4 += d1_; } // t387 mm8 a=r3 b=r7 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r1 += d0_; r2 += d1_; } // t388 mm8 a=r3 b=r0 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r5 += d0_; r3 += d1_; } // t389 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r1 += d0_; r4 += d1_; } // t390 mm8 a=r5 b=r4 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t391 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r3 += d0_; r0 += d1_; } // t392 mm8 a=r2 b=r3 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r5 += d0_; r0 += d1_; } // t393 mm8 a=r3 b=r6 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r1 += d0_; r2 += d1_; } // t394 mm8 a=r0 b=r7 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r2 += d0_; r4 += d1_; } // t395 mm8 a=r0 b=r7 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r1 += d0_; r4 += d1_; } // t396 mm8 a=r7 b=r7 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t397 mm8 a=r5 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r7 += d0_; r0 += d1_; } // t398 mm8 a=r6 b=r5 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r3 += d0_; r5 += d1_; } // t399 mm8 a=r6 b=r3 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r0 += d0_; r4 += d1_; } // t400 mm8 a=r2 b=r7 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r2 += d0_; r1 += d1_; } // t401 mm8 a=r7 b=r6 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r7 += d0_; r1 += d1_; } // t402 mm8 a=r0 b=r0 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r1 += d0_; r0 += d1_; } // t403 mm8 a=r7 b=r4 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r1 += d0_; r4 += d1_; } // t404 mm8 a=r1 b=r1 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r3 += d0_; r6 += d1_; } // t405 mm8 a=r6 b=r6 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r4 += d0_; r3 += d1_; } // t406 mm8 a=r6 b=r1 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r2 += d0_; r4 += d1_; } // t407 mm8 a=r5 b=r7 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t408 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r4 += d0_; r7 += d1_; } // t409 mm8 a=r3 b=r4 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r5 += d0_; r2 += d1_; } // t410 mm8 a=r0 b=r5 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r5 += d0_; r1 += d1_; } // t411 mm8 a=r2 b=r0 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t412 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r2 += d1_; } // t413 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r0 += d0_; r3 += d1_; } // t414 mm8 a=r7 b=r1 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r5 += d0_; r1 += d1_; } // t415 mm8 a=r4 b=r7 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r3 += d0_; r5 += d1_; } // t416 mm8 a=r5 b=r4 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t417 mm8 a=r6 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r1 += d0_; r5 += d1_; } // t418 mm8 a=r1 b=r3 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r0 += d0_; r6 += d1_; } // t419 mm8 a=r5 b=r0 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r7 += d0_; r6 += d1_; } // t420 mm8 a=r4 b=r2 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r3 += d0_; r7 += d1_; } // t421 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t422 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r0 += d1_; } // t423 mm8 a=r1 b=r7 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r6 += d0_; r5 += d1_; } // t424 mm8 a=r5 b=r1 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t425 mm8 a=r2 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r5 += d0_; r0 += d1_; } // t426 mm8 a=r1 b=r6 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t427 mm8 a=r6 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r7 += d0_; r4 += d1_; } // t428 mm8 a=r6 b=r2 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r7 += d0_; r0 += d1_; } // t429 mm8 a=r3 b=r3 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r3 += d0_; r5 += d1_; } // t430 mm8 a=r7 b=r7 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r1 += d0_; r0 += d1_; } // t431 mm8 a=r6 b=r4 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r3 += d0_; r4 += d1_; } // t432 mm8 a=r3 b=r2 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r5 += d0_; r0 += d1_; } // t433 mm8 a=r3 b=r0 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r5 += d0_; r3 += d1_; } // t434 mm8 a=r0 b=r7 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r6 += d0_; r3 += d1_; } // t435 mm8 a=r7 b=r5 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r0 += d0_; r5 += d1_; } // t436 mm8 a=r6 b=r4 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r3 += d0_; r4 += d1_; } // t437 mm8 a=r0 b=r7 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r4 += d0_; r7 += d1_; } // t438 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t439 mm8 a=r4 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r3 += d0_; r2 += d1_; } // t440 mm8 a=r3 b=r5 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r1 += d0_; r4 += d1_; } // t441 mm8 a=r5 b=r1 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r7 += d0_; r5 += d1_; } // t442 mm8 a=r1 b=r2 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r5 += d0_; r2 += d1_; } // t443 mm8 a=r1 b=r7 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r1 += d0_; r0 += d1_; } // t444 mm8 a=r3 b=r7 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r7 += d0_; r0 += d1_; } // t445 mm8 a=r7 b=r5 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r2 += d0_; r1 += d1_; } // t446 mm8 a=r3 b=r1 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r4 += d0_; r7 += d1_; } // t447 mm8 a=r6 b=r7 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r4 += d0_; r5 += d1_; } // t448 mm8 a=r5 b=r2 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r3 += d1_; } // t449 mm8 a=r7 b=r1 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r7 += d0_; r3 += d1_; } // t450 mm8 a=r7 b=r5 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r1 += d0_; r7 += d1_; } // t451 mm8 a=r0 b=r3 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r7 += d0_; r1 += d1_; } // t452 mm8 a=r2 b=r4 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r7 += d0_; r1 += d1_; } // t453 mm8 a=r5 b=r7 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r7 += d0_; r1 += d1_; } // t454 mm8 a=r7 b=r7 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t455 mm8 a=r7 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r5 += d0_; r6 += d1_; } // t456 mm8 a=r6 b=r5 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r1 += d0_; r5 += d1_; } // t457 mm8 a=r7 b=r7 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r4 += d0_; r1 += d1_; } // t458 mm8 a=r5 b=r1 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r1 += d0_; r0 += d1_; } // t459 mm8 a=r6 b=r7 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t460 mm8 a=r7 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r6 += d0_; r2 += d1_; } // t461 mm8 a=r0 b=r6 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t462 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r3 += d0_; r2 += d1_; } // t463 mm8 a=r2 b=r0 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r0 += d0_; r1 += d1_; } // t464 mm8 a=r0 b=r3 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r0 += d0_; r5 += d1_; } // t465 mm8 a=r1 b=r7 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r4 += d0_; r1 += d1_; } // t466 mm8 a=r1 b=r2 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r1 += d0_; r6 += d1_; } // t467 mm8 a=r1 b=r0 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r6 += d0_; r2 += d1_; } // t468 mm8 a=r4 b=r1 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r0 += d0_; r1 += d1_; } // t469 mm8 a=r2 b=r0 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r2 += d0_; r0 += d1_; } // t470 mm8 a=r3 b=r1 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r7 += d0_; r1 += d1_; } // t471 mm8 a=r2 b=r2 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t472 mm8 a=r1 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r1 += d1_; } // t473 mm8 a=r2 b=r7 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r6 += d0_; r3 += d1_; } // t474 mm8 a=r5 b=r3 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r6 += d0_; r0 += d1_; } // t475 mm8 a=r7 b=r6 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r5 += d0_; r6 += d1_; } // t476 mm8 a=r3 b=r0 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r4 += d0_; r6 += d1_; } // t477 mm8 a=r3 b=r1 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r2 += d0_; r5 += d1_; } // t478 mm8 a=r0 b=r6 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r7 += d0_; r1 += d1_; } // t479 mm8 a=r7 b=r6 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r1 += d0_; r0 += d1_; } // t480 mm8 a=r6 b=r0 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r7 += d1_; } // t481 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r0 += d0_; r6 += d1_; } // t482 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r7 += d0_; r6 += d1_; } // t483 mm8 a=r7 b=r0 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r6 += d0_; r4 += d1_; } // t484 mm8 a=r3 b=r7 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r1 += d0_; r2 += d1_; } // t485 mm8 a=r6 b=r4 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r2 += d0_; r5 += d1_; } // t486 mm8 a=r6 b=r5 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r4 += d0_; r5 += d1_; } // t487 mm8 a=r2 b=r5 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r1 += d0_; r2 += d1_; } // t488 mm8 a=r1 b=r2 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r6 += d0_; r4 += d1_; } // t489 mm8 a=r5 b=r2 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r2 += d0_; r6 += d1_; } // t490 mm8 a=r7 b=r1 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r0 += d0_; r6 += d1_; } // t491 mm8 a=r0 b=r0 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r6 += d0_; r4 += d1_; } // t492 mm8 a=r3 b=r1 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r7 += d0_; r6 += d1_; } // t493 mm8 a=r6 b=r1 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r3 += d0_; r1 += d1_; } // t494 mm8 a=r5 b=r6 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r7 += d0_; r2 += d1_; } // t495 mm8 a=r2 b=r2 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r5 += d0_; r0 += d1_; } // t496 mm8 a=r3 b=r5 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r2 += d0_; r0 += d1_; } // t497 mm8 a=r3 b=r2 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r6 += d0_; r2 += d1_; } // t498 mm8 a=r3 b=r1 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r1 += d0_; r5 += d1_; } // t499 mm8 a=r1 b=r6 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r1 += d0_; r5 += d1_; } // t500 mm8 a=r7 b=r6 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r1 += d0_; r0 += d1_; } // t501 mm8 a=r4 b=r4 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r7 += d0_; r6 += d1_; } // t502 mm8 a=r5 b=r7 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r3 += d0_; r4 += d1_; } // t503 mm8 a=r7 b=r5 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r4 += d0_; r2 += d1_; } // t504 mm8 a=r2 b=r0 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r0 += d0_; r1 += d1_; } // t505 mm8 a=r4 b=r0 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r7 += d0_; r0 += d1_; } // t506 mm8 a=r0 b=r2 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r4 += d0_; r7 += d1_; } // t507 mm8 a=r5 b=r6 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r4 += d0_; r7 += d1_; } // t508 mm8 a=r7 b=r3 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r6 += d0_; r0 += d1_; } // t509 mm8 a=r4 b=r4 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t510 mm8 a=r0 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t511 mm8 a=r2 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r4 += d0_; r1 += d1_; } // t512 mm8 a=r2 b=r7 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r5 += d0_; r7 += d1_; } // t513 mm8 a=r0 b=r3 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r4 += d0_; r2 += d1_; } // t514 mm8 a=r6 b=r7 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r0 += d0_; r4 += d1_; } // t515 mm8 a=r5 b=r2 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r2 += d0_; r4 += d1_; } // t516 mm8 a=r0 b=r2 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r6 += d0_; r7 += d1_; } // t517 mm8 a=r6 b=r7 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r0 += d0_; r6 += d1_; } // t518 mm8 a=r1 b=r4 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r7 += d0_; r4 += d1_; } // t519 mm8 a=r6 b=r0 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r0 += d0_; r2 += d1_; } // t520 mm8 a=r1 b=r2 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r0 += d0_; r2 += d1_; } // t521 mm8 a=r0 b=r4 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r6 += d0_; r7 += d1_; } // t522 mm8 a=r7 b=r2 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r7 += d0_; r0 += d1_; } // t523 mm8 a=r5 b=r3 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r7 += d0_; r0 += d1_; } // t524 mm8 a=r0 b=r4 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r5 += d0_; r1 += d1_; } // t525 mm8 a=r1 b=r5 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r6 += d1_; } // t526 mm8 a=r7 b=r1 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r6 += d0_; r7 += d1_; } // t527 mm8 a=r5 b=r1 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r3 += d0_; r2 += d1_; } // t528 mm8 a=r4 b=r1 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r2 += d0_; r1 += d1_; } // t529 mm8 a=r4 b=r2 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r6 += d0_; r4 += d1_; } // t530 mm8 a=r0 b=r2 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r3 += d0_; r4 += d1_; } // t531 mm8 a=r7 b=r0 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r2 += d0_; r7 += d1_; } // t532 mm8 a=r7 b=r2 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r6 += d0_; r4 += d1_; } // t533 mm8 a=r3 b=r2 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r3 += d0_; r6 += d1_; } // t534 mm8 a=r4 b=r3 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r1 += d0_; r6 += d1_; } // t535 mm8 a=r0 b=r6 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r4 += d0_; r6 += d1_; } // t536 mm8 a=r3 b=r1 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r4 += d0_; r5 += d1_; } // t537 mm8 a=r1 b=r6 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r6 += d0_; r7 += d1_; } // t538 mm8 a=r6 b=r6 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t539 mm8 a=r4 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r2 += d0_; r4 += d1_; } // t540 mm8 a=r1 b=r4 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r3 += d0_; r6 += d1_; } // t541 mm8 a=r5 b=r3 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r0 += d0_; r7 += d1_; } // t542 mm8 a=r7 b=r7 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r3 += d0_; r7 += d1_; } // t543 mm8 a=r4 b=r2 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r3 += d0_; r6 += d1_; } // t544 mm8 a=r5 b=r7 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r2 += d0_; r1 += d1_; } // t545 mm8 a=r5 b=r2 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r5 += d0_; r7 += d1_; } // t546 mm8 a=r7 b=r1 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r2 += d0_; r1 += d1_; } // t547 mm8 a=r3 b=r5 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r5 += d0_; r4 += d1_; } // t548 mm8 a=r6 b=r3 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r3 += d0_; r7 += d1_; } // t549 mm8 a=r2 b=r2 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r5 += d0_; r2 += d1_; } // t550 mm8 a=r5 b=r5 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r4 += d0_; r0 += d1_; } // t551 mm8 a=r7 b=r6 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r1 += d0_; r2 += d1_; } // t552 mm8 a=r6 b=r7 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r6 += d0_; r7 += d1_; } // t553 mm8 a=r6 b=r3 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r7 += d0_; r0 += d1_; } // t554 mm8 a=r4 b=r2 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r0 += d0_; r1 += d1_; } // t555 mm8 a=r2 b=r7 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r0 += d0_; r7 += d1_; } // t556 mm8 a=r3 b=r6 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r7 += d0_; r0 += d1_; } // t557 mm8 a=r7 b=r7 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r5 += d0_; r2 += d1_; } // t558 mm8 a=r5 b=r0 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r5 += d0_; r1 += d1_; } // t559 mm8 a=r4 b=r1 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r5 += d0_; r4 += d1_; } // t560 mm8 a=r5 b=r4 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t561 mm8 a=r6 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r7 += d0_; r0 += d1_; } // t562 mm8 a=r1 b=r2 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r3 += d0_; r2 += d1_; } // t563 mm8 a=r5 b=r3 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r4 += d0_; r6 += d1_; } // t564 mm8 a=r4 b=r3 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r2 += d0_; r7 += d1_; } // t565 mm8 a=r6 b=r0 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r5 += d0_; r0 += d1_; } // t566 mm8 a=r1 b=r6 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r2 += d0_; r0 += d1_; } // t567 mm8 a=r3 b=r6 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r3 += d0_; r1 += d1_; } // t568 mm8 a=r6 b=r5 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r2 += d0_; r5 += d1_; } // t569 mm8 a=r3 b=r7 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r3 += d0_; r2 += d1_; } // t570 mm8 a=r5 b=r1 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r2 += d0_; r6 += d1_; } // t571 mm8 a=r6 b=r0 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r3 += d0_; r1 += d1_; } // t572 mm8 a=r5 b=r0 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r4 += d0_; r3 += d1_; } // t573 mm8 a=r3 b=r0 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r0 += d0_; r5 += d1_; } // t574 mm8 a=r2 b=r2 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r6 += d0_; r0 += d1_; } // t575 mm8 a=r3 b=r1 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r0 += d0_; r4 += d1_; } // t576 mm8 a=r2 b=r3 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r6 += d0_; r1 += d1_; } // t577 mm8 a=r6 b=r3 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r2 += d0_; r4 += d1_; } // t578 mm8 a=r5 b=r6 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r7 += d0_; r4 += d1_; } // t579 mm8 a=r4 b=r7 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r5 += d0_; r7 += d1_; } // t580 mm8 a=r7 b=r1 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r3 += d0_; r4 += d1_; } // t581 mm8 a=r1 b=r6 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r1 += d0_; r3 += d1_; } // t582 mm8 a=r2 b=r1 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t583 mm8 a=r5 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r0 += d0_; r4 += d1_; } // t584 mm8 a=r1 b=r7 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r5 += d0_; r2 += d1_; } // t585 mm8 a=r0 b=r4 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r1 += d0_; r2 += d1_; } // t586 mm8 a=r4 b=r4 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r6 += d0_; r3 += d1_; } // t587 mm8 a=r0 b=r4 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r7 += d0_; r0 += d1_; } // t588 mm8 a=r5 b=r5 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r3 += d0_; r5 += d1_; } // t589 mm8 a=r3 b=r6 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r5 += d0_; r0 += d1_; } // t590 mm8 a=r3 b=r2 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r0 += d0_; r5 += d1_; } // t591 mm8 a=r5 b=r1 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r7 += d0_; r4 += d1_; } // t592 mm8 a=r5 b=r3 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r6 += d0_; r4 += d1_; } // t593 mm8 a=r5 b=r3 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r4 += d0_; r0 += d1_; } // t594 mm8 a=r6 b=r7 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r6 += d0_; r4 += d1_; } // t595 mm8 a=r3 b=r0 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r3 += d0_; r2 += d1_; } // t596 mm8 a=r7 b=r6 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r5 += d0_; r1 += d1_; } // t597 mm8 a=r2 b=r4 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r4 += d0_; r0 += d1_; } // t598 mm8 a=r7 b=r7 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r2 += d0_; r6 += d1_; } // t599 mm8 a=r7 b=r2 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r5 += d0_; r4 += d1_; } // t600 mm8 a=r7 b=r7 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r7 += d0_; r5 += d1_; } // t601 mm8 a=r2 b=r7 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r7 += d0_; r1 += d1_; } // t602 mm8 a=r0 b=r1 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r7 += d0_; r4 += d1_; } // t603 mm8 a=r0 b=r0 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r2 += d0_; r1 += d1_; } // t604 mm8 a=r5 b=r5 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r4 += d0_; r2 += d1_; } // t605 mm8 a=r2 b=r0 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r1 += d0_; r7 += d1_; } // t606 mm8 a=r7 b=r5 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r4 += d0_; r5 += d1_; } // t607 mm8 a=r0 b=r2 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r6 += d0_; r4 += d1_; } // t608 mm8 a=r7 b=r6 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r7 += d0_; r1 += d1_; } // t609 mm8 a=r1 b=r5 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r1 += d0_; r6 += d1_; } // t610 mm8 a=r3 b=r3 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r5 += d0_; r7 += d1_; } // t611 mm8 a=r2 b=r2 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r1 += d0_; r2 += d1_; } // t612 mm8 a=r1 b=r1 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r3 += d0_; r4 += d1_; } // t613 mm8 a=r6 b=r3 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t614 mm8 a=r3 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r7 += d0_; r6 += d1_; } // t615 mm8 a=r6 b=r2 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r1 += d0_; r7 += d1_; } // t616 mm8 a=r2 b=r3 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r3 += d0_; r2 += d1_; } // t617 mm8 a=r5 b=r4 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r5 += d0_; r0 += d1_; } // t618 mm8 a=r3 b=r6 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r0 += d0_; r7 += d1_; } // t619 mm8 a=r5 b=r0 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r6 += d0_; r4 += d1_; } // t620 mm8 a=r3 b=r5 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r7 += d0_; r2 += d1_; } // t621 mm8 a=r4 b=r2 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r4 += d0_; r3 += d1_; } // t622 mm8 a=r7 b=r0 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r2 += d0_; r1 += d1_; } // t623 mm8 a=r4 b=r6 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r1 += d0_; r0 += d1_; } // t624 mm8 a=r6 b=r2 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r7 += d0_; r5 += d1_; } // t625 mm8 a=r6 b=r2 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r5 += d0_; r6 += d1_; } // t626 mm8 a=r5 b=r2 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r2 += d0_; r3 += d1_; } // t627 mm8 a=r4 b=r0 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r0 += d1_; } // t628 mm8 a=r1 b=r5 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r7 += d0_; r3 += d1_; } // t629 mm8 a=r4 b=r5 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r6 += d0_; r4 += d1_; } // t630 mm8 a=r4 b=r5 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r7 += d0_; r4 += d1_; } // t631 mm8 a=r6 b=r4 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r7 += d0_; r1 += d1_; } // t632 mm8 a=r2 b=r0 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t633 mm8 a=r2 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t634 mm8 a=r1 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r2 += d0_; r4 += d1_; } // t635 mm8 a=r6 b=r7 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r4 += d0_; r1 += d1_; } // t636 mm8 a=r2 b=r5 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t637 mm8 a=r5 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r5 += d0_; r1 += d1_; } // t638 mm8 a=r1 b=r4 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r5 += d0_; r1 += d1_; } // t639 mm8 a=r1 b=r7 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r4 += d0_; r0 += d1_; } // t640 mm8 a=r0 b=r4 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r5 += d0_; r4 += d1_; } // t641 mm8 a=r5 b=r6 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r1 += d0_; r6 += d1_; } // t642 mm8 a=r7 b=r5 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r5 += d0_; r6 += d1_; } // t643 mm8 a=r3 b=r5 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r2 += d0_; r5 += d1_; } // t644 mm8 a=r1 b=r6 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r4 += d0_; r1 += d1_; } // t645 mm8 a=r0 b=r7 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r4 += d0_; r3 += d1_; } // t646 mm8 a=r5 b=r6 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r6 += d0_; r4 += d1_; } // t647 mm8 a=r6 b=r4 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r6 += d0_; r2 += d1_; } // t648 mm8 a=r3 b=r1 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r6 += d0_; r7 += d1_; } // t649 mm8 a=r7 b=r2 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r3 += d0_; r5 += d1_; } // t650 mm8 a=r7 b=r7 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r3 += d0_; r4 += d1_; } // t651 mm8 a=r6 b=r3 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r3 += d0_; r2 += d1_; } // t652 mm8 a=r1 b=r6 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r0 += d0_; r5 += d1_; } // t653 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r7 += d0_; r4 += d1_; } // t654 mm8 a=r1 b=r6 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r3 += d0_; r1 += d1_; } // t655 mm8 a=r7 b=r5 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r1 += d0_; r6 += d1_; } // t656 mm8 a=r6 b=r5 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r7 += d0_; r2 += d1_; } // t657 mm8 a=r7 b=r4 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r6 += d0_; r1 += d1_; } // t658 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r3 += d0_; r0 += d1_; } // t659 mm8 a=r0 b=r7 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r4 += d0_; r5 += d1_; } // t660 mm8 a=r6 b=r1 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r1 += d0_; r4 += d1_; } // t661 mm8 a=r3 b=r7 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r5 += d0_; r3 += d1_; } // t662 mm8 a=r2 b=r3 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r0 += d0_; r4 += d1_; } // t663 mm8 a=r0 b=r3 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r1 += d0_; r3 += d1_; } // t664 mm8 a=r2 b=r0 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r1 += d0_; r0 += d1_; } // t665 mm8 a=r1 b=r1 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r2 += d0_; r7 += d1_; } // t666 mm8 a=r7 b=r1 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r3 += d0_; r6 += d1_; } // t667 mm8 a=r2 b=r0 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r3 += d0_; r6 += d1_; } // t668 mm8 a=r3 b=r1 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r4 += d0_; r3 += d1_; } // t669 mm8 a=r1 b=r4 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r5 += d0_; r0 += d1_; } // t670 mm8 a=r7 b=r7 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r0 += d0_; r3 += d1_; } // t671 mm8 a=r0 b=r3 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r4 += d0_; r3 += d1_; } // t672 mm8 a=r0 b=r1 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r0 += d0_; r2 += d1_; } // t673 mm8 a=r6 b=r2 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r0 += d0_; r5 += d1_; } // t674 mm8 a=r3 b=r3 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r3 += d0_; r6 += d1_; } // t675 mm8 a=r5 b=r6 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r1 += d0_; r0 += d1_; } // t676 mm8 a=r4 b=r3 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t677 mm8 a=r0 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r7 += d0_; r1 += d1_; } // t678 mm8 a=r4 b=r7 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r1 += d0_; r4 += d1_; } // t679 mm8 a=r4 b=r7 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r5 += d0_; r2 += d1_; } // t680 mm8 a=r5 b=r1 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r4 += d0_; r2 += d1_; } // t681 mm8 a=r4 b=r2 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t682 mm8 a=r3 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r4 += d0_; r7 += d1_; } // t683 mm8 a=r0 b=r5 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r3 += d0_; r7 += d1_; } // t684 mm8 a=r6 b=r2 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r0 += d0_; r5 += d1_; } // t685 mm8 a=r7 b=r3 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r1 += d0_; r7 += d1_; } // t686 mm8 a=r4 b=r1 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r3 += d0_; r0 += d1_; } // t687 mm8 a=r0 b=r5 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r4 += d0_; r1 += d1_; } // t688 mm8 a=r3 b=r6 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r7 += d0_; r1 += d1_; } // t689 mm8 a=r3 b=r1 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r0 += d0_; r1 += d1_; } // t690 mm8 a=r2 b=r6 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r0 += d0_; r6 += d1_; } // t691 mm8 a=r2 b=r6 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r0 += d0_; r7 += d1_; } // t692 mm8 a=r1 b=r4 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t693 mm8 a=r7 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r7 += d0_; r3 += d1_; } // t694 mm8 a=r3 b=r5 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r2 += d0_; r4 += d1_; } // t695 mm8 a=r3 b=r2 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r3 += d0_; r0 += d1_; } // t696 mm8 a=r2 b=r4 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r3 += d0_; r4 += d1_; } // t697 mm8 a=r2 b=r3 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r4 += d0_; r7 += d1_; } // t698 mm8 a=r3 b=r0 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r4 += d0_; r2 += d1_; } // t699 mm8 a=r2 b=r1 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r1 += d0_; r3 += d1_; } // t700 mm8 a=r0 b=r1 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r1 += d0_; r3 += d1_; } // t701 mm8 a=r5 b=r7 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r4 += d0_; r6 += d1_; } // t702 mm8 a=r5 b=r5 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r1 += d0_; r2 += d1_; } // t703 mm8 a=r2 b=r2 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r7 += d0_; r2 += d1_; } // t704 mm8 a=r6 b=r5 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r7 += d0_; r3 += d1_; } // t705 mm8 a=r5 b=r4 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r4 += d0_; r1 += d1_; } // t706 mm8 a=r6 b=r0 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r7 += d0_; r6 += d1_; } // t707 mm8 a=r6 b=r4 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r7 += d0_; r4 += d1_; } // t708 mm8 a=r2 b=r0 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r7 += d0_; r5 += d1_; } // t709 mm8 a=r7 b=r0 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r5 += d0_; r6 += d1_; } // t710 mm8 a=r4 b=r2 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r6 += d0_; r1 += d1_; } // t711 mm8 a=r4 b=r7 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r2 += d1_; } // t712 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r5 += d0_; r1 += d1_; } // t713 mm8 a=r5 b=r2 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r5 += d0_; r1 += d1_; } // t714 mm8 a=r0 b=r0 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t715 mm8 a=r1 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r5 += d0_; r6 += d1_; } // t716 mm8 a=r3 b=r3 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r7 += d0_; r1 += d1_; } // t717 mm8 a=r3 b=r3 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r2 += d0_; r7 += d1_; } // t718 mm8 a=r7 b=r5 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r3 += d0_; r0 += d1_; } // t719 mm8 a=r4 b=r7 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r5 += d0_; r3 += d1_; } // t720 mm8 a=r4 b=r7 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r0 += d0_; r3 += d1_; } // t721 mm8 a=r6 b=r4 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r7 += d0_; r6 += d1_; } // t722 mm8 a=r7 b=r5 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r7 += d0_; r2 += d1_; } // t723 mm8 a=r5 b=r0 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r4 += d0_; r0 += d1_; } // t724 mm8 a=r7 b=r4 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r3 += d0_; r5 += d1_; } // t725 mm8 a=r6 b=r1 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r5 += d0_; r1 += d1_; } // t726 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r4 += d0_; r6 += d1_; } // t727 mm8 a=r0 b=r6 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t728 mm8 a=r5 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r2 += d0_; r3 += d1_; } // t729 mm8 a=r5 b=r6 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r4 += d0_; r7 += d1_; } // t730 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r6 += d0_; r7 += d1_; } // t731 mm8 a=r5 b=r6 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r0 += d0_; r6 += d1_; } // t732 mm8 a=r5 b=r1 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t733 mm8 a=r3 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r0 += d0_; r3 += d1_; } // t734 mm8 a=r4 b=r3 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r4 += d0_; r3 += d1_; } // t735 mm8 a=r7 b=r4 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r3 += d0_; r0 += d1_; } // t736 mm8 a=r4 b=r2 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r1 += d0_; r2 += d1_; } // t737 mm8 a=r4 b=r7 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r1 += d0_; r2 += d1_; } // t738 mm8 a=r6 b=r2 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r6 += d0_; r4 += d1_; } // t739 mm8 a=r7 b=r5 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r2 += d0_; r7 += d1_; } // t740 mm8 a=r4 b=r3 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t741 mm8 a=r7 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r4 += d0_; r1 += d1_; } // t742 mm8 a=r6 b=r5 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r6 += d0_; r4 += d1_; } // t743 mm8 a=r5 b=r3 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r3 += d0_; r6 += d1_; } // t744 mm8 a=r7 b=r4 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r0 += d0_; r4 += d1_; } // t745 mm8 a=r7 b=r0 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r1 += d0_; r3 += d1_; } // t746 mm8 a=r2 b=r5 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r1 += d0_; r0 += d1_; } // t747 mm8 a=r0 b=r3 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r5 += d0_; r7 += d1_; } // t748 mm8 a=r1 b=r6 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r7 += d0_; r5 += d1_; } // t749 mm8 a=r2 b=r5 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r7 += d0_; r0 += d1_; } // t750 mm8 a=r4 b=r3 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r5 += d0_; r3 += d1_; } // t751 mm8 a=r6 b=r6 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r5 += d0_; r3 += d1_; } // t752 mm8 a=r5 b=r0 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r6 += d0_; r0 += d1_; } // t753 mm8 a=r6 b=r7 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r5 += d0_; r2 += d1_; } // t754 mm8 a=r2 b=r4 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r4 += d0_; r0 += d1_; } // t755 mm8 a=r5 b=r6 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r5 += d0_; r4 += d1_; } // t756 mm8 a=r7 b=r3 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r3 += d0_; r5 += d1_; } // t757 mm8 a=r6 b=r6 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r4 += d0_; r2 += d1_; } // t758 mm8 a=r6 b=r2 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r0 += d1_; } // t759 mm8 a=r2 b=r7 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r1 += d0_; r0 += d1_; } // t760 mm8 a=r7 b=r7 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r2 += d0_; r4 += d1_; } // t761 mm8 a=r3 b=r3 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r5 += d0_; r6 += d1_; } // t762 mm8 a=r5 b=r2 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r6 += d0_; r0 += d1_; } // t763 mm8 a=r1 b=r1 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r0 += d0_; r2 += d1_; } // t764 mm8 a=r1 b=r2 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r3 += d1_; } // t765 mm8 a=r2 b=r4 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r6 += d0_; r0 += d1_; } // t766 mm8 a=r7 b=r6 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r5 += d0_; r7 += d1_; } // t767 mm8 a=r7 b=r5 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r7 += d0_; r3 += d1_; } // t768 mm8 a=r5 b=r6 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r1 += d0_; r5 += d1_; } // t769 mm8 a=r4 b=r5 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r2 += d0_; r6 += d1_; } // t770 mm8 a=r7 b=r5 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r3 += d0_; r6 += d1_; } // t771 mm8 a=r2 b=r5 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r1 += d0_; r6 += d1_; } // t772 mm8 a=r0 b=r4 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r1 += d0_; r3 += d1_; } // t773 mm8 a=r2 b=r5 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r2 += d0_; r0 += d1_; } // t774 mm8 a=r4 b=r0 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t775 mm8 a=r4 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r1 += d0_; r0 += d1_; } // t776 mm8 a=r5 b=r7 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r2 += d0_; r0 += d1_; } // t777 mm8 a=r1 b=r1 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r5 += d0_; r7 += d1_; } // t778 mm8 a=r1 b=r2 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r0 += d0_; r7 += d1_; } // t779 mm8 a=r7 b=r5 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r0 += d0_; r3 += d1_; } // t780 mm8 a=r3 b=r7 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r7 += d0_; r3 += d1_; } // t781 mm8 a=r6 b=r7 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r6 += d0_; r4 += d1_; } // t782 mm8 a=r6 b=r4 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r6 += d0_; r0 += d1_; } // t783 mm8 a=r6 b=r5 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r6 += d1_; } // t784 mm8 a=r1 b=r5 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r5 += d0_; r3 += d1_; } // t785 mm8 a=r7 b=r2 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r5 += d0_; r2 += d1_; } // t786 mm8 a=r2 b=r4 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r4 += d0_; r6 += d1_; } // t787 mm8 a=r7 b=r2 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r7 += d0_; r6 += d1_; } // t788 mm8 a=r0 b=r0 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r3 += d0_; r5 += d1_; } // t789 mm8 a=r1 b=r4 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r2 += d0_; r3 += d1_; } // t790 mm8 a=r3 b=r1 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r7 += d0_; r4 += d1_; } // t791 mm8 a=r3 b=r5 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r6 += d0_; r4 += d1_; } // t792 mm8 a=r7 b=r1 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r1 += d0_; r0 += d1_; } // t793 mm8 a=r7 b=r2 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r3 += d0_; r0 += d1_; } // t794 mm8 a=r2 b=r4 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r2 += d1_; } // t795 mm8 a=r1 b=r7 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r3 += d0_; r6 += d1_; } // t796 mm8 a=r2 b=r4 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r2 += d0_; r7 += d1_; } // t797 mm8 a=r6 b=r2 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r2 += d0_; r1 += d1_; } // t798 mm8 a=r1 b=r3 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r4 += d0_; r2 += d1_; } // t799 mm8 a=r1 b=r3 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r6 += d0_; r4 += d1_; } // t800 mm8 a=r0 b=r4 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r1 += d0_; r7 += d1_; } // t801 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r6 += d0_; r0 += d1_; } // t802 mm8 a=r4 b=r7 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r7 += d0_; r5 += d1_; } // t803 mm8 a=r3 b=r2 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r7 += d0_; r2 += d1_; } // t804 mm8 a=r0 b=r4 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r5 += d0_; r2 += d1_; } // t805 mm8 a=r4 b=r6 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r2 += d0_; r7 += d1_; } // t806 mm8 a=r0 b=r5 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r2 += d0_; r3 += d1_; } // t807 mm8 a=r5 b=r0 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r7 += d0_; r4 += d1_; } // t808 mm8 a=r2 b=r4 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r4 += d0_; r6 += d1_; } // t809 mm8 a=r1 b=r6 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r5 += d0_; r4 += d1_; } // t810 mm8 a=r1 b=r2 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t811 mm8 a=r2 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r6 += d0_; r1 += d1_; } // t812 mm8 a=r0 b=r6 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r2 += d0_; r3 += d1_; } // t813 mm8 a=r2 b=r2 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r2 += d0_; r6 += d1_; } // t814 mm8 a=r3 b=r4 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r5 += d0_; r7 += d1_; } // t815 mm8 a=r4 b=r5 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r1 += d0_; r7 += d1_; } // t816 mm8 a=r3 b=r0 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r5 += d0_; r7 += d1_; } // t817 mm8 a=r3 b=r4 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r5 += d0_; r2 += d1_; } // t818 mm8 a=r2 b=r2 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r7 += d0_; r4 += d1_; } // t819 mm8 a=r1 b=r2 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r2 += d0_; r6 += d1_; } // t820 mm8 a=r3 b=r6 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r5 += d0_; r1 += d1_; } // t821 mm8 a=r7 b=r3 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r4 += d0_; r3 += d1_; } // t822 mm8 a=r1 b=r5 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r2 += d0_; r5 += d1_; } // t823 mm8 a=r1 b=r4 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r0 += d0_; r5 += d1_; } // t824 mm8 a=r7 b=r7 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r1 += d0_; r2 += d1_; } // t825 mm8 a=r1 b=r1 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r7 += d0_; r1 += d1_; } // t826 mm8 a=r0 b=r2 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r6 += d0_; r7 += d1_; } // t827 mm8 a=r3 b=r1 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r2 += d0_; r4 += d1_; } // t828 mm8 a=r7 b=r4 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r0 += d0_; r5 += d1_; } // t829 mm8 a=r5 b=r2 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r4 += d0_; r0 += d1_; } // t830 mm8 a=r6 b=r3 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r5 += d1_; } // t831 mm8 a=r2 b=r7 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r6 += d0_; r1 += d1_; } // t832 mm8 a=r1 b=r6 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r5 += d0_; r2 += d1_; } // t833 mm8 a=r0 b=r6 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r3 += d0_; r6 += d1_; } // t834 mm8 a=r2 b=r4 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r5 += d0_; r0 += d1_; } // t835 mm8 a=r0 b=r7 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t836 mm8 a=r3 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r6 += d0_; r4 += d1_; } // t837 mm8 a=r4 b=r7 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r4 += d0_; r3 += d1_; } // t838 mm8 a=r5 b=r4 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r7 += d0_; r3 += d1_; } // t839 mm8 a=r0 b=r6 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r5 += d0_; r3 += d1_; } // t840 mm8 a=r1 b=r6 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r4 += d0_; r2 += d1_; } // t841 mm8 a=r4 b=r0 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r3 += d0_; r7 += d1_; } // t842 mm8 a=r3 b=r5 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r2 += d0_; r4 += d1_; } // t843 mm8 a=r2 b=r2 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r2 += d0_; r1 += d1_; } // t844 mm8 a=r3 b=r3 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t845 mm8 a=r1 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r2 += d0_; r3 += d1_; } // t846 mm8 a=r1 b=r4 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t847 mm8 a=r4 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r1 += d0_; r0 += d1_; } // t848 mm8 a=r6 b=r7 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r5 += d0_; r4 += d1_; } // t849 mm8 a=r6 b=r6 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r6 += d0_; r3 += d1_; } // t850 mm8 a=r6 b=r0 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r0 += d0_; r3 += d1_; } // t851 mm8 a=r0 b=r5 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r2 += d0_; r1 += d1_; } // t852 mm8 a=r2 b=r2 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r6 += d0_; r7 += d1_; } // t853 mm8 a=r6 b=r5 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r5 += d0_; r7 += d1_; } // t854 mm8 a=r1 b=r4 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r0 += d0_; r1 += d1_; } // t855 mm8 a=r1 b=r7 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r1 += d0_; r7 += d1_; } // t856 mm8 a=r2 b=r3 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r4 += d0_; r0 += d1_; } // t857 mm8 a=r6 b=r5 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r7 += d1_; } // t858 mm8 a=r0 b=r5 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t859 mm8 a=r2 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r4 += d0_; r7 += d1_; } // t860 mm8 a=r7 b=r2 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r7 += d0_; r4 += d1_; } // t861 mm8 a=r3 b=r3 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r4 += d0_; r5 += d1_; } // t862 mm8 a=r0 b=r1 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r4 += d0_; r1 += d1_; } // t863 mm8 a=r0 b=r0 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r2 += d0_; r1 += d1_; } // t864 mm8 a=r5 b=r5 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t865 mm8 a=r3 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r1 += d0_; r5 += d1_; } // t866 mm8 a=r6 b=r6 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t867 mm8 a=r7 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t868 mm8 a=r4 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r2 += d0_; r7 += d1_; } // t869 mm8 a=r3 b=r6 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r6 += d0_; r5 += d1_; } // t870 mm8 a=r0 b=r1 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r3 += d0_; r1 += d1_; } // t871 mm8 a=r5 b=r1 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r1 += d0_; r4 += d1_; } // t872 mm8 a=r0 b=r6 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r2 += d0_; r3 += d1_; } // t873 mm8 a=r2 b=r1 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r5 += d0_; r6 += d1_; } // t874 mm8 a=r5 b=r1 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r2 += d0_; r5 += d1_; } // t875 mm8 a=r1 b=r7 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r7 += d0_; r1 += d1_; } // t876 mm8 a=r7 b=r6 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r6 += d0_; r1 += d1_; } // t877 mm8 a=r6 b=r2 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r7 += d0_; r4 += d1_; } // t878 mm8 a=r5 b=r0 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r7 += d0_; r2 += d1_; } // t879 mm8 a=r4 b=r1 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r7 += d0_; r1 += d1_; } // t880 mm8 a=r3 b=r3 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r0 += d0_; r5 += d1_; } // t881 mm8 a=r2 b=r7 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r1 += d0_; r4 += d1_; } // t882 mm8 a=r7 b=r5 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r1 += d0_; r2 += d1_; } // t883 mm8 a=r2 b=r3 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r5 += d0_; r2 += d1_; } // t884 mm8 a=r2 b=r7 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r4 += d0_; r0 += d1_; } // t885 mm8 a=r3 b=r5 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r5 += d0_; r3 += d1_; } // t886 mm8 a=r5 b=r4 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r4 += d0_; r7 += d1_; } // t887 mm8 a=r1 b=r1 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r7 += d0_; r3 += d1_; } // t888 mm8 a=r6 b=r2 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r4 += d0_; r5 += d1_; } // t889 mm8 a=r4 b=r6 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r6 += d0_; r2 += d1_; } // t890 mm8 a=r0 b=r5 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r2 += d0_; r0 += d1_; } // t891 mm8 a=r0 b=r7 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r7 += d0_; r4 += d1_; } // t892 mm8 a=r5 b=r6 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r0 += d0_; r1 += d1_; } // t893 mm8 a=r2 b=r1 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r3 += d0_; r7 += d1_; } // t894 mm8 a=r0 b=r7 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r5 += d0_; r0 += d1_; } // t895 mm8 a=r2 b=r4 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r2 += d0_; r5 += d1_; } // t896 mm8 a=r0 b=r2 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r6 += d0_; r0 += d1_; } // t897 mm8 a=r7 b=r4 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r3 += d0_; r0 += d1_; } // t898 mm8 a=r1 b=r3 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r1 += d0_; r3 += d1_; } // t899 mm8 a=r5 b=r1 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r7 += d0_; r0 += d1_; } // t900 mm8 a=r7 b=r7 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r5 += d1_; } // t901 mm8 a=r4 b=r6 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r5 += d0_; r6 += d1_; } // t902 mm8 a=r2 b=r1 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r1 += d0_; r6 += d1_; } // t903 mm8 a=r0 b=r3 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r6 += d1_; } // t904 mm8 a=r6 b=r7 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r6 += d0_; r1 += d1_; } // t905 mm8 a=r0 b=r4 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r2 += d0_; r3 += d1_; } // t906 mm8 a=r3 b=r7 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r0 += d0_; r1 += d1_; } // t907 mm8 a=r2 b=r4 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r4 += d0_; r3 += d1_; } // t908 mm8 a=r7 b=r1 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r7 += d0_; r5 += d1_; } // t909 mm8 a=r4 b=r5 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r4 += d0_; r5 += d1_; } // t910 mm8 a=r3 b=r7 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r3 += d0_; r5 += d1_; } // t911 mm8 a=r2 b=r6 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r4 += d0_; r5 += d1_; } // t912 mm8 a=r6 b=r1 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r2 += d0_; r6 += d1_; } // t913 mm8 a=r2 b=r3 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r5 += d0_; r4 += d1_; } // t914 mm8 a=r7 b=r4 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r3 += d0_; r5 += d1_; } // t915 mm8 a=r4 b=r3 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r3 += d0_; r6 += d1_; } // t916 mm8 a=r4 b=r2 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r6 += d0_; r7 += d1_; } // t917 mm8 a=r5 b=r5 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r3 += d0_; r7 += d1_; } // t918 mm8 a=r7 b=r2 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r6 += d0_; r7 += d1_; } // t919 mm8 a=r3 b=r4 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r2 += d0_; r6 += d1_; } // t920 mm8 a=r6 b=r5 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r0 += d0_; r2 += d1_; } // t921 mm8 a=r6 b=r5 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r4 += d0_; r7 += d1_; } // t922 mm8 a=r1 b=r0 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r6 += d0_; r7 += d1_; } // t923 mm8 a=r6 b=r3 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r7 += d0_; r6 += d1_; } // t924 mm8 a=r2 b=r0 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r3 += d0_; r4 += d1_; } // t925 mm8 a=r6 b=r6 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r2 += d0_; r0 += d1_; } // t926 mm8 a=r5 b=r0 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t927 mm8 a=r1 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r7 += d0_; r5 += d1_; } // t928 mm8 a=r5 b=r7 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r7 += d0_; r5 += d1_; } // t929 mm8 a=r1 b=r0 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t930 mm8 a=r7 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t931 mm8 a=r7 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r6 += d0_; r4 += d1_; } // t932 mm8 a=r1 b=r3 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r0 += d0_; r3 += d1_; } // t933 mm8 a=r1 b=r3 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r6 += d0_; r0 += d1_; } // t934 mm8 a=r6 b=r0 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r6 += d0_; r7 += d1_; } // t935 mm8 a=r5 b=r0 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r1 += d0_; r6 += d1_; } // t936 mm8 a=r3 b=r3 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r7 += d0_; r5 += d1_; } // t937 mm8 a=r0 b=r6 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r1 += d0_; r3 += d1_; } // t938 mm8 a=r1 b=r5 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r3 += d0_; r0 += d1_; } // t939 mm8 a=r7 b=r0 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r4 += d0_; r5 += d1_; } // t940 mm8 a=r1 b=r2 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r2 += d0_; r7 += d1_; } // t941 mm8 a=r1 b=r6 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r1 += d0_; r3 += d1_; } // t942 mm8 a=r0 b=r6 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r1 += d0_; r4 += d1_; } // t943 mm8 a=r4 b=r0 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r2 += d0_; r1 += d1_; } // t944 mm8 a=r6 b=r5 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r4 += d0_; r2 += d1_; } // t945 mm8 a=r3 b=r7 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t946 mm8 a=r2 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r5 += d0_; r3 += d1_; } // t947 mm8 a=r7 b=r2 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t948 mm8 a=r6 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r0 += d0_; r6 += d1_; } // t949 mm8 a=r6 b=r6 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t950 mm8 a=r7 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t951 mm8 a=r2 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r1 += d0_; r2 += d1_; } // t952 mm8 a=r3 b=r2 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r0 += d0_; r5 += d1_; } // t953 mm8 a=r4 b=r4 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r2 += d1_; } // t954 mm8 a=r2 b=r4 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r0 += d0_; r7 += d1_; } // t955 mm8 a=r0 b=r7 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t956 mm8 a=r7 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t957 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r7 += d0_; r4 += d1_; } // t958 mm8 a=r6 b=r5 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r1 += d0_; r5 += d1_; } // t959 mm8 a=r4 b=r7 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r4 += d0_; r2 += d1_; } // t960 mm8 a=r1 b=r1 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r6 += d0_; r3 += d1_; } // t961 mm8 a=r1 b=r5 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r0 += d0_; r2 += d1_; } // t962 mm8 a=r6 b=r4 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r4 += d0_; r7 += d1_; } // t963 mm8 a=r7 b=r1 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r1 += d0_; r0 += d1_; } // t964 mm8 a=r2 b=r5 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t965 mm8 a=r3 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r6 += d0_; r0 += d1_; } // t966 mm8 a=r7 b=r3 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r7 += d0_; r1 += d1_; } // t967 mm8 a=r0 b=r0 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r5 += d0_; r7 += d1_; } // t968 mm8 a=r5 b=r7 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r6 += d0_; r1 += d1_; } // t969 mm8 a=r4 b=r4 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r0 += d0_; r7 += d1_; } // t970 mm8 a=r4 b=r3 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r7 += d0_; r3 += d1_; } // t971 mm8 a=r3 b=r4 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r4 += d0_; r7 += d1_; } // t972 mm8 a=r1 b=r7 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r1 += d0_; r7 += d1_; } // t973 mm8 a=r4 b=r1 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r3 += d0_; r2 += d1_; } // t974 mm8 a=r5 b=r6 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r6 += d0_; r4 += d1_; } // t975 mm8 a=r1 b=r2 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r7 += d0_; r6 += d1_; } // t976 mm8 a=r0 b=r6 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r4 += d0_; r0 += d1_; } // t977 mm8 a=r2 b=r0 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r2 += d0_; r6 += d1_; } // t978 mm8 a=r0 b=r2 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r3 += d0_; r6 += d1_; } // t979 mm8 a=r4 b=r0 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r2 += d0_; r7 += d1_; } // t980 mm8 a=r3 b=r1 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r2 += d0_; r1 += d1_; } // t981 mm8 a=r2 b=r0 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t982 mm8 a=r1 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r7 += d0_; r4 += d1_; } // t983 mm8 a=r2 b=r1 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r7 += d0_; r1 += d1_; } // t984 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r1 += d0_; r5 += d1_; } // t985 mm8 a=r4 b=r3 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r5 += d1_; } // t986 mm8 a=r2 b=r7 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r7 += d0_; r3 += d1_; } // t987 mm8 a=r5 b=r1 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r3 += d0_; r7 += d1_; } // t988 mm8 a=r6 b=r0 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r1 += d0_; r3 += d1_; } // t989 mm8 a=r2 b=r3 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r4 += d0_; r3 += d1_; } // t990 mm8 a=r1 b=r5 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r1 += d0_; r0 += d1_; } // t991 mm8 a=r6 b=r0 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r0 += d0_; r4 += d1_; } // t992 mm8 a=r2 b=r1 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r1 += d0_; r7 += d1_; } // t993 mm8 a=r6 b=r0 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r6 += d0_; r2 += d1_; } // t994 mm8 a=r6 b=r0 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r1 += d0_; r3 += d1_; } // t995 mm8 a=r0 b=r3 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r6 += d0_; r0 += d1_; } // t996 mm8 a=r7 b=r0 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r0 += d0_; r6 += d1_; } // t997 mm8 a=r5 b=r5 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r1 += d0_; r7 += d1_; } // t998 mm8 a=r3 b=r4 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r4 += d0_; r7 += d1_; } // t999 mm8 a=r7 b=r2 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r5 += d0_; r3 += d1_; } // t1000 mm8 a=r5 b=r3 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r0 += d0_; r5 += d1_; } // t1001 mm8 a=r5 b=r6 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r1 += d0_; r4 += d1_; } // t1002 mm8 a=r6 b=r2 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r1 += d0_; r5 += d1_; } // t1003 mm8 a=r4 b=r2 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r4 += d0_; r5 += d1_; } // t1004 mm8 a=r7 b=r3 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r4 += d0_; r1 += d1_; } // t1005 mm8 a=r1 b=r7 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r2 += d0_; r7 += d1_; } // t1006 mm8 a=r4 b=r2 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r4 += d0_; r5 += d1_; } // t1007 mm8 a=r0 b=r2 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r5 += d0_; r7 += d1_; } // t1008 mm8 a=r3 b=r5 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r6 += d0_; r7 += d1_; } // t1009 mm8 a=r0 b=r7 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r2 += d0_; r5 += d1_; } // t1010 mm8 a=r6 b=r7 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r5 += d0_; r2 += d1_; } // t1011 mm8 a=r7 b=r5 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r1 += d0_; r5 += d1_; } // t1012 mm8 a=r3 b=r5 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r6 += d0_; r1 += d1_; } // t1013 mm8 a=r6 b=r6 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r2 += d0_; r3 += d1_; } // t1014 mm8 a=r7 b=r3 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r2 += d0_; r6 += d1_; } // t1015 mm8 a=r7 b=r6 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r1 += d0_; r6 += d1_; } // t1016 mm8 a=r4 b=r0 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r6 += d0_; r3 += d1_; } // t1017 mm8 a=r5 b=r2 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r2 += d0_; r6 += d1_; } // t1018 mm8 a=r7 b=r6 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r5 += d0_; r4 += d1_; } // t1019 mm8 a=r0 b=r7 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r5 += d0_; r4 += d1_; } // t1020 mm8 a=r5 b=r6 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r3 += d0_; r4 += d1_; } // t1021 mm8 a=r7 b=r6 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r2 += d0_; r6 += d1_; } // t1022 mm8 a=r3 b=r5 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r6 += d0_; r7 += d1_; } // t1023 mm8 a=r0 b=r3 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r2 += d0_; r1 += d1_; } // t1024 mm8 a=r2 b=r6 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r2 += d0_; r0 += d1_; } // t1025 mm8 a=r4 b=r7 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r2 += d0_; r4 += d1_; } // t1026 mm8 a=r0 b=r1 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r5 += d0_; r0 += d1_; } // t1027 mm8 a=r1 b=r5 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r3 += d0_; r6 += d1_; } // t1028 mm8 a=r7 b=r6 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r0 += d0_; r7 += d1_; } // t1029 mm8 a=r2 b=r1 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r0 += d0_; r2 += d1_; } // t1030 mm8 a=r1 b=r7 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r0 += d0_; r3 += d1_; } // t1031 mm8 a=r3 b=r7 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r7 += d0_; r2 += d1_; } // t1032 mm8 a=r2 b=r1 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r5 += d0_; r6 += d1_; } // t1033 mm8 a=r2 b=r7 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r4 += d0_; r3 += d1_; } // t1034 mm8 a=r4 b=r3 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r5 += d0_; r6 += d1_; } // t1035 mm8 a=r3 b=r0 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r7 += d0_; r5 += d1_; } // t1036 mm8 a=r3 b=r0 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r4 += d0_; r2 += d1_; } // t1037 mm8 a=r7 b=r1 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r1 += d0_; r3 += d1_; } // t1038 mm8 a=r2 b=r1 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t1039 mm8 a=r5 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r3 += d0_; r7 += d1_; } // t1040 mm8 a=r0 b=r7 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r7 += d0_; r1 += d1_; } // t1041 mm8 a=r2 b=r2 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r2 += d0_; r4 += d1_; } // t1042 mm8 a=r6 b=r6 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r3 += d0_; r6 += d1_; } // t1043 mm8 a=r6 b=r7 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r7 += d0_; r1 += d1_; } // t1044 mm8 a=r4 b=r2 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r1 += d0_; r6 += d1_; } // t1045 mm8 a=r0 b=r0 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r3 += d0_; r2 += d1_; } // t1046 mm8 a=r7 b=r1 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r5 += d0_; r6 += d1_; } // t1047 mm8 a=r5 b=r7 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r6 += d0_; r0 += d1_; } // t1048 mm8 a=r2 b=r3 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r6 += d0_; r7 += d1_; } // t1049 mm8 a=r5 b=r1 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r0 += d1_; } // t1050 mm8 a=r0 b=r5 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r4 += d0_; r2 += d1_; } // t1051 mm8 a=r7 b=r7 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t1052 mm8 a=r2 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r2 += d0_; r0 += d1_; } // t1053 mm8 a=r6 b=r0 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r4 += d0_; r5 += d1_; } // t1054 mm8 a=r4 b=r2 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r7 += d0_; r2 += d1_; } // t1055 mm8 a=r5 b=r3 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r5 += d0_; r6 += d1_; } // t1056 mm8 a=r5 b=r1 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r5 += d0_; r4 += d1_; } // t1057 mm8 a=r1 b=r5 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r4 += d0_; r3 += d1_; } // t1058 mm8 a=r0 b=r5 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r4 += d0_; r5 += d1_; } // t1059 mm8 a=r5 b=r6 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r5 += d0_; r0 += d1_; } // t1060 mm8 a=r7 b=r3 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r6 += d0_; r1 += d1_; } // t1061 mm8 a=r5 b=r6 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r2 += d0_; r0 += d1_; } // t1062 mm8 a=r3 b=r0 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r7 += d0_; r0 += d1_; } // t1063 mm8 a=r1 b=r1 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r2 += d0_; r1 += d1_; } // t1064 mm8 a=r5 b=r4 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r4 += d0_; r5 += d1_; } // t1065 mm8 a=r3 b=r1 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r4 += d0_; r5 += d1_; } // t1066 mm8 a=r7 b=r7 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r6 += d0_; r1 += d1_; } // t1067 mm8 a=r0 b=r5 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r4 += d0_; r6 += d1_; } // t1068 mm8 a=r4 b=r6 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r6 += d0_; r5 += d1_; } // t1069 mm8 a=r2 b=r0 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r2 += d0_; r1 += d1_; } // t1070 mm8 a=r2 b=r3 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r1 += d0_; r7 += d1_; } // t1071 mm8 a=r2 b=r6 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r2 += d0_; r4 += d1_; } // t1072 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r2 += d0_; r0 += d1_; } // t1073 mm8 a=r3 b=r6 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r7 += d0_; r1 += d1_; } // t1074 mm8 a=r0 b=r0 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r7 += d0_; r1 += d1_; } // t1075 mm8 a=r4 b=r4 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r3 += d0_; r6 += d1_; } // t1076 mm8 a=r1 b=r7 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t1077 mm8 a=r7 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r1 += d0_; r3 += d1_; } // t1078 mm8 a=r3 b=r7 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r7 += d0_; r4 += d1_; } // t1079 mm8 a=r6 b=r6 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r1 += d0_; r4 += d1_; } // t1080 mm8 a=r4 b=r5 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r4 += d0_; r5 += d1_; } // t1081 mm8 a=r1 b=r5 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r3 += d0_; r6 += d1_; } // t1082 mm8 a=r5 b=r4 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r6 += d0_; r7 += d1_; } // t1083 mm8 a=r2 b=r2 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r2 += d0_; r1 += d1_; } // t1084 mm8 a=r2 b=r0 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r0 += d0_; r4 += d1_; } // t1085 mm8 a=r3 b=r3 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r3 += d0_; r2 += d1_; } // t1086 mm8 a=r3 b=r1 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r6 += d0_; r5 += d1_; } // t1087 mm8 a=r1 b=r3 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r5 += d0_; r7 += d1_; } // t1088 mm8 a=r3 b=r5 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r2 += d0_; r3 += d1_; } // t1089 mm8 a=r0 b=r0 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r7 += d0_; r2 += d1_; } // t1090 mm8 a=r4 b=r6 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r6 += d0_; r4 += d1_; } // t1091 mm8 a=r4 b=r3 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r1 += d0_; r6 += d1_; } // t1092 mm8 a=r2 b=r6 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r0 += d0_; r4 += d1_; } // t1093 mm8 a=r5 b=r7 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r2 += d0_; r6 += d1_; } // t1094 mm8 a=r7 b=r3 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r4 += d0_; r6 += d1_; } // t1095 mm8 a=r0 b=r2 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r6 += d0_; r1 += d1_; } // t1096 mm8 a=r3 b=r3 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r5 += d0_; r4 += d1_; } // t1097 mm8 a=r6 b=r2 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r1 += d0_; r6 += d1_; } // t1098 mm8 a=r3 b=r6 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r4 += d0_; r0 += d1_; } // t1099 mm8 a=r2 b=r5 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r1 += d0_; r0 += d1_; } // t1100 mm8 a=r1 b=r4 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r5 += d0_; r7 += d1_; } // t1101 mm8 a=r0 b=r6 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r6 += d0_; r5 += d1_; } // t1102 mm8 a=r0 b=r5 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t1103 mm8 a=r7 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r7 += d0_; r4 += d1_; } // t1104 mm8 a=r4 b=r2 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r2 += d0_; r0 += d1_; } // t1105 mm8 a=r3 b=r4 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r7 += d0_; r0 += d1_; } // t1106 mm8 a=r2 b=r1 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r6 += d0_; r5 += d1_; } // t1107 mm8 a=r3 b=r0 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r4 += d0_; r2 += d1_; } // t1108 mm8 a=r3 b=r5 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r4 += d0_; r3 += d1_; } // t1109 mm8 a=r6 b=r2 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r4 += d0_; r7 += d1_; } // t1110 mm8 a=r2 b=r4 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r2 += d0_; r1 += d1_; } // t1111 mm8 a=r0 b=r7 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r3 += d0_; r0 += d1_; } // t1112 mm8 a=r1 b=r0 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r5 += d0_; r6 += d1_; } // t1113 mm8 a=r4 b=r6 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r0 += d0_; r6 += d1_; } // t1114 mm8 a=r0 b=r5 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r3 += d0_; r4 += d1_; } // t1115 mm8 a=r4 b=r5 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r7 += d0_; r5 += d1_; } // t1116 mm8 a=r4 b=r6 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r0 += d0_; r7 += d1_; } // t1117 mm8 a=r0 b=r6 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r5 += d0_; r6 += d1_; } // t1118 mm8 a=r7 b=r6 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r0 += d0_; r5 += d1_; } // t1119 mm8 a=r2 b=r7 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r0 += d0_; r5 += d1_; } // t1120 mm8 a=r7 b=r5 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r6 += d0_; r7 += d1_; } // t1121 mm8 a=r4 b=r0 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r3 += d0_; r2 += d1_; } // t1122 mm8 a=r0 b=r5 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t1123 mm8 a=r0 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r4 += d0_; r0 += d1_; } // t1124 mm8 a=r0 b=r1 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r2 += d0_; r0 += d1_; } // t1125 mm8 a=r6 b=r6 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r0 += d0_; r3 += d1_; } // t1126 mm8 a=r0 b=r0 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r7 += d0_; r1 += d1_; } // t1127 mm8 a=r1 b=r1 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r7 += d0_; r3 += d1_; } // t1128 mm8 a=r4 b=r1 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r6 += d0_; r2 += d1_; } // t1129 mm8 a=r0 b=r0 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r7 += d0_; r5 += d1_; } // t1130 mm8 a=r0 b=r4 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r3 += d0_; r0 += d1_; } // t1131 mm8 a=r3 b=r0 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r5 += d0_; r7 += d1_; } // t1132 mm8 a=r0 b=r6 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r2 += d0_; r1 += d1_; } // t1133 mm8 a=r6 b=r0 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r1 += d0_; r4 += d1_; } // t1134 mm8 a=r7 b=r3 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r6 += d0_; r4 += d1_; } // t1135 mm8 a=r6 b=r7 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r7 += d0_; r2 += d1_; } // t1136 mm8 a=r7 b=r4 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r5 += d0_; r7 += d1_; } // t1137 mm8 a=r4 b=r0 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r3 += d0_; r4 += d1_; } // t1138 mm8 a=r6 b=r4 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r1 += d0_; r7 += d1_; } // t1139 mm8 a=r6 b=r2 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r6 += d0_; r3 += d1_; } // t1140 mm8 a=r1 b=r2 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r6 += d1_; } // t1141 mm8 a=r0 b=r5 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r2 += d0_; r5 += d1_; } // t1142 mm8 a=r4 b=r5 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r6 += d0_; r1 += d1_; } // t1143 mm8 a=r5 b=r6 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r7 += d0_; r3 += d1_; } // t1144 mm8 a=r6 b=r4 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r2 += d0_; r7 += d1_; } // t1145 mm8 a=r5 b=r1 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r5 += d0_; r3 += d1_; } // t1146 mm8 a=r6 b=r5 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r4 += d0_; r6 += d1_; } // t1147 mm8 a=r7 b=r1 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r4 += d0_; r6 += d1_; } // t1148 mm8 a=r1 b=r6 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r5 += d0_; r7 += d1_; } // t1149 mm8 a=r5 b=r5 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r5 += d0_; r1 += d1_; } // t1150 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r7 += d0_; r5 += d1_; } // t1151 mm8 a=r7 b=r5 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r5 += d0_; r3 += d1_; } // t1152 mm8 a=r0 b=r6 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r0 += d0_; r3 += d1_; } // t1153 mm8 a=r2 b=r5 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r7 += d0_; r0 += d1_; } // t1154 mm8 a=r5 b=r3 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r0 += d0_; r4 += d1_; } // t1155 mm8 a=r0 b=r6 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r4 += d0_; r7 += d1_; } // t1156 mm8 a=r2 b=r6 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r1 += d0_; r2 += d1_; } // t1157 mm8 a=r6 b=r7 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r3 += d0_; r7 += d1_; } // t1158 mm8 a=r2 b=r3 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t1159 mm8 a=r1 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r2 += d0_; r7 += d1_; } // t1160 mm8 a=r5 b=r0 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r3 += d0_; r4 += d1_; } // t1161 mm8 a=r7 b=r0 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r0 += d0_; r4 += d1_; } // t1162 mm8 a=r5 b=r3 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r2 += d0_; r1 += d1_; } // t1163 mm8 a=r4 b=r3 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r2 += d0_; r0 += d1_; } // t1164 mm8 a=r3 b=r4 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r7 += d0_; r1 += d1_; } // t1165 mm8 a=r1 b=r2 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r7 += d0_; r4 += d1_; } // t1166 mm8 a=r4 b=r7 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r5 += d1_; } // t1167 mm8 a=r2 b=r7 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r4 += d0_; r0 += d1_; } // t1168 mm8 a=r2 b=r6 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r3 += d0_; r7 += d1_; } // t1169 mm8 a=r0 b=r4 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r3 += d0_; r4 += d1_; } // t1170 mm8 a=r2 b=r1 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r7 += d1_; } // t1171 mm8 a=r1 b=r7 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r0 += d0_; r4 += d1_; } // t1172 mm8 a=r0 b=r1 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t1173 mm8 a=r6 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r2 += d0_; r6 += d1_; } // t1174 mm8 a=r0 b=r1 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r3 += d0_; r1 += d1_; } // t1175 mm8 a=r0 b=r0 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r5 += d0_; r6 += d1_; } // t1176 mm8 a=r4 b=r5 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r5 += d0_; r2 += d1_; } // t1177 mm8 a=r7 b=r2 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r7 += d0_; r0 += d1_; } // t1178 mm8 a=r1 b=r5 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t1179 mm8 a=r0 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r5 += d0_; r2 += d1_; } // t1180 mm8 a=r7 b=r3 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r0 += d0_; r7 += d1_; } // t1181 mm8 a=r7 b=r3 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r7 += d0_; r3 += d1_; } // t1182 mm8 a=r1 b=r2 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t1183 mm8 a=r2 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r2 += d0_; r7 += d1_; } // t1184 mm8 a=r7 b=r0 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r4 += d0_; r5 += d1_; } // t1185 mm8 a=r5 b=r2 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r2 += d0_; r3 += d1_; } // t1186 mm8 a=r1 b=r0 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r2 += d0_; r7 += d1_; } // t1187 mm8 a=r5 b=r4 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r4 += d0_; r6 += d1_; } // t1188 mm8 a=r3 b=r0 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t1189 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r2 += d0_; r7 += d1_; } // t1190 mm8 a=r5 b=r0 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r5 += d0_; r1 += d1_; } // t1191 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r2 += d0_; r1 += d1_; } // t1192 mm8 a=r0 b=r2 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r5 += d0_; r1 += d1_; } // t1193 mm8 a=r3 b=r7 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r1 += d0_; r2 += d1_; } // t1194 mm8 a=r7 b=r2 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r0 += d0_; r4 += d1_; } // t1195 mm8 a=r1 b=r4 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r5 += d0_; r0 += d1_; } // t1196 mm8 a=r4 b=r6 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r3 += d0_; r4 += d1_; } // t1197 mm8 a=r5 b=r6 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r2 += d0_; r5 += d1_; } // t1198 mm8 a=r0 b=r6 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r7 += d0_; r2 += d1_; } // t1199 mm8 a=r5 b=r1 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r1 += d0_; r2 += d1_; } // t1200 mm8 a=r1 b=r3 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r2 += d0_; r0 += d1_; } // t1201 mm8 a=r6 b=r6 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r4 += d0_; r7 += d1_; } // t1202 mm8 a=r2 b=r4 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r4 += d0_; r2 += d1_; } // t1203 mm8 a=r0 b=r5 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r7 += d0_; r4 += d1_; } // t1204 mm8 a=r0 b=r2 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r7 += d0_; r3 += d1_; } // t1205 mm8 a=r4 b=r6 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r4 += d0_; r6 += d1_; } // t1206 mm8 a=r4 b=r6 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r7 += d0_; r6 += d1_; } // t1207 mm8 a=r3 b=r7 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r2 += d0_; r4 += d1_; } // t1208 mm8 a=r6 b=r1 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r2 += d0_; r4 += d1_; } // t1209 mm8 a=r5 b=r5 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r3 += d0_; r0 += d1_; } // t1210 mm8 a=r0 b=r0 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r2 += d0_; r3 += d1_; } // t1211 mm8 a=r7 b=r5 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r7 += d0_; r1 += d1_; } // t1212 mm8 a=r2 b=r6 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r0 += d0_; r1 += d1_; } // t1213 mm8 a=r6 b=r2 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r3 += d0_; r5 += d1_; } // t1214 mm8 a=r4 b=r2 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r1 += d0_; r7 += d1_; } // t1215 mm8 a=r7 b=r2 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r7 += d0_; r0 += d1_; } // t1216 mm8 a=r3 b=r3 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r3 += d0_; r5 += d1_; } // t1217 mm8 a=r5 b=r5 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r3 += d0_; r6 += d1_; } // t1218 mm8 a=r0 b=r7 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r0 += d0_; r4 += d1_; } // t1219 mm8 a=r2 b=r4 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r7 += d0_; r4 += d1_; } // t1220 mm8 a=r1 b=r7 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r3 += d0_; r2 += d1_; } // t1221 mm8 a=r5 b=r5 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r2 += d0_; r0 += d1_; } // t1222 mm8 a=r1 b=r3 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r7 += d0_; r5 += d1_; } // t1223 mm8 a=r0 b=r2 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r6 += d0_; r3 += d1_; } // t1224 mm8 a=r1 b=r1 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r6 += d0_; r3 += d1_; } // t1225 mm8 a=r4 b=r1 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r4 += d0_; r5 += d1_; } // t1226 mm8 a=r5 b=r6 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r1 += d0_; r6 += d1_; } // t1227 mm8 a=r1 b=r7 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r4 += d0_; r5 += d1_; } // t1228 mm8 a=r4 b=r2 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r0 += d0_; r6 += d1_; } // t1229 mm8 a=r5 b=r3 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r2 += d0_; r7 += d1_; } // t1230 mm8 a=r6 b=r2 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r5 += d0_; r3 += d1_; } // t1231 mm8 a=r4 b=r3 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r4 += d1_; } // t1232 mm8 a=r6 b=r7 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r4 += d0_; r2 += d1_; } // t1233 mm8 a=r0 b=r5 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r7 += d0_; r1 += d1_; } // t1234 mm8 a=r5 b=r4 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r3 += d0_; r5 += d1_; } // t1235 mm8 a=r5 b=r7 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r1 += d0_; r3 += d1_; } // t1236 mm8 a=r0 b=r7 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r4 += d0_; r1 += d1_; } // t1237 mm8 a=r7 b=r2 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r5 += d0_; r0 += d1_; } // t1238 mm8 a=r7 b=r0 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r3 += d0_; r2 += d1_; } // t1239 mm8 a=r3 b=r2 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r2 += d0_; r4 += d1_; } // t1240 mm8 a=r0 b=r5 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r3 += d0_; r7 += d1_; } // t1241 mm8 a=r5 b=r0 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r7 += d0_; r4 += d1_; } // t1242 mm8 a=r5 b=r7 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r6 += d0_; r7 += d1_; } // t1243 mm8 a=r6 b=r3 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t1244 mm8 a=r1 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r5 += d0_; r1 += d1_; } // t1245 mm8 a=r0 b=r6 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r1 += d0_; r4 += d1_; } // t1246 mm8 a=r5 b=r5 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r7 += d0_; r1 += d1_; } // t1247 mm8 a=r1 b=r6 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r2 += d0_; r7 += d1_; } // t1248 mm8 a=r4 b=r0 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r1 += d0_; r7 += d1_; } // t1249 mm8 a=r7 b=r6 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r6 += d0_; r1 += d1_; } // t1250 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r1 += d0_; r0 += d1_; } // t1251 mm8 a=r3 b=r7 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r2 += d0_; r7 += d1_; } // t1252 mm8 a=r5 b=r5 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r6 += d0_; r2 += d1_; } // t1253 mm8 a=r5 b=r5 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r3 += d0_; r5 += d1_; } // t1254 mm8 a=r0 b=r6 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t1255 mm8 a=r1 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r5 += d0_; r6 += d1_; } // t1256 mm8 a=r1 b=r3 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r7 += d0_; r3 += d1_; } // t1257 mm8 a=r0 b=r7 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r4 += d0_; r3 += d1_; } // t1258 mm8 a=r5 b=r4 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r5 += d0_; r2 += d1_; } // t1259 mm8 a=r0 b=r3 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r5 += d0_; r7 += d1_; } // t1260 mm8 a=r1 b=r5 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r4 += d0_; r2 += d1_; } // t1261 mm8 a=r6 b=r7 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r5 += d0_; r1 += d1_; } // t1262 mm8 a=r2 b=r5 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r4 += d0_; r1 += d1_; } // t1263 mm8 a=r3 b=r7 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r6 += d0_; r7 += d1_; } // t1264 mm8 a=r1 b=r6 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r3 += d0_; r6 += d1_; } // t1265 mm8 a=r2 b=r3 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r3 += d1_; } // t1266 mm8 a=r4 b=r6 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r7 += d0_; r3 += d1_; } // t1267 mm8 a=r6 b=r6 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r5 += d0_; r7 += d1_; } // t1268 mm8 a=r4 b=r5 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r7 += d0_; r4 += d1_; } // t1269 mm8 a=r5 b=r7 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r7 += d0_; r3 += d1_; } // t1270 mm8 a=r7 b=r3 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r6 += d0_; r0 += d1_; } // t1271 mm8 a=r7 b=r0 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r1 += d0_; r3 += d1_; } // t1272 mm8 a=r1 b=r3 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r7 += d0_; r5 += d1_; } // t1273 mm8 a=r3 b=r2 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r6 += d0_; r3 += d1_; } // t1274 mm8 a=r2 b=r0 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r0 += d0_; r5 += d1_; } // t1275 mm8 a=r7 b=r3 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r5 += d0_; r2 += d1_; } // t1276 mm8 a=r4 b=r3 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r2 += d0_; r4 += d1_; } // t1277 mm8 a=r5 b=r4 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r1 += d0_; r6 += d1_; } // t1278 mm8 a=r6 b=r3 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r7 += d0_; r1 += d1_; } // t1279 mm8 a=r0 b=r1 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r4 += d0_; r2 += d1_; } // t1280 mm8 a=r4 b=r1 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r2 += d0_; r4 += d1_; } // t1281 mm8 a=r6 b=r3 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r2 += d0_; r5 += d1_; } // t1282 mm8 a=r1 b=r7 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r7 += d0_; r1 += d1_; } // t1283 mm8 a=r6 b=r2 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r7 += d0_; r2 += d1_; } // t1284 mm8 a=r1 b=r6 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r1 += d0_; r4 += d1_; } // t1285 mm8 a=r3 b=r5 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r2 += d0_; r4 += d1_; } // t1286 mm8 a=r3 b=r1 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r5 += d0_; r3 += d1_; } // t1287 mm8 a=r4 b=r1 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r6 += d0_; r0 += d1_; } // t1288 mm8 a=r3 b=r6 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r2 += d0_; r5 += d1_; } // t1289 mm8 a=r6 b=r2 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r1 += d0_; r5 += d1_; } // t1290 mm8 a=r6 b=r5 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r7 += d0_; r6 += d1_; } // t1291 mm8 a=r5 b=r6 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r1 += d0_; r6 += d1_; } // t1292 mm8 a=r6 b=r6 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r0 += d0_; r4 += d1_; } // t1293 mm8 a=r6 b=r2 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r2 += d0_; r3 += d1_; } // t1294 mm8 a=r0 b=r5 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r2 += d0_; r6 += d1_; } // t1295 mm8 a=r5 b=r0 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r3 += d0_; r5 += d1_; } // t1296 mm8 a=r1 b=r0 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r6 += d0_; r1 += d1_; } // t1297 mm8 a=r0 b=r0 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r0 += d0_; r4 += d1_; } // t1298 mm8 a=r2 b=r6 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r7 += d0_; r4 += d1_; } // t1299 mm8 a=r7 b=r0 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r4 += d0_; r5 += d1_; } // t1300 mm8 a=r4 b=r1 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r2 += d0_; r5 += d1_; } // t1301 mm8 a=r4 b=r5 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r7 += d0_; r2 += d1_; } // t1302 mm8 a=r0 b=r4 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r7 += d0_; r1 += d1_; } // t1303 mm8 a=r4 b=r2 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r3 += d0_; r7 += d1_; } // t1304 mm8 a=r4 b=r7 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r2 += d0_; r4 += d1_; } // t1305 mm8 a=r3 b=r2 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t1306 mm8 a=r3 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r6 += d0_; r7 += d1_; } // t1307 mm8 a=r0 b=r2 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r7 += d0_; r1 += d1_; } // t1308 mm8 a=r2 b=r5 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r5 += d0_; r4 += d1_; } // t1309 mm8 a=r5 b=r7 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r4 += d0_; r7 += d1_; } // t1310 mm8 a=r4 b=r5 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r1 += d0_; r4 += d1_; } // t1311 mm8 a=r4 b=r7 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t1312 mm8 a=r3 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r7 += d0_; r0 += d1_; } // t1313 mm8 a=r5 b=r1 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r0 += d0_; r2 += d1_; } // t1314 mm8 a=r5 b=r2 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r3 += d0_; r7 += d1_; } // t1315 mm8 a=r1 b=r0 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r6 += d0_; r7 += d1_; } // t1316 mm8 a=r7 b=r3 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r0 += d0_; r3 += d1_; } // t1317 mm8 a=r2 b=r5 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r5 += d0_; r4 += d1_; } // t1318 mm8 a=r2 b=r3 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r0 += d0_; r5 += d1_; } // t1319 mm8 a=r7 b=r5 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r4 += d0_; r6 += d1_; } // t1320 mm8 a=r0 b=r3 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r6 += d0_; r1 += d1_; } // t1321 mm8 a=r1 b=r0 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r6 += d0_; r3 += d1_; } // t1322 mm8 a=r4 b=r0 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r0 += d0_; r3 += d1_; } // t1323 mm8 a=r5 b=r3 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r4 += d0_; r6 += d1_; } // t1324 mm8 a=r3 b=r0 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r1 += d0_; r0 += d1_; } // t1325 mm8 a=r6 b=r4 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r0 += d1_; } // t1326 mm8 a=r4 b=r6 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r0 += d0_; r4 += d1_; } // t1327 mm8 a=r1 b=r3 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r7 += d0_; r6 += d1_; } // t1328 mm8 a=r7 b=r3 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r6 += d0_; r2 += d1_; } // t1329 mm8 a=r3 b=r7 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r5 += d0_; r3 += d1_; } // t1330 mm8 a=r3 b=r6 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r2 += d0_; r3 += d1_; } // t1331 mm8 a=r0 b=r7 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r5 += d0_; r4 += d1_; } // t1332 mm8 a=r0 b=r7 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r5 += d0_; r1 += d1_; } // t1333 mm8 a=r3 b=r3 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r0 += d0_; r3 += d1_; } // t1334 mm8 a=r7 b=r5 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r7 += d0_; r6 += d1_; } // t1335 mm8 a=r5 b=r7 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r1 += d0_; r2 += d1_; } // t1336 mm8 a=r5 b=r5 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r6 += d0_; r7 += d1_; } // t1337 mm8 a=r6 b=r0 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r7 += d0_; r6 += d1_; } // t1338 mm8 a=r6 b=r4 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r6 += d0_; r0 += d1_; } // t1339 mm8 a=r4 b=r6 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r7 += d0_; r0 += d1_; } // t1340 mm8 a=r6 b=r1 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r1 += d0_; r4 += d1_; } // t1341 mm8 a=r0 b=r6 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r0 += d0_; r1 += d1_; } // t1342 mm8 a=r4 b=r6 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r2 += d0_; r1 += d1_; } // t1343 mm8 a=r5 b=r5 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r3 += d0_; r1 += d1_; } // t1344 mm8 a=r2 b=r6 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r4 += d0_; r2 += d1_; } // t1345 mm8 a=r7 b=r3 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r0 += d0_; r2 += d1_; } // t1346 mm8 a=r5 b=r0 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r4 += d0_; r2 += d1_; } // t1347 mm8 a=r2 b=r3 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r2 += d0_; r1 += d1_; } // t1348 mm8 a=r4 b=r3 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r3 += d0_; r1 += d1_; } // t1349 mm8 a=r0 b=r1 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r0 += d0_; r4 += d1_; } // t1350 mm8 a=r1 b=r6 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r2 += d1_; } // t1351 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r5 += d0_; r0 += d1_; } // t1352 mm8 a=r5 b=r4 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t1353 mm8 a=r2 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r7 += d0_; r0 += d1_; } // t1354 mm8 a=r7 b=r4 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r3 += d0_; r0 += d1_; } // t1355 mm8 a=r7 b=r4 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r5 += d0_; r3 += d1_; } // t1356 mm8 a=r4 b=r6 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r6 += d1_; } // t1357 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r7 += d0_; r1 += d1_; } // t1358 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t1359 mm8 a=r7 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t1360 mm8 a=r0 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r7 += d0_; r4 += d1_; } // t1361 mm8 a=r0 b=r5 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r4 += d0_; r0 += d1_; } // t1362 mm8 a=r0 b=r2 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r5 += d0_; r6 += d1_; } // t1363 mm8 a=r4 b=r4 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r0 += d0_; r2 += d1_; } // t1364 mm8 a=r7 b=r3 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r3 += d0_; r4 += d1_; } // t1365 mm8 a=r2 b=r4 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r1 += d0_; r5 += d1_; } // t1366 mm8 a=r4 b=r3 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r6 += d0_; r2 += d1_; } // t1367 mm8 a=r3 b=r3 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r5 += d0_; r6 += d1_; } // t1368 mm8 a=r4 b=r0 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r2 += d0_; r7 += d1_; } // t1369 mm8 a=r5 b=r3 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t1370 mm8 a=r0 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r0 += d0_; r3 += d1_; } // t1371 mm8 a=r0 b=r5 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r7 += d0_; r2 += d1_; } // t1372 mm8 a=r5 b=r6 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r3 += d0_; r2 += d1_; } // t1373 mm8 a=r7 b=r1 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t1374 mm8 a=r4 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r2 += d0_; r5 += d1_; } // t1375 mm8 a=r5 b=r2 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r5 += d0_; r2 += d1_; } // t1376 mm8 a=r5 b=r5 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r7 += d0_; r5 += d1_; } // t1377 mm8 a=r7 b=r3 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r2 += d0_; r6 += d1_; } // t1378 mm8 a=r4 b=r1 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r7 += d0_; r2 += d1_; } // t1379 mm8 a=r4 b=r0 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r3 += d0_; r6 += d1_; } // t1380 mm8 a=r0 b=r7 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r0 += d0_; r7 += d1_; } // t1381 mm8 a=r2 b=r0 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r4 += d0_; r7 += d1_; } // t1382 mm8 a=r7 b=r5 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r1 += d1_; } // t1383 mm8 a=r1 b=r5 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r2 += d0_; r6 += d1_; } // t1384 mm8 a=r5 b=r2 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r5 += d0_; r1 += d1_; } // t1385 mm8 a=r0 b=r7 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r0 += d0_; r1 += d1_; } // t1386 mm8 a=r7 b=r0 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r0 += d0_; r2 += d1_; } // t1387 mm8 a=r4 b=r3 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r7 += d0_; r5 += d1_; } // t1388 mm8 a=r4 b=r6 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r3 += d0_; r2 += d1_; } // t1389 mm8 a=r4 b=r2 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r0 += d0_; r3 += d1_; } // t1390 mm8 a=r5 b=r1 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r4 += d0_; r0 += d1_; } // t1391 mm8 a=r6 b=r1 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r0 += d0_; r6 += d1_; } // t1392 mm8 a=r5 b=r7 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r7 += d0_; r4 += d1_; } // t1393 mm8 a=r3 b=r4 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r3 += d0_; r5 += d1_; } // t1394 mm8 a=r4 b=r7 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r6 += d0_; r5 += d1_; } // t1395 mm8 a=r7 b=r1 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r6 += d0_; r1 += d1_; } // t1396 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t1397 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r4 += d0_; r3 += d1_; } // t1398 mm8 a=r3 b=r0 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t1399 mm8 a=r3 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r2 += d0_; r5 += d1_; } // t1400 mm8 a=r6 b=r4 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r3 += d0_; r5 += d1_; } // t1401 mm8 a=r0 b=r5 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r6 += d0_; r2 += d1_; } // t1402 mm8 a=r7 b=r5 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r4 += d0_; r7 += d1_; } // t1403 mm8 a=r0 b=r0 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r6 += d0_; r2 += d1_; } // t1404 mm8 a=r3 b=r6 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r2 += d0_; r3 += d1_; } // t1405 mm8 a=r0 b=r0 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r4 += d0_; r5 += d1_; } // t1406 mm8 a=r5 b=r3 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r3 += d0_; r1 += d1_; } // t1407 mm8 a=r7 b=r3 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r0 += d0_; r1 += d1_; } // t1408 mm8 a=r4 b=r7 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r6 += d0_; r2 += d1_; } // t1409 mm8 a=r4 b=r2 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r6 += d0_; r3 += d1_; } // t1410 mm8 a=r6 b=r0 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r4 += d1_; } // t1411 mm8 a=r1 b=r7 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r3 += d0_; r2 += d1_; } // t1412 mm8 a=r4 b=r3 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r4 += d0_; r2 += d1_; } // t1413 mm8 a=r4 b=r6 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r2 += d0_; r5 += d1_; } // t1414 mm8 a=r0 b=r7 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r7 += d0_; r5 += d1_; } // t1415 mm8 a=r3 b=r6 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r4 += d0_; r7 += d1_; } // t1416 mm8 a=r3 b=r2 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r0 += d0_; r2 += d1_; } // t1417 mm8 a=r2 b=r7 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r2 += d0_; r7 += d1_; } // t1418 mm8 a=r0 b=r6 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t1419 mm8 a=r0 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r5 += d0_; r1 += d1_; } // t1420 mm8 a=r1 b=r3 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r3 += d0_; r2 += d1_; } // t1421 mm8 a=r4 b=r0 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r6 += d0_; r2 += d1_; } // t1422 mm8 a=r6 b=r4 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r3 += d0_; r1 += d1_; } // t1423 mm8 a=r4 b=r6 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r2 += d0_; r5 += d1_; } // t1424 mm8 a=r6 b=r3 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r0 += d0_; r1 += d1_; } // t1425 mm8 a=r7 b=r3 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r5 += d0_; r4 += d1_; } // t1426 mm8 a=r0 b=r3 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r3 += d0_; r6 += d1_; } // t1427 mm8 a=r5 b=r6 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r5 += d0_; r7 += d1_; } // t1428 mm8 a=r0 b=r3 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r3 += d0_; r7 += d1_; } // t1429 mm8 a=r7 b=r1 c=r3 c2=r7 - } - uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((ulong)hi << 32) | (ulong)lo; -} - -#if IGNEUM_EXCHANGE != 0 -// Reports the sub-group size this device uses for a work-group of IGNEUM_GROUP items. host.c runs it only when the -// per-kernel query (clGetKernelSubGroupInfoKHR on igneum_hash) is unavailable; that query is preferred because a -// compiler may pick a different wave width per kernel (RDNA: wave32 or wave64). See WAVEFRONT.md. -IGNEUM_KERNEL_HASH void igneum_probe_subgroup(__global uint* out) { - if (get_local_id(0) == 0u) { out[0] = get_sub_group_size(); out[1] = get_num_sub_groups(); } -} -#endif - -// Header-bound variant (bind.rs): the init words come from initw, not SEEDW. Same body as igneum_hash. -IGNEUM_KERNEL_HASH void igneum_hash_bound(__global const uint* ds, __global ulong* out, uint baseNonce, uint mask, __global const uint* initw) { - uint gid = (uint)get_global_id(0); - uint lid = (uint)get_local_id(0); - uint nonce = baseNonce + gid; - uint r0, r1, r2, r3, r4, r5, r6, r7; - uint iw0 = initw[0], iw1 = initw[1], iw2 = initw[2], iw3 = initw[3], iw4 = initw[4], iw5 = initw[5], iw6 = initw[6], iw7 = initw[7]; -#if IGNEUM_EXCHANGE == 0 - IGNEUM_LOCAL_WORDS(xch, 2 * IGNEUM_GROUP); - uint xk = 0u; -#else - (void)lid; -#endif - uint lane = lid & 31u; - { uint x = nonce ^ iw0; x += 0x9e3779b9u * 1u; x = splitmix32(x); r0 = x ^ iw1; } - { uint x = nonce ^ iw1; x += 0x9e3779b9u * 2u; x = splitmix32(x); r1 = x ^ iw2; } - { uint x = nonce ^ iw2; x += 0x9e3779b9u * 3u; x = splitmix32(x); r2 = x ^ iw3; } - { uint x = nonce ^ iw3; x += 0x9e3779b9u * 4u; x = splitmix32(x); r3 = x ^ iw4; } - { uint x = nonce ^ iw4; x += 0x9e3779b9u * 5u; x = splitmix32(x); r4 = x ^ iw5; } - { uint x = nonce ^ iw5; x += 0x9e3779b9u * 6u; x = splitmix32(x); r5 = x ^ iw6; } - { uint x = nonce ^ iw6; x += 0x9e3779b9u * 7u; x = splitmix32(x); r6 = x ^ iw7; } - { uint x = nonce ^ iw7; x += 0x9e3779b9u * 8u; x = splitmix32(x); r7 = x ^ iw0; } - - for (uint it = 0u; it < 8u; ++it) { - uint sel = r0; - r2 = r3 * r4 + r2; // 0 mad - r2 = r1 * r1 + r2; // 1 mad - r2 = r3 * r2 + r2; // 2 mad - r3 = r3 ^ r5; // 3 xor - r7 = r7 ^ ds[r2 & mask]; // 4 load - r5 = r5 ^ ds[r7 & mask]; // 5 load - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 8u); r1 = r1 ^ t_; } // 6 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 8u); r7 = r7 ^ t_; } // 7 shfl - r1 = mul_hi(r1, r5); // 8 mulhi - r6 = rotr_var(r6, r3); // 9 rotr - r3 = r3 | r4; // 10 or - r4 = r4 ^ ds[r3 & mask]; // 11 load - r0 = mul_hi(r0, r4); // 12 mulhi - r5 = r5 + r1 + ((((sel >> 30u) & 1u) != 0u) ? 0xd3177981u : 0xc7934706u); // 13 add - r0 = r0 ^ ds[r4 & mask]; // 14 load - r2 = r2 - r4; // 15 sub - r2 = r2 ^ ds[r0 & mask]; // 16 load - r7 = r7 ^ ds[r2 & mask]; // 17 load - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 4u); r7 = r7 ^ t_; } // 18 shfl - r5 = r5 * r0; // 19 mul - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 2u); r3 = r3 ^ t_; } // 20 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 16u); r2 = r2 ^ t_; } // 21 shfl - r6 = mul_hi(r6, r2); // 22 mulhi - r6 = r6 ^ ds[r1 & mask]; // 23 load - r5 = r5 * r0; // 24 mul - r5 = rotl_imm(r5, 19u); // 25 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 2u); r7 = r7 ^ t_; } // 26 shfl - r0 = r0 ^ r5; // 27 xor - r0 = r0 ^ r4; // 28 xor - r3 = r3 - r0; // 29 sub - r5 = r5 * r1; // 30 mul - r7 = r7 ^ ds[r2 & mask]; // 31 load - r1 = r1 ^ ds[r0 & mask]; // 32 load - r5 = r5 ^ r6; // 33 xor - r5 = r5 ^ ds[r1 & mask]; // 34 load - r0 = mul_hi(r0, r5); // 35 mulhi - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 4u); r5 = r5 ^ t_; } // 36 shfl - r7 = r7 ^ ds[r0 & mask]; // 37 load - r3 = r3 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0x230c005cu : 0x75ba2fadu); // 38 add - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 4u); r1 = r1 ^ t_; } // 39 shfl - r2 = r2 ^ r5; // 40 xor - r3 = r6 * r3 + r3; // 41 mad - r6 = r6 - r7; // 42 sub - r7 = r7 ^ r0; // 43 xor - r1 = r1 ^ ds[r7 & mask]; // 44 load - r2 = r2 * r3; // 45 mul - r1 = mul_hi(r1, r5); // 46 mulhi - r4 = r4 - r3; // 47 sub - r2 = rotr_var(r2, r6); // 48 rotr - r3 = r3 ^ ds[r5 & mask]; // 49 load - r1 = r1 + r5 + ((((sel >> 7u) & 1u) != 0u) ? 0x1907970cu : 0x81b8bc2cu); // 50 add - r0 = r0 * r2; // 51 mul - r0 = r0 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x699fd448u : 0x4f92b968u); // 52 add - r1 = r1 + r0 + ((((sel >> 12u) & 1u) != 0u) ? 0x77b1520du : 0x2bb965afu); // 53 add - r7 = rotl_imm(r7, 14u); // 54 rotl - r3 = r3 + r7 + ((((sel >> 1u) & 1u) != 0u) ? 0xa54c55a0u : 0x7b0fe07au); // 55 add - r6 = r6 ^ ds[r7 & mask]; // 56 load - r1 = rotr_var(r1, r5); // 57 rotr - r5 = r5 ^ ds[r4 & mask]; // 58 load - r6 = r6 ^ ds[r2 & mask]; // 59 load - r3 = r5 * r0 + r3; // 60 mad - r5 = r5 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0xad7493e7u : 0xaf9dd72du); // 61 add - r4 = r4 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x1e07c3d9u : 0x89841d87u); // 62 add - r5 = rotl_imm(r5, 19u); // 63 rotl - // int8 tile block (Counter ASIC 4.0 research, experimental): 1430 mma.m8n8k16 u8 tiles per iteration, both outputs consumed - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r2 += d0_; r3 += d1_; } // t0 mm8 a=r6 b=r5 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r4 += d0_; r5 += d1_; } // t1 mm8 a=r1 b=r1 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r0 += d0_; r7 += d1_; } // t2 mm8 a=r2 b=r5 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r3 += d1_; } // t3 mm8 a=r1 b=r5 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r2 += d0_; r6 += d1_; } // t4 mm8 a=r2 b=r0 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r7 += d0_; r3 += d1_; } // t5 mm8 a=r2 b=r6 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t6 mm8 a=r1 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r0 += d0_; r4 += d1_; } // t7 mm8 a=r4 b=r6 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r1 += d0_; r5 += d1_; } // t8 mm8 a=r1 b=r1 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r4 += d0_; r6 += d1_; } // t9 mm8 a=r3 b=r7 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r0 += d0_; r4 += d1_; } // t10 mm8 a=r3 b=r0 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r0 += d0_; r6 += d1_; } // t11 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r6 += d0_; r0 += d1_; } // t12 mm8 a=r3 b=r1 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r5 += d1_; } // t13 mm8 a=r1 b=r5 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r3 += d0_; r4 += d1_; } // t14 mm8 a=r3 b=r3 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r5 += d0_; r4 += d1_; } // t15 mm8 a=r1 b=r0 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r0 += d0_; r4 += d1_; } // t16 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r7 += d1_; } // t17 mm8 a=r0 b=r5 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r2 += d0_; r0 += d1_; } // t18 mm8 a=r1 b=r1 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r2 += d0_; r4 += d1_; } // t19 mm8 a=r3 b=r3 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r6 += d0_; r1 += d1_; } // t20 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r6 += d0_; r0 += d1_; } // t21 mm8 a=r0 b=r3 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r4 += d1_; } // t22 mm8 a=r1 b=r7 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r0 += d0_; r3 += d1_; } // t23 mm8 a=r1 b=r2 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r0 += d0_; r2 += d1_; } // t24 mm8 a=r3 b=r4 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r3 += d0_; r7 += d1_; } // t25 mm8 a=r5 b=r0 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t26 mm8 a=r1 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r0 += d0_; r5 += d1_; } // t27 mm8 a=r3 b=r2 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r5 += d1_; } // t28 mm8 a=r2 b=r4 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r1 += d0_; r5 += d1_; } // t29 mm8 a=r3 b=r4 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r6 += d0_; r3 += d1_; } // t30 mm8 a=r6 b=r5 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t31 mm8 a=r1 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r3 += d0_; r0 += d1_; } // t32 mm8 a=r7 b=r6 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r4 += d0_; r6 += d1_; } // t33 mm8 a=r7 b=r1 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r5 += d0_; r1 += d1_; } // t34 mm8 a=r1 b=r0 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t35 mm8 a=r2 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r7 += d0_; r1 += d1_; } // t36 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r0 += d0_; r2 += d1_; } // t37 mm8 a=r1 b=r7 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r2 += d0_; r4 += d1_; } // t38 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r2 += d0_; r5 += d1_; } // t39 mm8 a=r6 b=r2 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r4 += d0_; r1 += d1_; } // t40 mm8 a=r6 b=r1 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r6 += d0_; r2 += d1_; } // t41 mm8 a=r6 b=r0 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r5 += d0_; r1 += d1_; } // t42 mm8 a=r7 b=r4 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r6 += d0_; r0 += d1_; } // t43 mm8 a=r6 b=r1 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r4 += d0_; r1 += d1_; } // t44 mm8 a=r0 b=r6 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r6 += d0_; r5 += d1_; } // t45 mm8 a=r6 b=r5 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r1 += d0_; r0 += d1_; } // t46 mm8 a=r6 b=r6 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r3 += d0_; r1 += d1_; } // t47 mm8 a=r7 b=r3 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t48 mm8 a=r7 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r1 += d0_; r7 += d1_; } // t49 mm8 a=r4 b=r5 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r4 += d0_; r5 += d1_; } // t50 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r0 += d0_; r3 += d1_; } // t51 mm8 a=r3 b=r4 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r7 += d0_; r4 += d1_; } // t52 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r1 += d0_; r3 += d1_; } // t53 mm8 a=r7 b=r7 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t54 mm8 a=r7 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r3 += d0_; r7 += d1_; } // t55 mm8 a=r1 b=r7 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t56 mm8 a=r4 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r6 += d0_; r7 += d1_; } // t57 mm8 a=r4 b=r3 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r6 += d0_; r0 += d1_; } // t58 mm8 a=r2 b=r3 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r7 += d1_; } // t59 mm8 a=r6 b=r7 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t60 mm8 a=r6 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r2 += d0_; r6 += d1_; } // t61 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r6 += d1_; } // t62 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t63 mm8 a=r1 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r5 += d0_; r3 += d1_; } // t64 mm8 a=r0 b=r5 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r2 += d0_; r4 += d1_; } // t65 mm8 a=r0 b=r1 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r1 += d0_; r3 += d1_; } // t66 mm8 a=r3 b=r5 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r3 += d0_; r4 += d1_; } // t67 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r2 += d1_; } // t68 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r4 += d1_; } // t69 mm8 a=r6 b=r7 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r1 += d0_; r5 += d1_; } // t70 mm8 a=r1 b=r7 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r1 += d0_; r0 += d1_; } // t71 mm8 a=r3 b=r0 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r3 += d0_; r6 += d1_; } // t72 mm8 a=r4 b=r0 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r0 += d0_; r2 += d1_; } // t73 mm8 a=r4 b=r1 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r3 += d0_; r4 += d1_; } // t74 mm8 a=r5 b=r1 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r7 += d0_; r3 += d1_; } // t75 mm8 a=r4 b=r4 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r1 += d0_; r7 += d1_; } // t76 mm8 a=r5 b=r6 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r3 += d0_; r2 += d1_; } // t77 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r0 += d0_; r5 += d1_; } // t78 mm8 a=r2 b=r5 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t79 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r3 += d0_; r4 += d1_; } // t80 mm8 a=r6 b=r7 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r1 += d0_; r6 += d1_; } // t81 mm8 a=r4 b=r3 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r1 += d0_; r7 += d1_; } // t82 mm8 a=r6 b=r0 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r1 += d0_; r6 += d1_; } // t83 mm8 a=r2 b=r2 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t84 mm8 a=r0 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t85 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r2 += d0_; r3 += d1_; } // t86 mm8 a=r3 b=r4 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r3 += d0_; r7 += d1_; } // t87 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r5 += d0_; r3 += d1_; } // t88 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r2 += d0_; r1 += d1_; } // t89 mm8 a=r4 b=r2 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r3 += d0_; r6 += d1_; } // t90 mm8 a=r6 b=r5 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r5 += d0_; r4 += d1_; } // t91 mm8 a=r6 b=r5 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r2 += d0_; r3 += d1_; } // t92 mm8 a=r2 b=r6 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r5 += d0_; r7 += d1_; } // t93 mm8 a=r2 b=r2 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r7 += d0_; r3 += d1_; } // t94 mm8 a=r2 b=r4 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r1 += d0_; r6 += d1_; } // t95 mm8 a=r5 b=r4 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r3 += d0_; r1 += d1_; } // t96 mm8 a=r0 b=r6 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r2 += d1_; } // t97 mm8 a=r0 b=r5 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t98 mm8 a=r2 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r0 += d0_; r2 += d1_; } // t99 mm8 a=r1 b=r4 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r2 += d0_; r5 += d1_; } // t100 mm8 a=r2 b=r4 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r7 += d1_; } // t101 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t102 mm8 a=r5 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t103 mm8 a=r4 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r3 += d0_; r7 += d1_; } // t104 mm8 a=r2 b=r0 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r0 += d0_; r5 += d1_; } // t105 mm8 a=r6 b=r1 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r0 += d0_; r5 += d1_; } // t106 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r4 += d0_; r2 += d1_; } // t107 mm8 a=r2 b=r1 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t108 mm8 a=r4 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r5 += d0_; r3 += d1_; } // t109 mm8 a=r2 b=r7 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r0 += d0_; r2 += d1_; } // t110 mm8 a=r1 b=r0 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r4 += d0_; r6 += d1_; } // t111 mm8 a=r6 b=r5 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r4 += d0_; r2 += d1_; } // t112 mm8 a=r5 b=r7 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t113 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t114 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r1 += d0_; r4 += d1_; } // t115 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r5 += d0_; r0 += d1_; } // t116 mm8 a=r7 b=r2 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r7 += d0_; r6 += d1_; } // t117 mm8 a=r7 b=r6 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r2 += d0_; r1 += d1_; } // t118 mm8 a=r0 b=r1 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r5 += d0_; r3 += d1_; } // t119 mm8 a=r4 b=r3 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r7 += d0_; r2 += d1_; } // t120 mm8 a=r5 b=r3 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r1 += d1_; } // t121 mm8 a=r2 b=r7 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r3 += d0_; r0 += d1_; } // t122 mm8 a=r0 b=r1 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r3 += d0_; r5 += d1_; } // t123 mm8 a=r5 b=r2 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t124 mm8 a=r5 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r7 += d0_; r2 += d1_; } // t125 mm8 a=r7 b=r2 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r3 += d0_; r2 += d1_; } // t126 mm8 a=r6 b=r3 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r1 += d1_; } // t127 mm8 a=r6 b=r7 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r4 += d0_; r1 += d1_; } // t128 mm8 a=r6 b=r7 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r6 += d0_; r3 += d1_; } // t129 mm8 a=r3 b=r2 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r2 += d0_; r1 += d1_; } // t130 mm8 a=r0 b=r4 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r3 += d0_; r1 += d1_; } // t131 mm8 a=r3 b=r5 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r0 += d0_; r6 += d1_; } // t132 mm8 a=r0 b=r7 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r0 += d0_; r1 += d1_; } // t133 mm8 a=r2 b=r6 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r3 += d0_; r1 += d1_; } // t134 mm8 a=r5 b=r6 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r4 += d0_; r5 += d1_; } // t135 mm8 a=r0 b=r2 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r1 += d0_; r5 += d1_; } // t136 mm8 a=r3 b=r1 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r4 += d0_; r6 += d1_; } // t137 mm8 a=r7 b=r0 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r4 += d0_; r2 += d1_; } // t138 mm8 a=r2 b=r2 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r2 += d0_; r7 += d1_; } // t139 mm8 a=r0 b=r4 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r6 += d1_; } // t140 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r1 += d0_; r2 += d1_; } // t141 mm8 a=r5 b=r5 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r6 += d0_; r2 += d1_; } // t142 mm8 a=r2 b=r2 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r7 += d0_; r1 += d1_; } // t143 mm8 a=r4 b=r3 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r1 += d0_; r7 += d1_; } // t144 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r6 += d0_; r5 += d1_; } // t145 mm8 a=r7 b=r2 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r7 += d1_; } // t146 mm8 a=r2 b=r7 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r3 += d0_; r0 += d1_; } // t147 mm8 a=r3 b=r1 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r0 += d0_; r6 += d1_; } // t148 mm8 a=r2 b=r1 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r7 += d0_; r1 += d1_; } // t149 mm8 a=r4 b=r6 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r7 += d0_; r0 += d1_; } // t150 mm8 a=r3 b=r6 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r2 += d0_; r4 += d1_; } // t151 mm8 a=r1 b=r4 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r5 += d0_; r1 += d1_; } // t152 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r2 += d0_; r3 += d1_; } // t153 mm8 a=r0 b=r3 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r0 += d0_; r4 += d1_; } // t154 mm8 a=r7 b=r3 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r7 += d0_; r5 += d1_; } // t155 mm8 a=r1 b=r3 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r2 += d0_; r3 += d1_; } // t156 mm8 a=r6 b=r3 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r4 += d0_; r5 += d1_; } // t157 mm8 a=r6 b=r4 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t158 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r5 += d0_; r2 += d1_; } // t159 mm8 a=r1 b=r0 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r5 += d0_; r7 += d1_; } // t160 mm8 a=r0 b=r1 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r7 += d0_; r0 += d1_; } // t161 mm8 a=r4 b=r0 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r6 += d0_; r1 += d1_; } // t162 mm8 a=r6 b=r6 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r7 += d0_; r3 += d1_; } // t163 mm8 a=r5 b=r3 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t164 mm8 a=r0 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r1 += d0_; r5 += d1_; } // t165 mm8 a=r6 b=r5 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r4 += d0_; r0 += d1_; } // t166 mm8 a=r6 b=r5 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r3 += d0_; r7 += d1_; } // t167 mm8 a=r4 b=r5 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r0 += d0_; r7 += d1_; } // t168 mm8 a=r3 b=r6 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r4 += d0_; r3 += d1_; } // t169 mm8 a=r6 b=r2 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t170 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r2 += d0_; r3 += d1_; } // t171 mm8 a=r6 b=r3 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r5 += d0_; r2 += d1_; } // t172 mm8 a=r7 b=r3 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t173 mm8 a=r0 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r5 += d0_; r6 += d1_; } // t174 mm8 a=r6 b=r6 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r3 += d0_; r4 += d1_; } // t175 mm8 a=r1 b=r6 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r7 += d0_; r5 += d1_; } // t176 mm8 a=r2 b=r4 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r7 += d0_; r2 += d1_; } // t177 mm8 a=r1 b=r7 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r7 += d0_; r3 += d1_; } // t178 mm8 a=r4 b=r2 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r5 += d0_; r1 += d1_; } // t179 mm8 a=r5 b=r2 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r2 += d0_; r1 += d1_; } // t180 mm8 a=r0 b=r4 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r2 += d0_; r4 += d1_; } // t181 mm8 a=r3 b=r5 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r4 += d0_; r3 += d1_; } // t182 mm8 a=r1 b=r1 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r0 += d0_; r7 += d1_; } // t183 mm8 a=r3 b=r1 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r0 += d0_; r7 += d1_; } // t184 mm8 a=r7 b=r4 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r6 += d0_; r3 += d1_; } // t185 mm8 a=r5 b=r0 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r7 += d0_; r6 += d1_; } // t186 mm8 a=r2 b=r5 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r0 += d0_; r7 += d1_; } // t187 mm8 a=r3 b=r0 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r5 += d0_; r0 += d1_; } // t188 mm8 a=r1 b=r4 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r5 += d0_; r4 += d1_; } // t189 mm8 a=r7 b=r2 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r0 += d1_; } // t190 mm8 a=r1 b=r5 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t191 mm8 a=r2 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t192 mm8 a=r5 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r4 += d0_; r6 += d1_; } // t193 mm8 a=r2 b=r1 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r0 += d0_; r5 += d1_; } // t194 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r4 += d1_; } // t195 mm8 a=r2 b=r7 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r2 += d0_; r5 += d1_; } // t196 mm8 a=r5 b=r6 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r6 += d0_; r5 += d1_; } // t197 mm8 a=r5 b=r3 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r7 += d0_; r1 += d1_; } // t198 mm8 a=r1 b=r0 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t199 mm8 a=r2 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t200 mm8 a=r5 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r2 += d0_; r1 += d1_; } // t201 mm8 a=r7 b=r2 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r5 += d0_; r6 += d1_; } // t202 mm8 a=r2 b=r5 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r6 += d1_; } // t203 mm8 a=r0 b=r5 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r3 += d0_; r7 += d1_; } // t204 mm8 a=r4 b=r7 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r1 += d0_; r3 += d1_; } // t205 mm8 a=r3 b=r7 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r7 += d0_; r3 += d1_; } // t206 mm8 a=r0 b=r6 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r1 += d0_; r4 += d1_; } // t207 mm8 a=r7 b=r7 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r6 += d0_; r5 += d1_; } // t208 mm8 a=r0 b=r1 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r6 += d0_; r2 += d1_; } // t209 mm8 a=r4 b=r4 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t210 mm8 a=r7 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r5 += d0_; r6 += d1_; } // t211 mm8 a=r1 b=r5 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r5 += d0_; r3 += d1_; } // t212 mm8 a=r2 b=r4 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r3 += d0_; r5 += d1_; } // t213 mm8 a=r2 b=r2 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r5 += d0_; r7 += d1_; } // t214 mm8 a=r4 b=r6 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r2 += d0_; r6 += d1_; } // t215 mm8 a=r7 b=r3 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r3 += d0_; r4 += d1_; } // t216 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r0 += d0_; r6 += d1_; } // t217 mm8 a=r5 b=r7 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r0 += d1_; } // t218 mm8 a=r7 b=r1 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r3 += d0_; r1 += d1_; } // t219 mm8 a=r2 b=r1 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r4 += d0_; r5 += d1_; } // t220 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r2 += d0_; r1 += d1_; } // t221 mm8 a=r7 b=r1 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r2 += d0_; r1 += d1_; } // t222 mm8 a=r0 b=r6 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r7 += d0_; r0 += d1_; } // t223 mm8 a=r2 b=r2 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r6 += d0_; r2 += d1_; } // t224 mm8 a=r1 b=r1 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r4 += d1_; } // t225 mm8 a=r2 b=r7 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t226 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r5 += d0_; r6 += d1_; } // t227 mm8 a=r3 b=r5 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r3 += d0_; r1 += d1_; } // t228 mm8 a=r7 b=r1 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r1 += d0_; r4 += d1_; } // t229 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r5 += d0_; r6 += d1_; } // t230 mm8 a=r0 b=r3 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r4 += d0_; r0 += d1_; } // t231 mm8 a=r1 b=r3 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r0 += d0_; r1 += d1_; } // t232 mm8 a=r5 b=r4 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r0 += d0_; r4 += d1_; } // t233 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r2 += d0_; r1 += d1_; } // t234 mm8 a=r2 b=r0 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t235 mm8 a=r3 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r0 += d0_; r5 += d1_; } // t236 mm8 a=r1 b=r7 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r6 += d0_; r3 += d1_; } // t237 mm8 a=r3 b=r5 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r0 += d0_; r2 += d1_; } // t238 mm8 a=r2 b=r0 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r1 += d0_; r0 += d1_; } // t239 mm8 a=r5 b=r7 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r4 += d0_; r6 += d1_; } // t240 mm8 a=r2 b=r7 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r3 += d0_; r7 += d1_; } // t241 mm8 a=r0 b=r1 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r2 += d0_; r7 += d1_; } // t242 mm8 a=r0 b=r2 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r1 += d0_; r5 += d1_; } // t243 mm8 a=r4 b=r4 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r0 += d0_; r7 += d1_; } // t244 mm8 a=r7 b=r6 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r2 += d0_; r0 += d1_; } // t245 mm8 a=r4 b=r4 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t246 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r2 += d0_; r4 += d1_; } // t247 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r7 += d0_; r3 += d1_; } // t248 mm8 a=r7 b=r5 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r0 += d0_; r5 += d1_; } // t249 mm8 a=r4 b=r4 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r0 += d0_; r1 += d1_; } // t250 mm8 a=r0 b=r6 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t251 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r5 += d0_; r4 += d1_; } // t252 mm8 a=r0 b=r2 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r7 += d0_; r4 += d1_; } // t253 mm8 a=r5 b=r5 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t254 mm8 a=r6 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r0 += d0_; r3 += d1_; } // t255 mm8 a=r5 b=r0 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r3 += d0_; r4 += d1_; } // t256 mm8 a=r2 b=r1 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r6 += d0_; r0 += d1_; } // t257 mm8 a=r4 b=r3 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r7 += d0_; r3 += d1_; } // t258 mm8 a=r4 b=r1 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r3 += d0_; r0 += d1_; } // t259 mm8 a=r5 b=r5 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r3 += d0_; r0 += d1_; } // t260 mm8 a=r3 b=r2 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r3 += d0_; r2 += d1_; } // t261 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r2 += d0_; r5 += d1_; } // t262 mm8 a=r6 b=r4 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r2 += d0_; r1 += d1_; } // t263 mm8 a=r6 b=r4 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r1 += d0_; r4 += d1_; } // t264 mm8 a=r7 b=r2 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r3 += d0_; r1 += d1_; } // t265 mm8 a=r6 b=r1 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r7 += d0_; r0 += d1_; } // t266 mm8 a=r0 b=r0 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r3 += d0_; r4 += d1_; } // t267 mm8 a=r6 b=r3 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r6 += d0_; r5 += d1_; } // t268 mm8 a=r1 b=r0 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r7 += d0_; r3 += d1_; } // t269 mm8 a=r7 b=r7 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r7 += d0_; r0 += d1_; } // t270 mm8 a=r6 b=r2 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r5 += d0_; r3 += d1_; } // t271 mm8 a=r0 b=r3 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r0 += d0_; r3 += d1_; } // t272 mm8 a=r6 b=r0 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r7 += d1_; } // t273 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r7 += d0_; r6 += d1_; } // t274 mm8 a=r1 b=r0 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r1 += d0_; r4 += d1_; } // t275 mm8 a=r1 b=r3 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r6 += d0_; r2 += d1_; } // t276 mm8 a=r1 b=r5 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r4 += d0_; r7 += d1_; } // t277 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r6 += d0_; r7 += d1_; } // t278 mm8 a=r0 b=r0 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r2 += d0_; r4 += d1_; } // t279 mm8 a=r2 b=r4 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r3 += d0_; r5 += d1_; } // t280 mm8 a=r7 b=r2 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r4 += d0_; r2 += d1_; } // t281 mm8 a=r1 b=r0 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r5 += d0_; r1 += d1_; } // t282 mm8 a=r0 b=r1 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r7 += d0_; r2 += d1_; } // t283 mm8 a=r0 b=r2 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r0 += d0_; r2 += d1_; } // t284 mm8 a=r1 b=r5 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r5 += d0_; r0 += d1_; } // t285 mm8 a=r4 b=r7 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r2 += d0_; r0 += d1_; } // t286 mm8 a=r1 b=r0 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r7 += d0_; r4 += d1_; } // t287 mm8 a=r7 b=r1 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t288 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r5 += d0_; r4 += d1_; } // t289 mm8 a=r0 b=r5 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r6 += d0_; r7 += d1_; } // t290 mm8 a=r3 b=r2 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r7 += d0_; r1 += d1_; } // t291 mm8 a=r7 b=r4 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t292 mm8 a=r6 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r6 += d0_; r4 += d1_; } // t293 mm8 a=r5 b=r5 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r1 += d0_; r6 += d1_; } // t294 mm8 a=r5 b=r7 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r1 += d0_; r3 += d1_; } // t295 mm8 a=r0 b=r7 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r5 += d0_; r6 += d1_; } // t296 mm8 a=r6 b=r1 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r3 += d0_; r5 += d1_; } // t297 mm8 a=r4 b=r5 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r3 += d0_; r1 += d1_; } // t298 mm8 a=r6 b=r0 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r2 += d0_; r3 += d1_; } // t299 mm8 a=r6 b=r0 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r4 += d0_; r2 += d1_; } // t300 mm8 a=r6 b=r4 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r6 += d0_; r3 += d1_; } // t301 mm8 a=r6 b=r4 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r6 += d0_; r4 += d1_; } // t302 mm8 a=r7 b=r6 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r3 += d0_; r6 += d1_; } // t303 mm8 a=r3 b=r2 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r2 += d0_; r4 += d1_; } // t304 mm8 a=r6 b=r5 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r0 += d0_; r2 += d1_; } // t305 mm8 a=r4 b=r5 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r0 += d0_; r2 += d1_; } // t306 mm8 a=r3 b=r1 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r5 += d0_; r0 += d1_; } // t307 mm8 a=r4 b=r3 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r2 += d0_; r6 += d1_; } // t308 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r1 += d0_; r4 += d1_; } // t309 mm8 a=r2 b=r1 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r2 += d0_; r3 += d1_; } // t310 mm8 a=r6 b=r1 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r3 += d0_; r2 += d1_; } // t311 mm8 a=r4 b=r6 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t312 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r7 += d0_; r1 += d1_; } // t313 mm8 a=r2 b=r6 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r2 += d0_; r1 += d1_; } // t314 mm8 a=r0 b=r3 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r6 += d0_; r2 += d1_; } // t315 mm8 a=r7 b=r1 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r1 += d0_; r6 += d1_; } // t316 mm8 a=r2 b=r4 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r3 += d0_; r6 += d1_; } // t317 mm8 a=r1 b=r6 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r1 += d0_; r0 += d1_; } // t318 mm8 a=r2 b=r1 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r6 += d0_; r3 += d1_; } // t319 mm8 a=r0 b=r2 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r6 += d0_; r2 += d1_; } // t320 mm8 a=r5 b=r2 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t321 mm8 a=r7 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r7 += d0_; r1 += d1_; } // t322 mm8 a=r2 b=r4 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r2 += d0_; r3 += d1_; } // t323 mm8 a=r4 b=r3 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r2 += d0_; r0 += d1_; } // t324 mm8 a=r5 b=r7 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r7 += d0_; r2 += d1_; } // t325 mm8 a=r6 b=r2 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r7 += d0_; r6 += d1_; } // t326 mm8 a=r6 b=r1 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r3 += d0_; r0 += d1_; } // t327 mm8 a=r1 b=r5 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r1 += d0_; r6 += d1_; } // t328 mm8 a=r0 b=r1 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r7 += d0_; r1 += d1_; } // t329 mm8 a=r0 b=r3 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r4 += d0_; r0 += d1_; } // t330 mm8 a=r6 b=r7 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t331 mm8 a=r4 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r6 += d0_; r3 += d1_; } // t332 mm8 a=r4 b=r4 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r4 += d0_; r2 += d1_; } // t333 mm8 a=r4 b=r6 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r0 += d0_; r1 += d1_; } // t334 mm8 a=r2 b=r2 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r2 += d1_; } // t335 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r2 += d0_; r3 += d1_; } // t336 mm8 a=r4 b=r0 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r0 += d0_; r3 += d1_; } // t337 mm8 a=r1 b=r4 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r2 += d0_; r7 += d1_; } // t338 mm8 a=r3 b=r7 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r7 += d0_; r6 += d1_; } // t339 mm8 a=r5 b=r1 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r7 += d0_; r5 += d1_; } // t340 mm8 a=r5 b=r0 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r7 += d0_; r5 += d1_; } // t341 mm8 a=r6 b=r6 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r0 += d0_; r4 += d1_; } // t342 mm8 a=r4 b=r7 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r4 += d0_; r6 += d1_; } // t343 mm8 a=r0 b=r4 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r4 += d0_; r6 += d1_; } // t344 mm8 a=r2 b=r5 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t345 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t346 mm8 a=r6 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r3 += d0_; r5 += d1_; } // t347 mm8 a=r5 b=r3 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r4 += d0_; r5 += d1_; } // t348 mm8 a=r0 b=r4 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r4 += d0_; r7 += d1_; } // t349 mm8 a=r6 b=r1 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r3 += d0_; r5 += d1_; } // t350 mm8 a=r1 b=r5 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r4 += d0_; r5 += d1_; } // t351 mm8 a=r6 b=r4 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r6 += d0_; r0 += d1_; } // t352 mm8 a=r4 b=r1 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r1 += d0_; r4 += d1_; } // t353 mm8 a=r7 b=r3 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r1 += d0_; r5 += d1_; } // t354 mm8 a=r4 b=r1 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r0 += d0_; r1 += d1_; } // t355 mm8 a=r4 b=r5 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r0 += d0_; r4 += d1_; } // t356 mm8 a=r7 b=r6 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r7 += d0_; r5 += d1_; } // t357 mm8 a=r7 b=r3 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r5 += d0_; r7 += d1_; } // t358 mm8 a=r4 b=r7 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r3 += d0_; r5 += d1_; } // t359 mm8 a=r2 b=r7 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r0 += d0_; r2 += d1_; } // t360 mm8 a=r4 b=r4 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r5 += d0_; r6 += d1_; } // t361 mm8 a=r2 b=r2 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r3 += d0_; r0 += d1_; } // t362 mm8 a=r2 b=r7 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r5 += d0_; r0 += d1_; } // t363 mm8 a=r7 b=r6 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r0 += d0_; r5 += d1_; } // t364 mm8 a=r6 b=r4 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r5 += d0_; r4 += d1_; } // t365 mm8 a=r6 b=r6 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r2 += d0_; r5 += d1_; } // t366 mm8 a=r7 b=r7 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t367 mm8 a=r4 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r3 += d0_; r5 += d1_; } // t368 mm8 a=r7 b=r0 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r6 += d0_; r0 += d1_; } // t369 mm8 a=r1 b=r0 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r7 += d0_; r4 += d1_; } // t370 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r2 += d0_; r6 += d1_; } // t371 mm8 a=r5 b=r4 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r2 += d1_; } // t372 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r2 += d0_; r4 += d1_; } // t373 mm8 a=r4 b=r6 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t374 mm8 a=r1 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r6 += d0_; r2 += d1_; } // t375 mm8 a=r6 b=r5 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t376 mm8 a=r3 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r6 += d0_; r4 += d1_; } // t377 mm8 a=r6 b=r4 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r2 += d0_; r4 += d1_; } // t378 mm8 a=r6 b=r3 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t379 mm8 a=r3 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r6 += d0_; r2 += d1_; } // t380 mm8 a=r2 b=r0 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r1 += d0_; r2 += d1_; } // t381 mm8 a=r3 b=r5 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r2 += d0_; r4 += d1_; } // t382 mm8 a=r5 b=r2 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r0 += d0_; r2 += d1_; } // t383 mm8 a=r1 b=r2 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r1 += d0_; r0 += d1_; } // t384 mm8 a=r3 b=r1 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r7 += d0_; r5 += d1_; } // t385 mm8 a=r6 b=r0 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r1 += d0_; r7 += d1_; } // t386 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r2 += d0_; r4 += d1_; } // t387 mm8 a=r3 b=r7 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r1 += d0_; r2 += d1_; } // t388 mm8 a=r3 b=r0 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r5 += d0_; r3 += d1_; } // t389 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r1 += d0_; r4 += d1_; } // t390 mm8 a=r5 b=r4 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t391 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r3 += d0_; r0 += d1_; } // t392 mm8 a=r2 b=r3 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r5 += d0_; r0 += d1_; } // t393 mm8 a=r3 b=r6 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r1 += d0_; r2 += d1_; } // t394 mm8 a=r0 b=r7 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r2 += d0_; r4 += d1_; } // t395 mm8 a=r0 b=r7 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r1 += d0_; r4 += d1_; } // t396 mm8 a=r7 b=r7 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t397 mm8 a=r5 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r7 += d0_; r0 += d1_; } // t398 mm8 a=r6 b=r5 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r3 += d0_; r5 += d1_; } // t399 mm8 a=r6 b=r3 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r0 += d0_; r4 += d1_; } // t400 mm8 a=r2 b=r7 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r2 += d0_; r1 += d1_; } // t401 mm8 a=r7 b=r6 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r7 += d0_; r1 += d1_; } // t402 mm8 a=r0 b=r0 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r1 += d0_; r0 += d1_; } // t403 mm8 a=r7 b=r4 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r1 += d0_; r4 += d1_; } // t404 mm8 a=r1 b=r1 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r3 += d0_; r6 += d1_; } // t405 mm8 a=r6 b=r6 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r4 += d0_; r3 += d1_; } // t406 mm8 a=r6 b=r1 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r2 += d0_; r4 += d1_; } // t407 mm8 a=r5 b=r7 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t408 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r4 += d0_; r7 += d1_; } // t409 mm8 a=r3 b=r4 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r5 += d0_; r2 += d1_; } // t410 mm8 a=r0 b=r5 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r5 += d0_; r1 += d1_; } // t411 mm8 a=r2 b=r0 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t412 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r2 += d1_; } // t413 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r0 += d0_; r3 += d1_; } // t414 mm8 a=r7 b=r1 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r5 += d0_; r1 += d1_; } // t415 mm8 a=r4 b=r7 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r3 += d0_; r5 += d1_; } // t416 mm8 a=r5 b=r4 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t417 mm8 a=r6 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r1 += d0_; r5 += d1_; } // t418 mm8 a=r1 b=r3 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r0 += d0_; r6 += d1_; } // t419 mm8 a=r5 b=r0 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r7 += d0_; r6 += d1_; } // t420 mm8 a=r4 b=r2 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r3 += d0_; r7 += d1_; } // t421 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t422 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r0 += d1_; } // t423 mm8 a=r1 b=r7 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r6 += d0_; r5 += d1_; } // t424 mm8 a=r5 b=r1 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t425 mm8 a=r2 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r5 += d0_; r0 += d1_; } // t426 mm8 a=r1 b=r6 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t427 mm8 a=r6 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r7 += d0_; r4 += d1_; } // t428 mm8 a=r6 b=r2 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r7 += d0_; r0 += d1_; } // t429 mm8 a=r3 b=r3 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r3 += d0_; r5 += d1_; } // t430 mm8 a=r7 b=r7 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r1 += d0_; r0 += d1_; } // t431 mm8 a=r6 b=r4 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r3 += d0_; r4 += d1_; } // t432 mm8 a=r3 b=r2 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r5 += d0_; r0 += d1_; } // t433 mm8 a=r3 b=r0 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r5 += d0_; r3 += d1_; } // t434 mm8 a=r0 b=r7 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r6 += d0_; r3 += d1_; } // t435 mm8 a=r7 b=r5 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r0 += d0_; r5 += d1_; } // t436 mm8 a=r6 b=r4 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r3 += d0_; r4 += d1_; } // t437 mm8 a=r0 b=r7 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r4 += d0_; r7 += d1_; } // t438 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t439 mm8 a=r4 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r3 += d0_; r2 += d1_; } // t440 mm8 a=r3 b=r5 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r1 += d0_; r4 += d1_; } // t441 mm8 a=r5 b=r1 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r7 += d0_; r5 += d1_; } // t442 mm8 a=r1 b=r2 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r5 += d0_; r2 += d1_; } // t443 mm8 a=r1 b=r7 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r1 += d0_; r0 += d1_; } // t444 mm8 a=r3 b=r7 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r7 += d0_; r0 += d1_; } // t445 mm8 a=r7 b=r5 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r2 += d0_; r1 += d1_; } // t446 mm8 a=r3 b=r1 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r4 += d0_; r7 += d1_; } // t447 mm8 a=r6 b=r7 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r4 += d0_; r5 += d1_; } // t448 mm8 a=r5 b=r2 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r3 += d1_; } // t449 mm8 a=r7 b=r1 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r7 += d0_; r3 += d1_; } // t450 mm8 a=r7 b=r5 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r1 += d0_; r7 += d1_; } // t451 mm8 a=r0 b=r3 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r7 += d0_; r1 += d1_; } // t452 mm8 a=r2 b=r4 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r7 += d0_; r1 += d1_; } // t453 mm8 a=r5 b=r7 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r7 += d0_; r1 += d1_; } // t454 mm8 a=r7 b=r7 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t455 mm8 a=r7 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r5 += d0_; r6 += d1_; } // t456 mm8 a=r6 b=r5 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r1 += d0_; r5 += d1_; } // t457 mm8 a=r7 b=r7 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r4 += d0_; r1 += d1_; } // t458 mm8 a=r5 b=r1 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r1 += d0_; r0 += d1_; } // t459 mm8 a=r6 b=r7 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t460 mm8 a=r7 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r6 += d0_; r2 += d1_; } // t461 mm8 a=r0 b=r6 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t462 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r3 += d0_; r2 += d1_; } // t463 mm8 a=r2 b=r0 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r0 += d0_; r1 += d1_; } // t464 mm8 a=r0 b=r3 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r0 += d0_; r5 += d1_; } // t465 mm8 a=r1 b=r7 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r4 += d0_; r1 += d1_; } // t466 mm8 a=r1 b=r2 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r1 += d0_; r6 += d1_; } // t467 mm8 a=r1 b=r0 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r6 += d0_; r2 += d1_; } // t468 mm8 a=r4 b=r1 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r0 += d0_; r1 += d1_; } // t469 mm8 a=r2 b=r0 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r2 += d0_; r0 += d1_; } // t470 mm8 a=r3 b=r1 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r7 += d0_; r1 += d1_; } // t471 mm8 a=r2 b=r2 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t472 mm8 a=r1 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r1 += d1_; } // t473 mm8 a=r2 b=r7 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r6 += d0_; r3 += d1_; } // t474 mm8 a=r5 b=r3 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r6 += d0_; r0 += d1_; } // t475 mm8 a=r7 b=r6 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r5 += d0_; r6 += d1_; } // t476 mm8 a=r3 b=r0 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r4 += d0_; r6 += d1_; } // t477 mm8 a=r3 b=r1 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r2 += d0_; r5 += d1_; } // t478 mm8 a=r0 b=r6 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r7 += d0_; r1 += d1_; } // t479 mm8 a=r7 b=r6 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r1 += d0_; r0 += d1_; } // t480 mm8 a=r6 b=r0 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r7 += d1_; } // t481 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r0 += d0_; r6 += d1_; } // t482 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r7 += d0_; r6 += d1_; } // t483 mm8 a=r7 b=r0 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r6 += d0_; r4 += d1_; } // t484 mm8 a=r3 b=r7 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r1 += d0_; r2 += d1_; } // t485 mm8 a=r6 b=r4 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r2 += d0_; r5 += d1_; } // t486 mm8 a=r6 b=r5 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r4 += d0_; r5 += d1_; } // t487 mm8 a=r2 b=r5 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r1 += d0_; r2 += d1_; } // t488 mm8 a=r1 b=r2 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r6 += d0_; r4 += d1_; } // t489 mm8 a=r5 b=r2 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r2 += d0_; r6 += d1_; } // t490 mm8 a=r7 b=r1 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r0 += d0_; r6 += d1_; } // t491 mm8 a=r0 b=r0 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r6 += d0_; r4 += d1_; } // t492 mm8 a=r3 b=r1 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r7 += d0_; r6 += d1_; } // t493 mm8 a=r6 b=r1 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r3 += d0_; r1 += d1_; } // t494 mm8 a=r5 b=r6 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r7 += d0_; r2 += d1_; } // t495 mm8 a=r2 b=r2 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r5 += d0_; r0 += d1_; } // t496 mm8 a=r3 b=r5 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r2 += d0_; r0 += d1_; } // t497 mm8 a=r3 b=r2 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r6 += d0_; r2 += d1_; } // t498 mm8 a=r3 b=r1 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r1 += d0_; r5 += d1_; } // t499 mm8 a=r1 b=r6 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r1 += d0_; r5 += d1_; } // t500 mm8 a=r7 b=r6 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r1 += d0_; r0 += d1_; } // t501 mm8 a=r4 b=r4 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r7 += d0_; r6 += d1_; } // t502 mm8 a=r5 b=r7 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r3 += d0_; r4 += d1_; } // t503 mm8 a=r7 b=r5 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r4 += d0_; r2 += d1_; } // t504 mm8 a=r2 b=r0 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r0 += d0_; r1 += d1_; } // t505 mm8 a=r4 b=r0 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r7 += d0_; r0 += d1_; } // t506 mm8 a=r0 b=r2 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r4 += d0_; r7 += d1_; } // t507 mm8 a=r5 b=r6 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r4 += d0_; r7 += d1_; } // t508 mm8 a=r7 b=r3 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r6 += d0_; r0 += d1_; } // t509 mm8 a=r4 b=r4 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t510 mm8 a=r0 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t511 mm8 a=r2 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r4 += d0_; r1 += d1_; } // t512 mm8 a=r2 b=r7 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r5 += d0_; r7 += d1_; } // t513 mm8 a=r0 b=r3 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r4 += d0_; r2 += d1_; } // t514 mm8 a=r6 b=r7 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r0 += d0_; r4 += d1_; } // t515 mm8 a=r5 b=r2 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r2 += d0_; r4 += d1_; } // t516 mm8 a=r0 b=r2 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r6 += d0_; r7 += d1_; } // t517 mm8 a=r6 b=r7 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r0 += d0_; r6 += d1_; } // t518 mm8 a=r1 b=r4 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r7 += d0_; r4 += d1_; } // t519 mm8 a=r6 b=r0 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r0 += d0_; r2 += d1_; } // t520 mm8 a=r1 b=r2 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r0 += d0_; r2 += d1_; } // t521 mm8 a=r0 b=r4 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r6 += d0_; r7 += d1_; } // t522 mm8 a=r7 b=r2 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r7 += d0_; r0 += d1_; } // t523 mm8 a=r5 b=r3 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r7 += d0_; r0 += d1_; } // t524 mm8 a=r0 b=r4 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r5 += d0_; r1 += d1_; } // t525 mm8 a=r1 b=r5 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r6 += d1_; } // t526 mm8 a=r7 b=r1 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r6 += d0_; r7 += d1_; } // t527 mm8 a=r5 b=r1 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r3 += d0_; r2 += d1_; } // t528 mm8 a=r4 b=r1 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r2 += d0_; r1 += d1_; } // t529 mm8 a=r4 b=r2 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r6 += d0_; r4 += d1_; } // t530 mm8 a=r0 b=r2 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r3 += d0_; r4 += d1_; } // t531 mm8 a=r7 b=r0 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r2 += d0_; r7 += d1_; } // t532 mm8 a=r7 b=r2 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r6 += d0_; r4 += d1_; } // t533 mm8 a=r3 b=r2 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r3 += d0_; r6 += d1_; } // t534 mm8 a=r4 b=r3 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r1 += d0_; r6 += d1_; } // t535 mm8 a=r0 b=r6 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r4 += d0_; r6 += d1_; } // t536 mm8 a=r3 b=r1 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r4 += d0_; r5 += d1_; } // t537 mm8 a=r1 b=r6 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r6 += d0_; r7 += d1_; } // t538 mm8 a=r6 b=r6 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t539 mm8 a=r4 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r2 += d0_; r4 += d1_; } // t540 mm8 a=r1 b=r4 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r3 += d0_; r6 += d1_; } // t541 mm8 a=r5 b=r3 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r0 += d0_; r7 += d1_; } // t542 mm8 a=r7 b=r7 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r3 += d0_; r7 += d1_; } // t543 mm8 a=r4 b=r2 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r3 += d0_; r6 += d1_; } // t544 mm8 a=r5 b=r7 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r2 += d0_; r1 += d1_; } // t545 mm8 a=r5 b=r2 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r5 += d0_; r7 += d1_; } // t546 mm8 a=r7 b=r1 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r2 += d0_; r1 += d1_; } // t547 mm8 a=r3 b=r5 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r5 += d0_; r4 += d1_; } // t548 mm8 a=r6 b=r3 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r3 += d0_; r7 += d1_; } // t549 mm8 a=r2 b=r2 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r5 += d0_; r2 += d1_; } // t550 mm8 a=r5 b=r5 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r4 += d0_; r0 += d1_; } // t551 mm8 a=r7 b=r6 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r1 += d0_; r2 += d1_; } // t552 mm8 a=r6 b=r7 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r6 += d0_; r7 += d1_; } // t553 mm8 a=r6 b=r3 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r7 += d0_; r0 += d1_; } // t554 mm8 a=r4 b=r2 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r0 += d0_; r1 += d1_; } // t555 mm8 a=r2 b=r7 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r0 += d0_; r7 += d1_; } // t556 mm8 a=r3 b=r6 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r7 += d0_; r0 += d1_; } // t557 mm8 a=r7 b=r7 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r5 += d0_; r2 += d1_; } // t558 mm8 a=r5 b=r0 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r5 += d0_; r1 += d1_; } // t559 mm8 a=r4 b=r1 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r5 += d0_; r4 += d1_; } // t560 mm8 a=r5 b=r4 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t561 mm8 a=r6 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r7 += d0_; r0 += d1_; } // t562 mm8 a=r1 b=r2 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r3 += d0_; r2 += d1_; } // t563 mm8 a=r5 b=r3 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r4 += d0_; r6 += d1_; } // t564 mm8 a=r4 b=r3 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r2 += d0_; r7 += d1_; } // t565 mm8 a=r6 b=r0 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r5 += d0_; r0 += d1_; } // t566 mm8 a=r1 b=r6 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r2 += d0_; r0 += d1_; } // t567 mm8 a=r3 b=r6 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r3 += d0_; r1 += d1_; } // t568 mm8 a=r6 b=r5 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r2 += d0_; r5 += d1_; } // t569 mm8 a=r3 b=r7 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r3 += d0_; r2 += d1_; } // t570 mm8 a=r5 b=r1 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r2 += d0_; r6 += d1_; } // t571 mm8 a=r6 b=r0 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r3 += d0_; r1 += d1_; } // t572 mm8 a=r5 b=r0 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r4 += d0_; r3 += d1_; } // t573 mm8 a=r3 b=r0 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r0 += d0_; r5 += d1_; } // t574 mm8 a=r2 b=r2 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r6 += d0_; r0 += d1_; } // t575 mm8 a=r3 b=r1 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r0 += d0_; r4 += d1_; } // t576 mm8 a=r2 b=r3 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r6 += d0_; r1 += d1_; } // t577 mm8 a=r6 b=r3 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r2 += d0_; r4 += d1_; } // t578 mm8 a=r5 b=r6 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r7 += d0_; r4 += d1_; } // t579 mm8 a=r4 b=r7 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r5 += d0_; r7 += d1_; } // t580 mm8 a=r7 b=r1 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r3 += d0_; r4 += d1_; } // t581 mm8 a=r1 b=r6 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r1 += d0_; r3 += d1_; } // t582 mm8 a=r2 b=r1 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t583 mm8 a=r5 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r0 += d0_; r4 += d1_; } // t584 mm8 a=r1 b=r7 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r5 += d0_; r2 += d1_; } // t585 mm8 a=r0 b=r4 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r1 += d0_; r2 += d1_; } // t586 mm8 a=r4 b=r4 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r6 += d0_; r3 += d1_; } // t587 mm8 a=r0 b=r4 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r7 += d0_; r0 += d1_; } // t588 mm8 a=r5 b=r5 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r3 += d0_; r5 += d1_; } // t589 mm8 a=r3 b=r6 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r5 += d0_; r0 += d1_; } // t590 mm8 a=r3 b=r2 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r0 += d0_; r5 += d1_; } // t591 mm8 a=r5 b=r1 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r7 += d0_; r4 += d1_; } // t592 mm8 a=r5 b=r3 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r6 += d0_; r4 += d1_; } // t593 mm8 a=r5 b=r3 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r4 += d0_; r0 += d1_; } // t594 mm8 a=r6 b=r7 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r6 += d0_; r4 += d1_; } // t595 mm8 a=r3 b=r0 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r3 += d0_; r2 += d1_; } // t596 mm8 a=r7 b=r6 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r5 += d0_; r1 += d1_; } // t597 mm8 a=r2 b=r4 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r4 += d0_; r0 += d1_; } // t598 mm8 a=r7 b=r7 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r2 += d0_; r6 += d1_; } // t599 mm8 a=r7 b=r2 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r5 += d0_; r4 += d1_; } // t600 mm8 a=r7 b=r7 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r7 += d0_; r5 += d1_; } // t601 mm8 a=r2 b=r7 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r7 += d0_; r1 += d1_; } // t602 mm8 a=r0 b=r1 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r7 += d0_; r4 += d1_; } // t603 mm8 a=r0 b=r0 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r2 += d0_; r1 += d1_; } // t604 mm8 a=r5 b=r5 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r4 += d0_; r2 += d1_; } // t605 mm8 a=r2 b=r0 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r1 += d0_; r7 += d1_; } // t606 mm8 a=r7 b=r5 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r4 += d0_; r5 += d1_; } // t607 mm8 a=r0 b=r2 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r6 += d0_; r4 += d1_; } // t608 mm8 a=r7 b=r6 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r7 += d0_; r1 += d1_; } // t609 mm8 a=r1 b=r5 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r1 += d0_; r6 += d1_; } // t610 mm8 a=r3 b=r3 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r5 += d0_; r7 += d1_; } // t611 mm8 a=r2 b=r2 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r1 += d0_; r2 += d1_; } // t612 mm8 a=r1 b=r1 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r3 += d0_; r4 += d1_; } // t613 mm8 a=r6 b=r3 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t614 mm8 a=r3 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r7 += d0_; r6 += d1_; } // t615 mm8 a=r6 b=r2 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r1 += d0_; r7 += d1_; } // t616 mm8 a=r2 b=r3 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r3 += d0_; r2 += d1_; } // t617 mm8 a=r5 b=r4 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r5 += d0_; r0 += d1_; } // t618 mm8 a=r3 b=r6 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r0 += d0_; r7 += d1_; } // t619 mm8 a=r5 b=r0 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r6 += d0_; r4 += d1_; } // t620 mm8 a=r3 b=r5 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r7 += d0_; r2 += d1_; } // t621 mm8 a=r4 b=r2 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r4 += d0_; r3 += d1_; } // t622 mm8 a=r7 b=r0 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r2 += d0_; r1 += d1_; } // t623 mm8 a=r4 b=r6 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r1 += d0_; r0 += d1_; } // t624 mm8 a=r6 b=r2 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r7 += d0_; r5 += d1_; } // t625 mm8 a=r6 b=r2 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r5 += d0_; r6 += d1_; } // t626 mm8 a=r5 b=r2 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r2 += d0_; r3 += d1_; } // t627 mm8 a=r4 b=r0 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r0 += d1_; } // t628 mm8 a=r1 b=r5 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r7 += d0_; r3 += d1_; } // t629 mm8 a=r4 b=r5 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r6 += d0_; r4 += d1_; } // t630 mm8 a=r4 b=r5 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r7 += d0_; r4 += d1_; } // t631 mm8 a=r6 b=r4 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r7 += d0_; r1 += d1_; } // t632 mm8 a=r2 b=r0 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t633 mm8 a=r2 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t634 mm8 a=r1 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r2 += d0_; r4 += d1_; } // t635 mm8 a=r6 b=r7 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r4 += d0_; r1 += d1_; } // t636 mm8 a=r2 b=r5 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t637 mm8 a=r5 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r5 += d0_; r1 += d1_; } // t638 mm8 a=r1 b=r4 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r5 += d0_; r1 += d1_; } // t639 mm8 a=r1 b=r7 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r4 += d0_; r0 += d1_; } // t640 mm8 a=r0 b=r4 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r5 += d0_; r4 += d1_; } // t641 mm8 a=r5 b=r6 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r1 += d0_; r6 += d1_; } // t642 mm8 a=r7 b=r5 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r5 += d0_; r6 += d1_; } // t643 mm8 a=r3 b=r5 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r2 += d0_; r5 += d1_; } // t644 mm8 a=r1 b=r6 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r4 += d0_; r1 += d1_; } // t645 mm8 a=r0 b=r7 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r4 += d0_; r3 += d1_; } // t646 mm8 a=r5 b=r6 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r6 += d0_; r4 += d1_; } // t647 mm8 a=r6 b=r4 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r6 += d0_; r2 += d1_; } // t648 mm8 a=r3 b=r1 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r6 += d0_; r7 += d1_; } // t649 mm8 a=r7 b=r2 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r3 += d0_; r5 += d1_; } // t650 mm8 a=r7 b=r7 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r3 += d0_; r4 += d1_; } // t651 mm8 a=r6 b=r3 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r3 += d0_; r2 += d1_; } // t652 mm8 a=r1 b=r6 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r0 += d0_; r5 += d1_; } // t653 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r7 += d0_; r4 += d1_; } // t654 mm8 a=r1 b=r6 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r3 += d0_; r1 += d1_; } // t655 mm8 a=r7 b=r5 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r1 += d0_; r6 += d1_; } // t656 mm8 a=r6 b=r5 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r7 += d0_; r2 += d1_; } // t657 mm8 a=r7 b=r4 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r6 += d0_; r1 += d1_; } // t658 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r3 += d0_; r0 += d1_; } // t659 mm8 a=r0 b=r7 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r4 += d0_; r5 += d1_; } // t660 mm8 a=r6 b=r1 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r1 += d0_; r4 += d1_; } // t661 mm8 a=r3 b=r7 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r5 += d0_; r3 += d1_; } // t662 mm8 a=r2 b=r3 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r0 += d0_; r4 += d1_; } // t663 mm8 a=r0 b=r3 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r1 += d0_; r3 += d1_; } // t664 mm8 a=r2 b=r0 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r1 += d0_; r0 += d1_; } // t665 mm8 a=r1 b=r1 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r2 += d0_; r7 += d1_; } // t666 mm8 a=r7 b=r1 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r3 += d0_; r6 += d1_; } // t667 mm8 a=r2 b=r0 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r3 += d0_; r6 += d1_; } // t668 mm8 a=r3 b=r1 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r4 += d0_; r3 += d1_; } // t669 mm8 a=r1 b=r4 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r5 += d0_; r0 += d1_; } // t670 mm8 a=r7 b=r7 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r0 += d0_; r3 += d1_; } // t671 mm8 a=r0 b=r3 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r4 += d0_; r3 += d1_; } // t672 mm8 a=r0 b=r1 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r0 += d0_; r2 += d1_; } // t673 mm8 a=r6 b=r2 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r0 += d0_; r5 += d1_; } // t674 mm8 a=r3 b=r3 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r3 += d0_; r6 += d1_; } // t675 mm8 a=r5 b=r6 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r1 += d0_; r0 += d1_; } // t676 mm8 a=r4 b=r3 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t677 mm8 a=r0 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r7 += d0_; r1 += d1_; } // t678 mm8 a=r4 b=r7 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r1 += d0_; r4 += d1_; } // t679 mm8 a=r4 b=r7 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r5 += d0_; r2 += d1_; } // t680 mm8 a=r5 b=r1 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r4 += d0_; r2 += d1_; } // t681 mm8 a=r4 b=r2 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t682 mm8 a=r3 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r4 += d0_; r7 += d1_; } // t683 mm8 a=r0 b=r5 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r3 += d0_; r7 += d1_; } // t684 mm8 a=r6 b=r2 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r0 += d0_; r5 += d1_; } // t685 mm8 a=r7 b=r3 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r1 += d0_; r7 += d1_; } // t686 mm8 a=r4 b=r1 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r3 += d0_; r0 += d1_; } // t687 mm8 a=r0 b=r5 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r4 += d0_; r1 += d1_; } // t688 mm8 a=r3 b=r6 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r7 += d0_; r1 += d1_; } // t689 mm8 a=r3 b=r1 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r0 += d0_; r1 += d1_; } // t690 mm8 a=r2 b=r6 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r0 += d0_; r6 += d1_; } // t691 mm8 a=r2 b=r6 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r0 += d0_; r7 += d1_; } // t692 mm8 a=r1 b=r4 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t693 mm8 a=r7 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r7 += d0_; r3 += d1_; } // t694 mm8 a=r3 b=r5 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r2 += d0_; r4 += d1_; } // t695 mm8 a=r3 b=r2 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r3 += d0_; r0 += d1_; } // t696 mm8 a=r2 b=r4 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r3 += d0_; r4 += d1_; } // t697 mm8 a=r2 b=r3 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r4 += d0_; r7 += d1_; } // t698 mm8 a=r3 b=r0 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r4 += d0_; r2 += d1_; } // t699 mm8 a=r2 b=r1 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r1 += d0_; r3 += d1_; } // t700 mm8 a=r0 b=r1 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r1 += d0_; r3 += d1_; } // t701 mm8 a=r5 b=r7 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r4 += d0_; r6 += d1_; } // t702 mm8 a=r5 b=r5 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r1 += d0_; r2 += d1_; } // t703 mm8 a=r2 b=r2 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r7 += d0_; r2 += d1_; } // t704 mm8 a=r6 b=r5 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r7 += d0_; r3 += d1_; } // t705 mm8 a=r5 b=r4 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r4 += d0_; r1 += d1_; } // t706 mm8 a=r6 b=r0 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r7 += d0_; r6 += d1_; } // t707 mm8 a=r6 b=r4 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r7 += d0_; r4 += d1_; } // t708 mm8 a=r2 b=r0 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r7 += d0_; r5 += d1_; } // t709 mm8 a=r7 b=r0 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r5 += d0_; r6 += d1_; } // t710 mm8 a=r4 b=r2 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r6 += d0_; r1 += d1_; } // t711 mm8 a=r4 b=r7 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r2 += d1_; } // t712 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r5 += d0_; r1 += d1_; } // t713 mm8 a=r5 b=r2 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r5 += d0_; r1 += d1_; } // t714 mm8 a=r0 b=r0 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t715 mm8 a=r1 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r5 += d0_; r6 += d1_; } // t716 mm8 a=r3 b=r3 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r7 += d0_; r1 += d1_; } // t717 mm8 a=r3 b=r3 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r2 += d0_; r7 += d1_; } // t718 mm8 a=r7 b=r5 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r3 += d0_; r0 += d1_; } // t719 mm8 a=r4 b=r7 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r5 += d0_; r3 += d1_; } // t720 mm8 a=r4 b=r7 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r0 += d0_; r3 += d1_; } // t721 mm8 a=r6 b=r4 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r7 += d0_; r6 += d1_; } // t722 mm8 a=r7 b=r5 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r7 += d0_; r2 += d1_; } // t723 mm8 a=r5 b=r0 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r4 += d0_; r0 += d1_; } // t724 mm8 a=r7 b=r4 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r3 += d0_; r5 += d1_; } // t725 mm8 a=r6 b=r1 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r5 += d0_; r1 += d1_; } // t726 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r4 += d0_; r6 += d1_; } // t727 mm8 a=r0 b=r6 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t728 mm8 a=r5 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r2 += d0_; r3 += d1_; } // t729 mm8 a=r5 b=r6 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r4 += d0_; r7 += d1_; } // t730 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r6 += d0_; r7 += d1_; } // t731 mm8 a=r5 b=r6 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r0 += d0_; r6 += d1_; } // t732 mm8 a=r5 b=r1 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t733 mm8 a=r3 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r0 += d0_; r3 += d1_; } // t734 mm8 a=r4 b=r3 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r4 += d0_; r3 += d1_; } // t735 mm8 a=r7 b=r4 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r3 += d0_; r0 += d1_; } // t736 mm8 a=r4 b=r2 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r1 += d0_; r2 += d1_; } // t737 mm8 a=r4 b=r7 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r1 += d0_; r2 += d1_; } // t738 mm8 a=r6 b=r2 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r6 += d0_; r4 += d1_; } // t739 mm8 a=r7 b=r5 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r2 += d0_; r7 += d1_; } // t740 mm8 a=r4 b=r3 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t741 mm8 a=r7 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r4 += d0_; r1 += d1_; } // t742 mm8 a=r6 b=r5 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r6 += d0_; r4 += d1_; } // t743 mm8 a=r5 b=r3 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r3 += d0_; r6 += d1_; } // t744 mm8 a=r7 b=r4 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r0 += d0_; r4 += d1_; } // t745 mm8 a=r7 b=r0 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r1 += d0_; r3 += d1_; } // t746 mm8 a=r2 b=r5 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r1 += d0_; r0 += d1_; } // t747 mm8 a=r0 b=r3 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r5 += d0_; r7 += d1_; } // t748 mm8 a=r1 b=r6 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r7 += d0_; r5 += d1_; } // t749 mm8 a=r2 b=r5 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r7 += d0_; r0 += d1_; } // t750 mm8 a=r4 b=r3 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r5 += d0_; r3 += d1_; } // t751 mm8 a=r6 b=r6 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r5 += d0_; r3 += d1_; } // t752 mm8 a=r5 b=r0 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r6 += d0_; r0 += d1_; } // t753 mm8 a=r6 b=r7 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r5 += d0_; r2 += d1_; } // t754 mm8 a=r2 b=r4 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r4 += d0_; r0 += d1_; } // t755 mm8 a=r5 b=r6 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r5 += d0_; r4 += d1_; } // t756 mm8 a=r7 b=r3 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r3 += d0_; r5 += d1_; } // t757 mm8 a=r6 b=r6 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r4 += d0_; r2 += d1_; } // t758 mm8 a=r6 b=r2 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r0 += d1_; } // t759 mm8 a=r2 b=r7 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r1 += d0_; r0 += d1_; } // t760 mm8 a=r7 b=r7 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r2 += d0_; r4 += d1_; } // t761 mm8 a=r3 b=r3 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r5 += d0_; r6 += d1_; } // t762 mm8 a=r5 b=r2 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r6 += d0_; r0 += d1_; } // t763 mm8 a=r1 b=r1 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r0 += d0_; r2 += d1_; } // t764 mm8 a=r1 b=r2 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r3 += d1_; } // t765 mm8 a=r2 b=r4 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r6 += d0_; r0 += d1_; } // t766 mm8 a=r7 b=r6 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r5 += d0_; r7 += d1_; } // t767 mm8 a=r7 b=r5 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r7 += d0_; r3 += d1_; } // t768 mm8 a=r5 b=r6 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r1 += d0_; r5 += d1_; } // t769 mm8 a=r4 b=r5 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r2 += d0_; r6 += d1_; } // t770 mm8 a=r7 b=r5 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r3 += d0_; r6 += d1_; } // t771 mm8 a=r2 b=r5 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r1 += d0_; r6 += d1_; } // t772 mm8 a=r0 b=r4 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r1 += d0_; r3 += d1_; } // t773 mm8 a=r2 b=r5 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r2 += d0_; r0 += d1_; } // t774 mm8 a=r4 b=r0 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t775 mm8 a=r4 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r1 += d0_; r0 += d1_; } // t776 mm8 a=r5 b=r7 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r2 += d0_; r0 += d1_; } // t777 mm8 a=r1 b=r1 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r5 += d0_; r7 += d1_; } // t778 mm8 a=r1 b=r2 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r0 += d0_; r7 += d1_; } // t779 mm8 a=r7 b=r5 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r0 += d0_; r3 += d1_; } // t780 mm8 a=r3 b=r7 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r7 += d0_; r3 += d1_; } // t781 mm8 a=r6 b=r7 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r6 += d0_; r4 += d1_; } // t782 mm8 a=r6 b=r4 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r6 += d0_; r0 += d1_; } // t783 mm8 a=r6 b=r5 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r6 += d1_; } // t784 mm8 a=r1 b=r5 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r5 += d0_; r3 += d1_; } // t785 mm8 a=r7 b=r2 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r5 += d0_; r2 += d1_; } // t786 mm8 a=r2 b=r4 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r4 += d0_; r6 += d1_; } // t787 mm8 a=r7 b=r2 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r7 += d0_; r6 += d1_; } // t788 mm8 a=r0 b=r0 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r3 += d0_; r5 += d1_; } // t789 mm8 a=r1 b=r4 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r2 += d0_; r3 += d1_; } // t790 mm8 a=r3 b=r1 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r7 += d0_; r4 += d1_; } // t791 mm8 a=r3 b=r5 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r6 += d0_; r4 += d1_; } // t792 mm8 a=r7 b=r1 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r1 += d0_; r0 += d1_; } // t793 mm8 a=r7 b=r2 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r3 += d0_; r0 += d1_; } // t794 mm8 a=r2 b=r4 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r2 += d1_; } // t795 mm8 a=r1 b=r7 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r3 += d0_; r6 += d1_; } // t796 mm8 a=r2 b=r4 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r2 += d0_; r7 += d1_; } // t797 mm8 a=r6 b=r2 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r2 += d0_; r1 += d1_; } // t798 mm8 a=r1 b=r3 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r4 += d0_; r2 += d1_; } // t799 mm8 a=r1 b=r3 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r6 += d0_; r4 += d1_; } // t800 mm8 a=r0 b=r4 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r1 += d0_; r7 += d1_; } // t801 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r6 += d0_; r0 += d1_; } // t802 mm8 a=r4 b=r7 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r7 += d0_; r5 += d1_; } // t803 mm8 a=r3 b=r2 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r7 += d0_; r2 += d1_; } // t804 mm8 a=r0 b=r4 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r5 += d0_; r2 += d1_; } // t805 mm8 a=r4 b=r6 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r2 += d0_; r7 += d1_; } // t806 mm8 a=r0 b=r5 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r2 += d0_; r3 += d1_; } // t807 mm8 a=r5 b=r0 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r7 += d0_; r4 += d1_; } // t808 mm8 a=r2 b=r4 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r4 += d0_; r6 += d1_; } // t809 mm8 a=r1 b=r6 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r5 += d0_; r4 += d1_; } // t810 mm8 a=r1 b=r2 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t811 mm8 a=r2 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r6 += d0_; r1 += d1_; } // t812 mm8 a=r0 b=r6 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r2 += d0_; r3 += d1_; } // t813 mm8 a=r2 b=r2 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r2 += d0_; r6 += d1_; } // t814 mm8 a=r3 b=r4 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r5 += d0_; r7 += d1_; } // t815 mm8 a=r4 b=r5 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r1 += d0_; r7 += d1_; } // t816 mm8 a=r3 b=r0 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r5 += d0_; r7 += d1_; } // t817 mm8 a=r3 b=r4 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r5 += d0_; r2 += d1_; } // t818 mm8 a=r2 b=r2 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r7 += d0_; r4 += d1_; } // t819 mm8 a=r1 b=r2 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r2 += d0_; r6 += d1_; } // t820 mm8 a=r3 b=r6 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r5 += d0_; r1 += d1_; } // t821 mm8 a=r7 b=r3 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r4 += d0_; r3 += d1_; } // t822 mm8 a=r1 b=r5 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r2 += d0_; r5 += d1_; } // t823 mm8 a=r1 b=r4 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r0 += d0_; r5 += d1_; } // t824 mm8 a=r7 b=r7 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r1 += d0_; r2 += d1_; } // t825 mm8 a=r1 b=r1 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r7 += d0_; r1 += d1_; } // t826 mm8 a=r0 b=r2 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r6 += d0_; r7 += d1_; } // t827 mm8 a=r3 b=r1 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r2 += d0_; r4 += d1_; } // t828 mm8 a=r7 b=r4 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r0 += d0_; r5 += d1_; } // t829 mm8 a=r5 b=r2 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r4 += d0_; r0 += d1_; } // t830 mm8 a=r6 b=r3 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r5 += d1_; } // t831 mm8 a=r2 b=r7 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r6 += d0_; r1 += d1_; } // t832 mm8 a=r1 b=r6 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r5 += d0_; r2 += d1_; } // t833 mm8 a=r0 b=r6 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r3 += d0_; r6 += d1_; } // t834 mm8 a=r2 b=r4 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r5 += d0_; r0 += d1_; } // t835 mm8 a=r0 b=r7 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t836 mm8 a=r3 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r6 += d0_; r4 += d1_; } // t837 mm8 a=r4 b=r7 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r4 += d0_; r3 += d1_; } // t838 mm8 a=r5 b=r4 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r7 += d0_; r3 += d1_; } // t839 mm8 a=r0 b=r6 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r5 += d0_; r3 += d1_; } // t840 mm8 a=r1 b=r6 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r4 += d0_; r2 += d1_; } // t841 mm8 a=r4 b=r0 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r3 += d0_; r7 += d1_; } // t842 mm8 a=r3 b=r5 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r2 += d0_; r4 += d1_; } // t843 mm8 a=r2 b=r2 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r2 += d0_; r1 += d1_; } // t844 mm8 a=r3 b=r3 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t845 mm8 a=r1 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r2 += d0_; r3 += d1_; } // t846 mm8 a=r1 b=r4 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t847 mm8 a=r4 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r1 += d0_; r0 += d1_; } // t848 mm8 a=r6 b=r7 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r5 += d0_; r4 += d1_; } // t849 mm8 a=r6 b=r6 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r6 += d0_; r3 += d1_; } // t850 mm8 a=r6 b=r0 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r0 += d0_; r3 += d1_; } // t851 mm8 a=r0 b=r5 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r2 += d0_; r1 += d1_; } // t852 mm8 a=r2 b=r2 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r6 += d0_; r7 += d1_; } // t853 mm8 a=r6 b=r5 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r5 += d0_; r7 += d1_; } // t854 mm8 a=r1 b=r4 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r0 += d0_; r1 += d1_; } // t855 mm8 a=r1 b=r7 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r1 += d0_; r7 += d1_; } // t856 mm8 a=r2 b=r3 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r4 += d0_; r0 += d1_; } // t857 mm8 a=r6 b=r5 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r7 += d1_; } // t858 mm8 a=r0 b=r5 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t859 mm8 a=r2 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r4 += d0_; r7 += d1_; } // t860 mm8 a=r7 b=r2 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r7 += d0_; r4 += d1_; } // t861 mm8 a=r3 b=r3 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r4 += d0_; r5 += d1_; } // t862 mm8 a=r0 b=r1 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r4 += d0_; r1 += d1_; } // t863 mm8 a=r0 b=r0 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r2 += d0_; r1 += d1_; } // t864 mm8 a=r5 b=r5 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t865 mm8 a=r3 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r1 += d0_; r5 += d1_; } // t866 mm8 a=r6 b=r6 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t867 mm8 a=r7 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t868 mm8 a=r4 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r2 += d0_; r7 += d1_; } // t869 mm8 a=r3 b=r6 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r6 += d0_; r5 += d1_; } // t870 mm8 a=r0 b=r1 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r3 += d0_; r1 += d1_; } // t871 mm8 a=r5 b=r1 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r1 += d0_; r4 += d1_; } // t872 mm8 a=r0 b=r6 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r2 += d0_; r3 += d1_; } // t873 mm8 a=r2 b=r1 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r5 += d0_; r6 += d1_; } // t874 mm8 a=r5 b=r1 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r2 += d0_; r5 += d1_; } // t875 mm8 a=r1 b=r7 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r7 += d0_; r1 += d1_; } // t876 mm8 a=r7 b=r6 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r6 += d0_; r1 += d1_; } // t877 mm8 a=r6 b=r2 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r7 += d0_; r4 += d1_; } // t878 mm8 a=r5 b=r0 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r7 += d0_; r2 += d1_; } // t879 mm8 a=r4 b=r1 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r7 += d0_; r1 += d1_; } // t880 mm8 a=r3 b=r3 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r0 += d0_; r5 += d1_; } // t881 mm8 a=r2 b=r7 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r1 += d0_; r4 += d1_; } // t882 mm8 a=r7 b=r5 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r1 += d0_; r2 += d1_; } // t883 mm8 a=r2 b=r3 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r5 += d0_; r2 += d1_; } // t884 mm8 a=r2 b=r7 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r4 += d0_; r0 += d1_; } // t885 mm8 a=r3 b=r5 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r5 += d0_; r3 += d1_; } // t886 mm8 a=r5 b=r4 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r4 += d0_; r7 += d1_; } // t887 mm8 a=r1 b=r1 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r7 += d0_; r3 += d1_; } // t888 mm8 a=r6 b=r2 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r4 += d0_; r5 += d1_; } // t889 mm8 a=r4 b=r6 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r6 += d0_; r2 += d1_; } // t890 mm8 a=r0 b=r5 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r2 += d0_; r0 += d1_; } // t891 mm8 a=r0 b=r7 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r7 += d0_; r4 += d1_; } // t892 mm8 a=r5 b=r6 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r0 += d0_; r1 += d1_; } // t893 mm8 a=r2 b=r1 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r3 += d0_; r7 += d1_; } // t894 mm8 a=r0 b=r7 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r5 += d0_; r0 += d1_; } // t895 mm8 a=r2 b=r4 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r2 += d0_; r5 += d1_; } // t896 mm8 a=r0 b=r2 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r6 += d0_; r0 += d1_; } // t897 mm8 a=r7 b=r4 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r3 += d0_; r0 += d1_; } // t898 mm8 a=r1 b=r3 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r1 += d0_; r3 += d1_; } // t899 mm8 a=r5 b=r1 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r7 += d0_; r0 += d1_; } // t900 mm8 a=r7 b=r7 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r5 += d1_; } // t901 mm8 a=r4 b=r6 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r5 += d0_; r6 += d1_; } // t902 mm8 a=r2 b=r1 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r1 += d0_; r6 += d1_; } // t903 mm8 a=r0 b=r3 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r6 += d1_; } // t904 mm8 a=r6 b=r7 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r6 += d0_; r1 += d1_; } // t905 mm8 a=r0 b=r4 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r2 += d0_; r3 += d1_; } // t906 mm8 a=r3 b=r7 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r0 += d0_; r1 += d1_; } // t907 mm8 a=r2 b=r4 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r4 += d0_; r3 += d1_; } // t908 mm8 a=r7 b=r1 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r7 += d0_; r5 += d1_; } // t909 mm8 a=r4 b=r5 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r4 += d0_; r5 += d1_; } // t910 mm8 a=r3 b=r7 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r3 += d0_; r5 += d1_; } // t911 mm8 a=r2 b=r6 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r4 += d0_; r5 += d1_; } // t912 mm8 a=r6 b=r1 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r2 += d0_; r6 += d1_; } // t913 mm8 a=r2 b=r3 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r5 += d0_; r4 += d1_; } // t914 mm8 a=r7 b=r4 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r3 += d0_; r5 += d1_; } // t915 mm8 a=r4 b=r3 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r3 += d0_; r6 += d1_; } // t916 mm8 a=r4 b=r2 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r6 += d0_; r7 += d1_; } // t917 mm8 a=r5 b=r5 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r3 += d0_; r7 += d1_; } // t918 mm8 a=r7 b=r2 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r6 += d0_; r7 += d1_; } // t919 mm8 a=r3 b=r4 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r2 += d0_; r6 += d1_; } // t920 mm8 a=r6 b=r5 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r0 += d0_; r2 += d1_; } // t921 mm8 a=r6 b=r5 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r4 += d0_; r7 += d1_; } // t922 mm8 a=r1 b=r0 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r6 += d0_; r7 += d1_; } // t923 mm8 a=r6 b=r3 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r7 += d0_; r6 += d1_; } // t924 mm8 a=r2 b=r0 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r3 += d0_; r4 += d1_; } // t925 mm8 a=r6 b=r6 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r2 += d0_; r0 += d1_; } // t926 mm8 a=r5 b=r0 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t927 mm8 a=r1 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r7 += d0_; r5 += d1_; } // t928 mm8 a=r5 b=r7 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r7 += d0_; r5 += d1_; } // t929 mm8 a=r1 b=r0 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t930 mm8 a=r7 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t931 mm8 a=r7 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r6 += d0_; r4 += d1_; } // t932 mm8 a=r1 b=r3 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r0 += d0_; r3 += d1_; } // t933 mm8 a=r1 b=r3 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r6 += d0_; r0 += d1_; } // t934 mm8 a=r6 b=r0 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r6 += d0_; r7 += d1_; } // t935 mm8 a=r5 b=r0 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r1 += d0_; r6 += d1_; } // t936 mm8 a=r3 b=r3 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r7 += d0_; r5 += d1_; } // t937 mm8 a=r0 b=r6 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r1 += d0_; r3 += d1_; } // t938 mm8 a=r1 b=r5 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r3 += d0_; r0 += d1_; } // t939 mm8 a=r7 b=r0 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r4 += d0_; r5 += d1_; } // t940 mm8 a=r1 b=r2 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r2 += d0_; r7 += d1_; } // t941 mm8 a=r1 b=r6 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r1 += d0_; r3 += d1_; } // t942 mm8 a=r0 b=r6 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r1 += d0_; r4 += d1_; } // t943 mm8 a=r4 b=r0 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r2 += d0_; r1 += d1_; } // t944 mm8 a=r6 b=r5 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r4 += d0_; r2 += d1_; } // t945 mm8 a=r3 b=r7 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t946 mm8 a=r2 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r5 += d0_; r3 += d1_; } // t947 mm8 a=r7 b=r2 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t948 mm8 a=r6 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r0 += d0_; r6 += d1_; } // t949 mm8 a=r6 b=r6 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t950 mm8 a=r7 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t951 mm8 a=r2 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r1 += d0_; r2 += d1_; } // t952 mm8 a=r3 b=r2 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r0 += d0_; r5 += d1_; } // t953 mm8 a=r4 b=r4 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r2 += d1_; } // t954 mm8 a=r2 b=r4 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r0 += d0_; r7 += d1_; } // t955 mm8 a=r0 b=r7 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t956 mm8 a=r7 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t957 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r7 += d0_; r4 += d1_; } // t958 mm8 a=r6 b=r5 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r1 += d0_; r5 += d1_; } // t959 mm8 a=r4 b=r7 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r4 += d0_; r2 += d1_; } // t960 mm8 a=r1 b=r1 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r6 += d0_; r3 += d1_; } // t961 mm8 a=r1 b=r5 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r0 += d0_; r2 += d1_; } // t962 mm8 a=r6 b=r4 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r4 += d0_; r7 += d1_; } // t963 mm8 a=r7 b=r1 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r1 += d0_; r0 += d1_; } // t964 mm8 a=r2 b=r5 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t965 mm8 a=r3 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r6 += d0_; r0 += d1_; } // t966 mm8 a=r7 b=r3 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r7 += d0_; r1 += d1_; } // t967 mm8 a=r0 b=r0 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r5 += d0_; r7 += d1_; } // t968 mm8 a=r5 b=r7 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r6 += d0_; r1 += d1_; } // t969 mm8 a=r4 b=r4 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r0 += d0_; r7 += d1_; } // t970 mm8 a=r4 b=r3 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r7 += d0_; r3 += d1_; } // t971 mm8 a=r3 b=r4 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r4 += d0_; r7 += d1_; } // t972 mm8 a=r1 b=r7 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r1 += d0_; r7 += d1_; } // t973 mm8 a=r4 b=r1 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r3 += d0_; r2 += d1_; } // t974 mm8 a=r5 b=r6 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r6 += d0_; r4 += d1_; } // t975 mm8 a=r1 b=r2 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r7 += d0_; r6 += d1_; } // t976 mm8 a=r0 b=r6 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r4 += d0_; r0 += d1_; } // t977 mm8 a=r2 b=r0 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r2 += d0_; r6 += d1_; } // t978 mm8 a=r0 b=r2 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r3 += d0_; r6 += d1_; } // t979 mm8 a=r4 b=r0 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r2 += d0_; r7 += d1_; } // t980 mm8 a=r3 b=r1 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r2 += d0_; r1 += d1_; } // t981 mm8 a=r2 b=r0 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t982 mm8 a=r1 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r7 += d0_; r4 += d1_; } // t983 mm8 a=r2 b=r1 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r7 += d0_; r1 += d1_; } // t984 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r1 += d0_; r5 += d1_; } // t985 mm8 a=r4 b=r3 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r5 += d1_; } // t986 mm8 a=r2 b=r7 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r7 += d0_; r3 += d1_; } // t987 mm8 a=r5 b=r1 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r3 += d0_; r7 += d1_; } // t988 mm8 a=r6 b=r0 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r1 += d0_; r3 += d1_; } // t989 mm8 a=r2 b=r3 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r4 += d0_; r3 += d1_; } // t990 mm8 a=r1 b=r5 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r1 += d0_; r0 += d1_; } // t991 mm8 a=r6 b=r0 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r0 += d0_; r4 += d1_; } // t992 mm8 a=r2 b=r1 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r1 += d0_; r7 += d1_; } // t993 mm8 a=r6 b=r0 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r6 += d0_; r2 += d1_; } // t994 mm8 a=r6 b=r0 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r1 += d0_; r3 += d1_; } // t995 mm8 a=r0 b=r3 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r6 += d0_; r0 += d1_; } // t996 mm8 a=r7 b=r0 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r0 += d0_; r6 += d1_; } // t997 mm8 a=r5 b=r5 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r1 += d0_; r7 += d1_; } // t998 mm8 a=r3 b=r4 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r4 += d0_; r7 += d1_; } // t999 mm8 a=r7 b=r2 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r5 += d0_; r3 += d1_; } // t1000 mm8 a=r5 b=r3 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r0 += d0_; r5 += d1_; } // t1001 mm8 a=r5 b=r6 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r1 += d0_; r4 += d1_; } // t1002 mm8 a=r6 b=r2 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r1 += d0_; r5 += d1_; } // t1003 mm8 a=r4 b=r2 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r4 += d0_; r5 += d1_; } // t1004 mm8 a=r7 b=r3 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r4 += d0_; r1 += d1_; } // t1005 mm8 a=r1 b=r7 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r2 += d0_; r7 += d1_; } // t1006 mm8 a=r4 b=r2 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r4 += d0_; r5 += d1_; } // t1007 mm8 a=r0 b=r2 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r5 += d0_; r7 += d1_; } // t1008 mm8 a=r3 b=r5 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r6 += d0_; r7 += d1_; } // t1009 mm8 a=r0 b=r7 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r2 += d0_; r5 += d1_; } // t1010 mm8 a=r6 b=r7 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r5 += d0_; r2 += d1_; } // t1011 mm8 a=r7 b=r5 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r1 += d0_; r5 += d1_; } // t1012 mm8 a=r3 b=r5 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r6 += d0_; r1 += d1_; } // t1013 mm8 a=r6 b=r6 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r2 += d0_; r3 += d1_; } // t1014 mm8 a=r7 b=r3 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r2 += d0_; r6 += d1_; } // t1015 mm8 a=r7 b=r6 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r1 += d0_; r6 += d1_; } // t1016 mm8 a=r4 b=r0 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r6 += d0_; r3 += d1_; } // t1017 mm8 a=r5 b=r2 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r2 += d0_; r6 += d1_; } // t1018 mm8 a=r7 b=r6 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r5 += d0_; r4 += d1_; } // t1019 mm8 a=r0 b=r7 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r5 += d0_; r4 += d1_; } // t1020 mm8 a=r5 b=r6 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r3 += d0_; r4 += d1_; } // t1021 mm8 a=r7 b=r6 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r2 += d0_; r6 += d1_; } // t1022 mm8 a=r3 b=r5 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r6 += d0_; r7 += d1_; } // t1023 mm8 a=r0 b=r3 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r2 += d0_; r1 += d1_; } // t1024 mm8 a=r2 b=r6 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r2 += d0_; r0 += d1_; } // t1025 mm8 a=r4 b=r7 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r2 += d0_; r4 += d1_; } // t1026 mm8 a=r0 b=r1 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r5 += d0_; r0 += d1_; } // t1027 mm8 a=r1 b=r5 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r3 += d0_; r6 += d1_; } // t1028 mm8 a=r7 b=r6 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r0 += d0_; r7 += d1_; } // t1029 mm8 a=r2 b=r1 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r0 += d0_; r2 += d1_; } // t1030 mm8 a=r1 b=r7 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r0 += d0_; r3 += d1_; } // t1031 mm8 a=r3 b=r7 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r7 += d0_; r2 += d1_; } // t1032 mm8 a=r2 b=r1 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r5 += d0_; r6 += d1_; } // t1033 mm8 a=r2 b=r7 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r4 += d0_; r3 += d1_; } // t1034 mm8 a=r4 b=r3 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r5 += d0_; r6 += d1_; } // t1035 mm8 a=r3 b=r0 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r7 += d0_; r5 += d1_; } // t1036 mm8 a=r3 b=r0 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r4 += d0_; r2 += d1_; } // t1037 mm8 a=r7 b=r1 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r1 += d0_; r3 += d1_; } // t1038 mm8 a=r2 b=r1 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t1039 mm8 a=r5 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r3 += d0_; r7 += d1_; } // t1040 mm8 a=r0 b=r7 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r7 += d0_; r1 += d1_; } // t1041 mm8 a=r2 b=r2 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r2 += d0_; r4 += d1_; } // t1042 mm8 a=r6 b=r6 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r3 += d0_; r6 += d1_; } // t1043 mm8 a=r6 b=r7 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r7 += d0_; r1 += d1_; } // t1044 mm8 a=r4 b=r2 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r1 += d0_; r6 += d1_; } // t1045 mm8 a=r0 b=r0 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r3 += d0_; r2 += d1_; } // t1046 mm8 a=r7 b=r1 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r5 += d0_; r6 += d1_; } // t1047 mm8 a=r5 b=r7 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r6 += d0_; r0 += d1_; } // t1048 mm8 a=r2 b=r3 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r6 += d0_; r7 += d1_; } // t1049 mm8 a=r5 b=r1 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r0 += d1_; } // t1050 mm8 a=r0 b=r5 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r4 += d0_; r2 += d1_; } // t1051 mm8 a=r7 b=r7 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t1052 mm8 a=r2 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r2 += d0_; r0 += d1_; } // t1053 mm8 a=r6 b=r0 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r4 += d0_; r5 += d1_; } // t1054 mm8 a=r4 b=r2 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r7 += d0_; r2 += d1_; } // t1055 mm8 a=r5 b=r3 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r5 += d0_; r6 += d1_; } // t1056 mm8 a=r5 b=r1 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r5 += d0_; r4 += d1_; } // t1057 mm8 a=r1 b=r5 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r4 += d0_; r3 += d1_; } // t1058 mm8 a=r0 b=r5 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r4 += d0_; r5 += d1_; } // t1059 mm8 a=r5 b=r6 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r5 += d0_; r0 += d1_; } // t1060 mm8 a=r7 b=r3 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r6 += d0_; r1 += d1_; } // t1061 mm8 a=r5 b=r6 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r2 += d0_; r0 += d1_; } // t1062 mm8 a=r3 b=r0 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r7 += d0_; r0 += d1_; } // t1063 mm8 a=r1 b=r1 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r2 += d0_; r1 += d1_; } // t1064 mm8 a=r5 b=r4 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r4 += d0_; r5 += d1_; } // t1065 mm8 a=r3 b=r1 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r4 += d0_; r5 += d1_; } // t1066 mm8 a=r7 b=r7 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r6 += d0_; r1 += d1_; } // t1067 mm8 a=r0 b=r5 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r4 += d0_; r6 += d1_; } // t1068 mm8 a=r4 b=r6 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r6 += d0_; r5 += d1_; } // t1069 mm8 a=r2 b=r0 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r2 += d0_; r1 += d1_; } // t1070 mm8 a=r2 b=r3 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r1 += d0_; r7 += d1_; } // t1071 mm8 a=r2 b=r6 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r2 += d0_; r4 += d1_; } // t1072 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r2 += d0_; r0 += d1_; } // t1073 mm8 a=r3 b=r6 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r7 += d0_; r1 += d1_; } // t1074 mm8 a=r0 b=r0 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r7 += d0_; r1 += d1_; } // t1075 mm8 a=r4 b=r4 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r3 += d0_; r6 += d1_; } // t1076 mm8 a=r1 b=r7 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t1077 mm8 a=r7 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r1 += d0_; r3 += d1_; } // t1078 mm8 a=r3 b=r7 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r7 += d0_; r4 += d1_; } // t1079 mm8 a=r6 b=r6 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r1 += d0_; r4 += d1_; } // t1080 mm8 a=r4 b=r5 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r4 += d0_; r5 += d1_; } // t1081 mm8 a=r1 b=r5 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r3 += d0_; r6 += d1_; } // t1082 mm8 a=r5 b=r4 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r6 += d0_; r7 += d1_; } // t1083 mm8 a=r2 b=r2 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r2 += d0_; r1 += d1_; } // t1084 mm8 a=r2 b=r0 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r0 += d0_; r4 += d1_; } // t1085 mm8 a=r3 b=r3 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r3 += d0_; r2 += d1_; } // t1086 mm8 a=r3 b=r1 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r6 += d0_; r5 += d1_; } // t1087 mm8 a=r1 b=r3 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r5 += d0_; r7 += d1_; } // t1088 mm8 a=r3 b=r5 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r2 += d0_; r3 += d1_; } // t1089 mm8 a=r0 b=r0 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r7 += d0_; r2 += d1_; } // t1090 mm8 a=r4 b=r6 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r6 += d0_; r4 += d1_; } // t1091 mm8 a=r4 b=r3 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r1 += d0_; r6 += d1_; } // t1092 mm8 a=r2 b=r6 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r0 += d0_; r4 += d1_; } // t1093 mm8 a=r5 b=r7 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r2 += d0_; r6 += d1_; } // t1094 mm8 a=r7 b=r3 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r4 += d0_; r6 += d1_; } // t1095 mm8 a=r0 b=r2 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r6 += d0_; r1 += d1_; } // t1096 mm8 a=r3 b=r3 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r5 += d0_; r4 += d1_; } // t1097 mm8 a=r6 b=r2 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r1 += d0_; r6 += d1_; } // t1098 mm8 a=r3 b=r6 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r4 += d0_; r0 += d1_; } // t1099 mm8 a=r2 b=r5 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r1 += d0_; r0 += d1_; } // t1100 mm8 a=r1 b=r4 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r5 += d0_; r7 += d1_; } // t1101 mm8 a=r0 b=r6 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r6 += d0_; r5 += d1_; } // t1102 mm8 a=r0 b=r5 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t1103 mm8 a=r7 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r7 += d0_; r4 += d1_; } // t1104 mm8 a=r4 b=r2 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r2 += d0_; r0 += d1_; } // t1105 mm8 a=r3 b=r4 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r7 += d0_; r0 += d1_; } // t1106 mm8 a=r2 b=r1 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r6 += d0_; r5 += d1_; } // t1107 mm8 a=r3 b=r0 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r4 += d0_; r2 += d1_; } // t1108 mm8 a=r3 b=r5 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r4 += d0_; r3 += d1_; } // t1109 mm8 a=r6 b=r2 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r4 += d0_; r7 += d1_; } // t1110 mm8 a=r2 b=r4 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r2 += d0_; r1 += d1_; } // t1111 mm8 a=r0 b=r7 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r3 += d0_; r0 += d1_; } // t1112 mm8 a=r1 b=r0 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r5 += d0_; r6 += d1_; } // t1113 mm8 a=r4 b=r6 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r0 += d0_; r6 += d1_; } // t1114 mm8 a=r0 b=r5 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r3 += d0_; r4 += d1_; } // t1115 mm8 a=r4 b=r5 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r7 += d0_; r5 += d1_; } // t1116 mm8 a=r4 b=r6 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r0 += d0_; r7 += d1_; } // t1117 mm8 a=r0 b=r6 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r5 += d0_; r6 += d1_; } // t1118 mm8 a=r7 b=r6 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r0 += d0_; r5 += d1_; } // t1119 mm8 a=r2 b=r7 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r0 += d0_; r5 += d1_; } // t1120 mm8 a=r7 b=r5 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r6 += d0_; r7 += d1_; } // t1121 mm8 a=r4 b=r0 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r3 += d0_; r2 += d1_; } // t1122 mm8 a=r0 b=r5 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t1123 mm8 a=r0 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r4 += d0_; r0 += d1_; } // t1124 mm8 a=r0 b=r1 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r2 += d0_; r0 += d1_; } // t1125 mm8 a=r6 b=r6 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r0 += d0_; r3 += d1_; } // t1126 mm8 a=r0 b=r0 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r7 += d0_; r1 += d1_; } // t1127 mm8 a=r1 b=r1 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r7 += d0_; r3 += d1_; } // t1128 mm8 a=r4 b=r1 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r6 += d0_; r2 += d1_; } // t1129 mm8 a=r0 b=r0 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r7 += d0_; r5 += d1_; } // t1130 mm8 a=r0 b=r4 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r3 += d0_; r0 += d1_; } // t1131 mm8 a=r3 b=r0 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r5 += d0_; r7 += d1_; } // t1132 mm8 a=r0 b=r6 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r2 += d0_; r1 += d1_; } // t1133 mm8 a=r6 b=r0 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r1 += d0_; r4 += d1_; } // t1134 mm8 a=r7 b=r3 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r6 += d0_; r4 += d1_; } // t1135 mm8 a=r6 b=r7 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r7 += d0_; r2 += d1_; } // t1136 mm8 a=r7 b=r4 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r5 += d0_; r7 += d1_; } // t1137 mm8 a=r4 b=r0 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r3 += d0_; r4 += d1_; } // t1138 mm8 a=r6 b=r4 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r1 += d0_; r7 += d1_; } // t1139 mm8 a=r6 b=r2 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r6 += d0_; r3 += d1_; } // t1140 mm8 a=r1 b=r2 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r6 += d1_; } // t1141 mm8 a=r0 b=r5 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r2 += d0_; r5 += d1_; } // t1142 mm8 a=r4 b=r5 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r6 += d0_; r1 += d1_; } // t1143 mm8 a=r5 b=r6 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r7 += d0_; r3 += d1_; } // t1144 mm8 a=r6 b=r4 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r2 += d0_; r7 += d1_; } // t1145 mm8 a=r5 b=r1 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r5 += d0_; r3 += d1_; } // t1146 mm8 a=r6 b=r5 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r4 += d0_; r6 += d1_; } // t1147 mm8 a=r7 b=r1 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r4 += d0_; r6 += d1_; } // t1148 mm8 a=r1 b=r6 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r5 += d0_; r7 += d1_; } // t1149 mm8 a=r5 b=r5 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r5 += d0_; r1 += d1_; } // t1150 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r7 += d0_; r5 += d1_; } // t1151 mm8 a=r7 b=r5 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r5 += d0_; r3 += d1_; } // t1152 mm8 a=r0 b=r6 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r0 += d0_; r3 += d1_; } // t1153 mm8 a=r2 b=r5 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r7 += d0_; r0 += d1_; } // t1154 mm8 a=r5 b=r3 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r0 += d0_; r4 += d1_; } // t1155 mm8 a=r0 b=r6 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r4 += d0_; r7 += d1_; } // t1156 mm8 a=r2 b=r6 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r1 += d0_; r2 += d1_; } // t1157 mm8 a=r6 b=r7 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r3 += d0_; r7 += d1_; } // t1158 mm8 a=r2 b=r3 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t1159 mm8 a=r1 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r2 += d0_; r7 += d1_; } // t1160 mm8 a=r5 b=r0 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r3 += d0_; r4 += d1_; } // t1161 mm8 a=r7 b=r0 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r0 += d0_; r4 += d1_; } // t1162 mm8 a=r5 b=r3 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r2 += d0_; r1 += d1_; } // t1163 mm8 a=r4 b=r3 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r2 += d0_; r0 += d1_; } // t1164 mm8 a=r3 b=r4 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r7 += d0_; r1 += d1_; } // t1165 mm8 a=r1 b=r2 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r7 += d0_; r4 += d1_; } // t1166 mm8 a=r4 b=r7 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r5 += d1_; } // t1167 mm8 a=r2 b=r7 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r4 += d0_; r0 += d1_; } // t1168 mm8 a=r2 b=r6 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r3 += d0_; r7 += d1_; } // t1169 mm8 a=r0 b=r4 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r3 += d0_; r4 += d1_; } // t1170 mm8 a=r2 b=r1 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r7 += d1_; } // t1171 mm8 a=r1 b=r7 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r0 += d0_; r4 += d1_; } // t1172 mm8 a=r0 b=r1 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t1173 mm8 a=r6 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r2 += d0_; r6 += d1_; } // t1174 mm8 a=r0 b=r1 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r3 += d0_; r1 += d1_; } // t1175 mm8 a=r0 b=r0 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r5 += d0_; r6 += d1_; } // t1176 mm8 a=r4 b=r5 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r5 += d0_; r2 += d1_; } // t1177 mm8 a=r7 b=r2 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r7 += d0_; r0 += d1_; } // t1178 mm8 a=r1 b=r5 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t1179 mm8 a=r0 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r5 += d0_; r2 += d1_; } // t1180 mm8 a=r7 b=r3 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r0 += d0_; r7 += d1_; } // t1181 mm8 a=r7 b=r3 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r7 += d0_; r3 += d1_; } // t1182 mm8 a=r1 b=r2 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t1183 mm8 a=r2 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r2 += d0_; r7 += d1_; } // t1184 mm8 a=r7 b=r0 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r4 += d0_; r5 += d1_; } // t1185 mm8 a=r5 b=r2 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r2 += d0_; r3 += d1_; } // t1186 mm8 a=r1 b=r0 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r2 += d0_; r7 += d1_; } // t1187 mm8 a=r5 b=r4 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r4 += d0_; r6 += d1_; } // t1188 mm8 a=r3 b=r0 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t1189 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r2 += d0_; r7 += d1_; } // t1190 mm8 a=r5 b=r0 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r5 += d0_; r1 += d1_; } // t1191 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r2 += d0_; r1 += d1_; } // t1192 mm8 a=r0 b=r2 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r5 += d0_; r1 += d1_; } // t1193 mm8 a=r3 b=r7 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r1 += d0_; r2 += d1_; } // t1194 mm8 a=r7 b=r2 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r0 += d0_; r4 += d1_; } // t1195 mm8 a=r1 b=r4 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r5 += d0_; r0 += d1_; } // t1196 mm8 a=r4 b=r6 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r3 += d0_; r4 += d1_; } // t1197 mm8 a=r5 b=r6 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r2 += d0_; r5 += d1_; } // t1198 mm8 a=r0 b=r6 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r7 += d0_; r2 += d1_; } // t1199 mm8 a=r5 b=r1 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r1 += d0_; r2 += d1_; } // t1200 mm8 a=r1 b=r3 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r2 += d0_; r0 += d1_; } // t1201 mm8 a=r6 b=r6 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r4 += d0_; r7 += d1_; } // t1202 mm8 a=r2 b=r4 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r4 += d0_; r2 += d1_; } // t1203 mm8 a=r0 b=r5 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r7 += d0_; r4 += d1_; } // t1204 mm8 a=r0 b=r2 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r7 += d0_; r3 += d1_; } // t1205 mm8 a=r4 b=r6 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r4 += d0_; r6 += d1_; } // t1206 mm8 a=r4 b=r6 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r7 += d0_; r6 += d1_; } // t1207 mm8 a=r3 b=r7 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r2 += d0_; r4 += d1_; } // t1208 mm8 a=r6 b=r1 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r2 += d0_; r4 += d1_; } // t1209 mm8 a=r5 b=r5 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r3 += d0_; r0 += d1_; } // t1210 mm8 a=r0 b=r0 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r2 += d0_; r3 += d1_; } // t1211 mm8 a=r7 b=r5 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r7 += d0_; r1 += d1_; } // t1212 mm8 a=r2 b=r6 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r0 += d0_; r1 += d1_; } // t1213 mm8 a=r6 b=r2 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r3 += d0_; r5 += d1_; } // t1214 mm8 a=r4 b=r2 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r1 += d0_; r7 += d1_; } // t1215 mm8 a=r7 b=r2 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r7 += d0_; r0 += d1_; } // t1216 mm8 a=r3 b=r3 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r3 += d0_; r5 += d1_; } // t1217 mm8 a=r5 b=r5 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r3 += d0_; r6 += d1_; } // t1218 mm8 a=r0 b=r7 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r0 += d0_; r4 += d1_; } // t1219 mm8 a=r2 b=r4 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r7 += d0_; r4 += d1_; } // t1220 mm8 a=r1 b=r7 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r3 += d0_; r2 += d1_; } // t1221 mm8 a=r5 b=r5 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r2 += d0_; r0 += d1_; } // t1222 mm8 a=r1 b=r3 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r7 += d0_; r5 += d1_; } // t1223 mm8 a=r0 b=r2 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r6 += d0_; r3 += d1_; } // t1224 mm8 a=r1 b=r1 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r6 += d0_; r3 += d1_; } // t1225 mm8 a=r4 b=r1 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r4 += d0_; r5 += d1_; } // t1226 mm8 a=r5 b=r6 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r1 += d0_; r6 += d1_; } // t1227 mm8 a=r1 b=r7 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r4 += d0_; r5 += d1_; } // t1228 mm8 a=r4 b=r2 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r0 += d0_; r6 += d1_; } // t1229 mm8 a=r5 b=r3 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r2 += d0_; r7 += d1_; } // t1230 mm8 a=r6 b=r2 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r5 += d0_; r3 += d1_; } // t1231 mm8 a=r4 b=r3 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r4 += d1_; } // t1232 mm8 a=r6 b=r7 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r4 += d0_; r2 += d1_; } // t1233 mm8 a=r0 b=r5 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r7 += d0_; r1 += d1_; } // t1234 mm8 a=r5 b=r4 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r3 += d0_; r5 += d1_; } // t1235 mm8 a=r5 b=r7 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r1 += d0_; r3 += d1_; } // t1236 mm8 a=r0 b=r7 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r4 += d0_; r1 += d1_; } // t1237 mm8 a=r7 b=r2 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r5 += d0_; r0 += d1_; } // t1238 mm8 a=r7 b=r0 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r3 += d0_; r2 += d1_; } // t1239 mm8 a=r3 b=r2 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r2 += d0_; r4 += d1_; } // t1240 mm8 a=r0 b=r5 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r3 += d0_; r7 += d1_; } // t1241 mm8 a=r5 b=r0 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r7 += d0_; r4 += d1_; } // t1242 mm8 a=r5 b=r7 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r6 += d0_; r7 += d1_; } // t1243 mm8 a=r6 b=r3 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t1244 mm8 a=r1 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r5 += d0_; r1 += d1_; } // t1245 mm8 a=r0 b=r6 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r1 += d0_; r4 += d1_; } // t1246 mm8 a=r5 b=r5 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r7 += d0_; r1 += d1_; } // t1247 mm8 a=r1 b=r6 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r2 += d0_; r7 += d1_; } // t1248 mm8 a=r4 b=r0 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r1 += d0_; r7 += d1_; } // t1249 mm8 a=r7 b=r6 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r6 += d0_; r1 += d1_; } // t1250 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r1 += d0_; r0 += d1_; } // t1251 mm8 a=r3 b=r7 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r2 += d0_; r7 += d1_; } // t1252 mm8 a=r5 b=r5 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r6 += d0_; r2 += d1_; } // t1253 mm8 a=r5 b=r5 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r3 += d0_; r5 += d1_; } // t1254 mm8 a=r0 b=r6 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t1255 mm8 a=r1 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r5 += d0_; r6 += d1_; } // t1256 mm8 a=r1 b=r3 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r7 += d0_; r3 += d1_; } // t1257 mm8 a=r0 b=r7 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r4 += d0_; r3 += d1_; } // t1258 mm8 a=r5 b=r4 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r5 += d0_; r2 += d1_; } // t1259 mm8 a=r0 b=r3 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r5 += d0_; r7 += d1_; } // t1260 mm8 a=r1 b=r5 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r4 += d0_; r2 += d1_; } // t1261 mm8 a=r6 b=r7 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r5 += d0_; r1 += d1_; } // t1262 mm8 a=r2 b=r5 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r4 += d0_; r1 += d1_; } // t1263 mm8 a=r3 b=r7 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r6 += d0_; r7 += d1_; } // t1264 mm8 a=r1 b=r6 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r3 += d0_; r6 += d1_; } // t1265 mm8 a=r2 b=r3 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r3 += d1_; } // t1266 mm8 a=r4 b=r6 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r7 += d0_; r3 += d1_; } // t1267 mm8 a=r6 b=r6 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r5 += d0_; r7 += d1_; } // t1268 mm8 a=r4 b=r5 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r7 += d0_; r4 += d1_; } // t1269 mm8 a=r5 b=r7 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r7 += d0_; r3 += d1_; } // t1270 mm8 a=r7 b=r3 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r6 += d0_; r0 += d1_; } // t1271 mm8 a=r7 b=r0 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r1 += d0_; r3 += d1_; } // t1272 mm8 a=r1 b=r3 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r7 += d0_; r5 += d1_; } // t1273 mm8 a=r3 b=r2 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r6 += d0_; r3 += d1_; } // t1274 mm8 a=r2 b=r0 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r0 += d0_; r5 += d1_; } // t1275 mm8 a=r7 b=r3 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r5 += d0_; r2 += d1_; } // t1276 mm8 a=r4 b=r3 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r2 += d0_; r4 += d1_; } // t1277 mm8 a=r5 b=r4 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r1 += d0_; r6 += d1_; } // t1278 mm8 a=r6 b=r3 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r7 += d0_; r1 += d1_; } // t1279 mm8 a=r0 b=r1 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r4 += d0_; r2 += d1_; } // t1280 mm8 a=r4 b=r1 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r2 += d0_; r4 += d1_; } // t1281 mm8 a=r6 b=r3 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r2 += d0_; r5 += d1_; } // t1282 mm8 a=r1 b=r7 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r7 += d0_; r1 += d1_; } // t1283 mm8 a=r6 b=r2 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r7 += d0_; r2 += d1_; } // t1284 mm8 a=r1 b=r6 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r1 += d0_; r4 += d1_; } // t1285 mm8 a=r3 b=r5 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r2 += d0_; r4 += d1_; } // t1286 mm8 a=r3 b=r1 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r5 += d0_; r3 += d1_; } // t1287 mm8 a=r4 b=r1 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r6 += d0_; r0 += d1_; } // t1288 mm8 a=r3 b=r6 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r2 += d0_; r5 += d1_; } // t1289 mm8 a=r6 b=r2 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r1 += d0_; r5 += d1_; } // t1290 mm8 a=r6 b=r5 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r7 += d0_; r6 += d1_; } // t1291 mm8 a=r5 b=r6 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r1 += d0_; r6 += d1_; } // t1292 mm8 a=r6 b=r6 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r0 += d0_; r4 += d1_; } // t1293 mm8 a=r6 b=r2 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r2 += d0_; r3 += d1_; } // t1294 mm8 a=r0 b=r5 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r2 += d0_; r6 += d1_; } // t1295 mm8 a=r5 b=r0 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r3 += d0_; r5 += d1_; } // t1296 mm8 a=r1 b=r0 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r6 += d0_; r1 += d1_; } // t1297 mm8 a=r0 b=r0 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r0 += d0_; r4 += d1_; } // t1298 mm8 a=r2 b=r6 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r7 += d0_; r4 += d1_; } // t1299 mm8 a=r7 b=r0 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r4 += d0_; r5 += d1_; } // t1300 mm8 a=r4 b=r1 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r2 += d0_; r5 += d1_; } // t1301 mm8 a=r4 b=r5 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r7 += d0_; r2 += d1_; } // t1302 mm8 a=r0 b=r4 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r7 += d0_; r1 += d1_; } // t1303 mm8 a=r4 b=r2 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r3 += d0_; r7 += d1_; } // t1304 mm8 a=r4 b=r7 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r2 += d0_; r4 += d1_; } // t1305 mm8 a=r3 b=r2 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t1306 mm8 a=r3 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r6 += d0_; r7 += d1_; } // t1307 mm8 a=r0 b=r2 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r7 += d0_; r1 += d1_; } // t1308 mm8 a=r2 b=r5 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r5 += d0_; r4 += d1_; } // t1309 mm8 a=r5 b=r7 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r4 += d0_; r7 += d1_; } // t1310 mm8 a=r4 b=r5 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r1 += d0_; r4 += d1_; } // t1311 mm8 a=r4 b=r7 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t1312 mm8 a=r3 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r7 += d0_; r0 += d1_; } // t1313 mm8 a=r5 b=r1 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r0 += d0_; r2 += d1_; } // t1314 mm8 a=r5 b=r2 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r3 += d0_; r7 += d1_; } // t1315 mm8 a=r1 b=r0 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r6 += d0_; r7 += d1_; } // t1316 mm8 a=r7 b=r3 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r0 += d0_; r3 += d1_; } // t1317 mm8 a=r2 b=r5 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r5 += d0_; r4 += d1_; } // t1318 mm8 a=r2 b=r3 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r0 += d0_; r5 += d1_; } // t1319 mm8 a=r7 b=r5 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r4 += d0_; r6 += d1_; } // t1320 mm8 a=r0 b=r3 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r6 += d0_; r1 += d1_; } // t1321 mm8 a=r1 b=r0 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r6 += d0_; r3 += d1_; } // t1322 mm8 a=r4 b=r0 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r0 += d0_; r3 += d1_; } // t1323 mm8 a=r5 b=r3 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r4 += d0_; r6 += d1_; } // t1324 mm8 a=r3 b=r0 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r1 += d0_; r0 += d1_; } // t1325 mm8 a=r6 b=r4 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r0 += d1_; } // t1326 mm8 a=r4 b=r6 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r0 += d0_; r4 += d1_; } // t1327 mm8 a=r1 b=r3 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r7 += d0_; r6 += d1_; } // t1328 mm8 a=r7 b=r3 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r6 += d0_; r2 += d1_; } // t1329 mm8 a=r3 b=r7 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r5 += d0_; r3 += d1_; } // t1330 mm8 a=r3 b=r6 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r2 += d0_; r3 += d1_; } // t1331 mm8 a=r0 b=r7 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r5 += d0_; r4 += d1_; } // t1332 mm8 a=r0 b=r7 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r5 += d0_; r1 += d1_; } // t1333 mm8 a=r3 b=r3 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r0 += d0_; r3 += d1_; } // t1334 mm8 a=r7 b=r5 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r7 += d0_; r6 += d1_; } // t1335 mm8 a=r5 b=r7 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r1 += d0_; r2 += d1_; } // t1336 mm8 a=r5 b=r5 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r6 += d0_; r7 += d1_; } // t1337 mm8 a=r6 b=r0 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r7 += d0_; r6 += d1_; } // t1338 mm8 a=r6 b=r4 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r6 += d0_; r0 += d1_; } // t1339 mm8 a=r4 b=r6 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r7 += d0_; r0 += d1_; } // t1340 mm8 a=r6 b=r1 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r1 += d0_; r4 += d1_; } // t1341 mm8 a=r0 b=r6 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r0 += d0_; r1 += d1_; } // t1342 mm8 a=r4 b=r6 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r2 += d0_; r1 += d1_; } // t1343 mm8 a=r5 b=r5 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r3 += d0_; r1 += d1_; } // t1344 mm8 a=r2 b=r6 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r4 += d0_; r2 += d1_; } // t1345 mm8 a=r7 b=r3 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r0 += d0_; r2 += d1_; } // t1346 mm8 a=r5 b=r0 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r4 += d0_; r2 += d1_; } // t1347 mm8 a=r2 b=r3 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r2 += d0_; r1 += d1_; } // t1348 mm8 a=r4 b=r3 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r3 += d0_; r1 += d1_; } // t1349 mm8 a=r0 b=r1 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r0 += d0_; r4 += d1_; } // t1350 mm8 a=r1 b=r6 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r2 += d1_; } // t1351 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r5 += d0_; r0 += d1_; } // t1352 mm8 a=r5 b=r4 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t1353 mm8 a=r2 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r7 += d0_; r0 += d1_; } // t1354 mm8 a=r7 b=r4 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r3 += d0_; r0 += d1_; } // t1355 mm8 a=r7 b=r4 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r5 += d0_; r3 += d1_; } // t1356 mm8 a=r4 b=r6 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r6 += d1_; } // t1357 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r7 += d0_; r1 += d1_; } // t1358 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t1359 mm8 a=r7 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t1360 mm8 a=r0 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r7 += d0_; r4 += d1_; } // t1361 mm8 a=r0 b=r5 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r4 += d0_; r0 += d1_; } // t1362 mm8 a=r0 b=r2 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r5 += d0_; r6 += d1_; } // t1363 mm8 a=r4 b=r4 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r0 += d0_; r2 += d1_; } // t1364 mm8 a=r7 b=r3 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r3 += d0_; r4 += d1_; } // t1365 mm8 a=r2 b=r4 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r1 += d0_; r5 += d1_; } // t1366 mm8 a=r4 b=r3 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r6 += d0_; r2 += d1_; } // t1367 mm8 a=r3 b=r3 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r5 += d0_; r6 += d1_; } // t1368 mm8 a=r4 b=r0 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r2 += d0_; r7 += d1_; } // t1369 mm8 a=r5 b=r3 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t1370 mm8 a=r0 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r0 += d0_; r3 += d1_; } // t1371 mm8 a=r0 b=r5 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r7 += d0_; r2 += d1_; } // t1372 mm8 a=r5 b=r6 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r3 += d0_; r2 += d1_; } // t1373 mm8 a=r7 b=r1 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t1374 mm8 a=r4 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r2 += d0_; r5 += d1_; } // t1375 mm8 a=r5 b=r2 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r5 += d0_; r2 += d1_; } // t1376 mm8 a=r5 b=r5 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r7 += d0_; r5 += d1_; } // t1377 mm8 a=r7 b=r3 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r2 += d0_; r6 += d1_; } // t1378 mm8 a=r4 b=r1 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r7 += d0_; r2 += d1_; } // t1379 mm8 a=r4 b=r0 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r3 += d0_; r6 += d1_; } // t1380 mm8 a=r0 b=r7 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r0 += d0_; r7 += d1_; } // t1381 mm8 a=r2 b=r0 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r4 += d0_; r7 += d1_; } // t1382 mm8 a=r7 b=r5 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r1 += d1_; } // t1383 mm8 a=r1 b=r5 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r2 += d0_; r6 += d1_; } // t1384 mm8 a=r5 b=r2 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r5 += d0_; r1 += d1_; } // t1385 mm8 a=r0 b=r7 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r0 += d0_; r1 += d1_; } // t1386 mm8 a=r7 b=r0 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r0 += d0_; r2 += d1_; } // t1387 mm8 a=r4 b=r3 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r7 += d0_; r5 += d1_; } // t1388 mm8 a=r4 b=r6 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r3 += d0_; r2 += d1_; } // t1389 mm8 a=r4 b=r2 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r0 += d0_; r3 += d1_; } // t1390 mm8 a=r5 b=r1 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r4 += d0_; r0 += d1_; } // t1391 mm8 a=r6 b=r1 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r0 += d0_; r6 += d1_; } // t1392 mm8 a=r5 b=r7 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r7 += d0_; r4 += d1_; } // t1393 mm8 a=r3 b=r4 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r3 += d0_; r5 += d1_; } // t1394 mm8 a=r4 b=r7 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r6 += d0_; r5 += d1_; } // t1395 mm8 a=r7 b=r1 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r6 += d0_; r1 += d1_; } // t1396 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t1397 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r4 += d0_; r3 += d1_; } // t1398 mm8 a=r3 b=r0 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t1399 mm8 a=r3 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r2 += d0_; r5 += d1_; } // t1400 mm8 a=r6 b=r4 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r3 += d0_; r5 += d1_; } // t1401 mm8 a=r0 b=r5 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r6 += d0_; r2 += d1_; } // t1402 mm8 a=r7 b=r5 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r4 += d0_; r7 += d1_; } // t1403 mm8 a=r0 b=r0 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r6 += d0_; r2 += d1_; } // t1404 mm8 a=r3 b=r6 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r2 += d0_; r3 += d1_; } // t1405 mm8 a=r0 b=r0 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r4 += d0_; r5 += d1_; } // t1406 mm8 a=r5 b=r3 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r3 += d0_; r1 += d1_; } // t1407 mm8 a=r7 b=r3 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r0 += d0_; r1 += d1_; } // t1408 mm8 a=r4 b=r7 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r6 += d0_; r2 += d1_; } // t1409 mm8 a=r4 b=r2 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r6 += d0_; r3 += d1_; } // t1410 mm8 a=r6 b=r0 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r4 += d1_; } // t1411 mm8 a=r1 b=r7 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r3 += d0_; r2 += d1_; } // t1412 mm8 a=r4 b=r3 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r4 += d0_; r2 += d1_; } // t1413 mm8 a=r4 b=r6 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r2 += d0_; r5 += d1_; } // t1414 mm8 a=r0 b=r7 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r7 += d0_; r5 += d1_; } // t1415 mm8 a=r3 b=r6 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r4 += d0_; r7 += d1_; } // t1416 mm8 a=r3 b=r2 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r0 += d0_; r2 += d1_; } // t1417 mm8 a=r2 b=r7 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r2 += d0_; r7 += d1_; } // t1418 mm8 a=r0 b=r6 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t1419 mm8 a=r0 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r5 += d0_; r1 += d1_; } // t1420 mm8 a=r1 b=r3 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r3 += d0_; r2 += d1_; } // t1421 mm8 a=r4 b=r0 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r6 += d0_; r2 += d1_; } // t1422 mm8 a=r6 b=r4 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r3 += d0_; r1 += d1_; } // t1423 mm8 a=r4 b=r6 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r2 += d0_; r5 += d1_; } // t1424 mm8 a=r6 b=r3 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r0 += d0_; r1 += d1_; } // t1425 mm8 a=r7 b=r3 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r5 += d0_; r4 += d1_; } // t1426 mm8 a=r0 b=r3 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r3 += d0_; r6 += d1_; } // t1427 mm8 a=r5 b=r6 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r5 += d0_; r7 += d1_; } // t1428 mm8 a=r0 b=r3 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r3 += d0_; r7 += d1_; } // t1429 mm8 a=r7 b=r1 c=r3 c2=r7 - } - uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((ulong)hi << 32) | (ulong)lo; -} diff --git a/proto-cuda/packs-ca4/mx8_mm1430/kernel_bound.cu b/proto-cuda/packs-ca4/mx8_mm1430/kernel_bound.cu deleted file mode 100644 index f6687246c..000000000 --- a/proto-cuda/packs-ca4/mx8_mm1430/kernel_bound.cu +++ /dev/null @@ -1,1554 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// Header-bound twin of igneum_hash in kernel.cu: the init words come from a kernel argument, not SEEDW. -// Host declarations (also in program_bound.h if present): -// struct IgneumInitWords { uint32_t w[8]; }; -// cudaError_t igneum_launch_hash_bound(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask, -// IgneumInitWords iw, uint32_t nonces, uint32_t blockWarps); -// cudaError_t igneum_hash_bound_info(int* numRegs, int* blocksPerSM, uint32_t blockWarps); -#include -#include -#include "program.h" - -struct IgneumInitWords { uint32_t w[8]; }; - -__device__ __forceinline__ uint32_t splitmix32(uint32_t x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -__device__ __forceinline__ uint32_t rotl_imm(uint32_t x, uint32_t n) { return (x << n) | (x >> (32u - n)); } -__device__ __forceinline__ uint32_t rotr_var(uint32_t x, uint32_t n) { n &= 31u; return (x >> n) | (x << ((32u - n) & 31u)); } - -__global__ void igneum_hash_bound(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask, IgneumInitWords iw) { - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t nonce = baseNonce + gid; - uint32_t r0, r1, r2, r3, r4, r5, r6, r7; - { uint32_t x = nonce ^ iw.w[0]; x += 0x9e3779b9u * 1u; x = splitmix32(x); r0 = x ^ iw.w[1]; } - { uint32_t x = nonce ^ iw.w[1]; x += 0x9e3779b9u * 2u; x = splitmix32(x); r1 = x ^ iw.w[2]; } - { uint32_t x = nonce ^ iw.w[2]; x += 0x9e3779b9u * 3u; x = splitmix32(x); r2 = x ^ iw.w[3]; } - { uint32_t x = nonce ^ iw.w[3]; x += 0x9e3779b9u * 4u; x = splitmix32(x); r3 = x ^ iw.w[4]; } - { uint32_t x = nonce ^ iw.w[4]; x += 0x9e3779b9u * 5u; x = splitmix32(x); r4 = x ^ iw.w[5]; } - { uint32_t x = nonce ^ iw.w[5]; x += 0x9e3779b9u * 6u; x = splitmix32(x); r5 = x ^ iw.w[6]; } - { uint32_t x = nonce ^ iw.w[6]; x += 0x9e3779b9u * 7u; x = splitmix32(x); r6 = x ^ iw.w[7]; } - { uint32_t x = nonce ^ iw.w[7]; x += 0x9e3779b9u * 8u; x = splitmix32(x); r7 = x ^ iw.w[0]; } - - for (uint32_t it = 0u; it < 8u; ++it) { - uint32_t sel = r0; - r2 = r3 * r4 + r2; // 0 mad - r2 = r1 * r1 + r2; // 1 mad - r2 = r3 * r2 + r2; // 2 mad - r3 = r3 ^ r5; // 3 xor - r7 = r7 ^ ds[r2 & mask]; // 4 load - r5 = r5 ^ ds[r7 & mask]; // 5 load - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r4, 8); // 6 shfl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r3, 8); // 7 shfl - r1 = __umulhi(r1, r5); // 8 mulhi - r6 = rotr_var(r6, r3); // 9 rotr - r3 = r3 | r4; // 10 or - r4 = r4 ^ ds[r3 & mask]; // 11 load - r0 = __umulhi(r0, r4); // 12 mulhi - r5 = r5 + r1 + ((((sel >> 30u) & 1u) != 0u) ? 0xd3177981u : 0xc7934706u); // 13 add - r0 = r0 ^ ds[r4 & mask]; // 14 load - r2 = r2 - r4; // 15 sub - r2 = r2 ^ ds[r0 & mask]; // 16 load - r7 = r7 ^ ds[r2 & mask]; // 17 load - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r3, 4); // 18 shfl - r5 = r5 * r0; // 19 mul - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r4, 2); // 20 shfl - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r4, 16); // 21 shfl - r6 = __umulhi(r6, r2); // 22 mulhi - r6 = r6 ^ ds[r1 & mask]; // 23 load - r5 = r5 * r0; // 24 mul - r5 = rotl_imm(r5, 19u); // 25 rotl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r6, 2); // 26 shfl - r0 = r0 ^ r5; // 27 xor - r0 = r0 ^ r4; // 28 xor - r3 = r3 - r0; // 29 sub - r5 = r5 * r1; // 30 mul - r7 = r7 ^ ds[r2 & mask]; // 31 load - r1 = r1 ^ ds[r0 & mask]; // 32 load - r5 = r5 ^ r6; // 33 xor - r5 = r5 ^ ds[r1 & mask]; // 34 load - r0 = __umulhi(r0, r5); // 35 mulhi - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r2, 4); // 36 shfl - r7 = r7 ^ ds[r0 & mask]; // 37 load - r3 = r3 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0x230c005cu : 0x75ba2fadu); // 38 add - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r5, 4); // 39 shfl - r2 = r2 ^ r5; // 40 xor - r3 = r6 * r3 + r3; // 41 mad - r6 = r6 - r7; // 42 sub - r7 = r7 ^ r0; // 43 xor - r1 = r1 ^ ds[r7 & mask]; // 44 load - r2 = r2 * r3; // 45 mul - r1 = __umulhi(r1, r5); // 46 mulhi - r4 = r4 - r3; // 47 sub - r2 = rotr_var(r2, r6); // 48 rotr - r3 = r3 ^ ds[r5 & mask]; // 49 load - r1 = r1 + r5 + ((((sel >> 7u) & 1u) != 0u) ? 0x1907970cu : 0x81b8bc2cu); // 50 add - r0 = r0 * r2; // 51 mul - r0 = r0 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x699fd448u : 0x4f92b968u); // 52 add - r1 = r1 + r0 + ((((sel >> 12u) & 1u) != 0u) ? 0x77b1520du : 0x2bb965afu); // 53 add - r7 = rotl_imm(r7, 14u); // 54 rotl - r3 = r3 + r7 + ((((sel >> 1u) & 1u) != 0u) ? 0xa54c55a0u : 0x7b0fe07au); // 55 add - r6 = r6 ^ ds[r7 & mask]; // 56 load - r1 = rotr_var(r1, r5); // 57 rotr - r5 = r5 ^ ds[r4 & mask]; // 58 load - r6 = r6 ^ ds[r2 & mask]; // 59 load - r3 = r5 * r0 + r3; // 60 mad - r5 = r5 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0xad7493e7u : 0xaf9dd72du); // 61 add - r4 = r4 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x1e07c3d9u : 0x89841d87u); // 62 add - r5 = rotl_imm(r5, 19u); // 63 rotl - // int8 tile block (Counter ASIC 4.0 research, experimental): 1430 mma.m8n8k16 u8 tiles per iteration, both outputs consumed - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t0 mm8 a=r6 b=r5 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t1 mm8 a=r1 b=r1 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t2 mm8 a=r2 b=r5 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t3 mm8 a=r1 b=r5 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t4 mm8 a=r2 b=r0 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t5 mm8 a=r2 b=r6 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t6 mm8 a=r1 b=r4 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t7 mm8 a=r4 b=r6 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t8 mm8 a=r1 b=r1 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t9 mm8 a=r3 b=r7 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t10 mm8 a=r3 b=r0 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t11 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t12 mm8 a=r3 b=r1 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t13 mm8 a=r1 b=r5 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t14 mm8 a=r3 b=r3 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t15 mm8 a=r1 b=r0 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t16 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t17 mm8 a=r0 b=r5 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t18 mm8 a=r1 b=r1 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t19 mm8 a=r3 b=r3 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t20 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t21 mm8 a=r0 b=r3 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t22 mm8 a=r1 b=r7 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t23 mm8 a=r1 b=r2 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t24 mm8 a=r3 b=r4 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t25 mm8 a=r5 b=r0 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t26 mm8 a=r1 b=r0 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t27 mm8 a=r3 b=r2 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t28 mm8 a=r2 b=r4 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t29 mm8 a=r3 b=r4 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t30 mm8 a=r6 b=r5 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t31 mm8 a=r1 b=r7 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t32 mm8 a=r7 b=r6 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t33 mm8 a=r7 b=r1 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t34 mm8 a=r1 b=r0 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t35 mm8 a=r2 b=r4 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t36 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t37 mm8 a=r1 b=r7 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t38 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t39 mm8 a=r6 b=r2 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t40 mm8 a=r6 b=r1 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t41 mm8 a=r6 b=r0 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t42 mm8 a=r7 b=r4 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t43 mm8 a=r6 b=r1 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t44 mm8 a=r0 b=r6 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t45 mm8 a=r6 b=r5 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t46 mm8 a=r6 b=r6 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t47 mm8 a=r7 b=r3 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t48 mm8 a=r7 b=r7 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t49 mm8 a=r4 b=r5 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t50 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t51 mm8 a=r3 b=r4 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t52 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t53 mm8 a=r7 b=r7 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t54 mm8 a=r7 b=r6 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t55 mm8 a=r1 b=r7 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t56 mm8 a=r4 b=r0 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t57 mm8 a=r4 b=r3 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t58 mm8 a=r2 b=r3 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t59 mm8 a=r6 b=r7 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t60 mm8 a=r6 b=r0 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t61 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t62 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t63 mm8 a=r1 b=r6 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t64 mm8 a=r0 b=r5 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t65 mm8 a=r0 b=r1 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t66 mm8 a=r3 b=r5 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t67 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t68 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t69 mm8 a=r6 b=r7 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t70 mm8 a=r1 b=r7 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t71 mm8 a=r3 b=r0 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t72 mm8 a=r4 b=r0 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t73 mm8 a=r4 b=r1 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t74 mm8 a=r5 b=r1 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t75 mm8 a=r4 b=r4 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t76 mm8 a=r5 b=r6 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t77 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t78 mm8 a=r2 b=r5 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t79 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t80 mm8 a=r6 b=r7 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t81 mm8 a=r4 b=r3 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t82 mm8 a=r6 b=r0 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t83 mm8 a=r2 b=r2 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t84 mm8 a=r0 b=r6 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t85 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t86 mm8 a=r3 b=r4 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t87 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t88 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t89 mm8 a=r4 b=r2 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t90 mm8 a=r6 b=r5 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t91 mm8 a=r6 b=r5 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t92 mm8 a=r2 b=r6 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t93 mm8 a=r2 b=r2 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t94 mm8 a=r2 b=r4 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t95 mm8 a=r5 b=r4 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t96 mm8 a=r0 b=r6 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t97 mm8 a=r0 b=r5 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t98 mm8 a=r2 b=r7 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t99 mm8 a=r1 b=r4 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t100 mm8 a=r2 b=r4 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t101 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t102 mm8 a=r5 b=r7 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t103 mm8 a=r4 b=r7 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t104 mm8 a=r2 b=r0 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t105 mm8 a=r6 b=r1 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t106 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t107 mm8 a=r2 b=r1 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t108 mm8 a=r4 b=r6 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t109 mm8 a=r2 b=r7 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t110 mm8 a=r1 b=r0 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t111 mm8 a=r6 b=r5 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t112 mm8 a=r5 b=r7 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t113 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t114 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t115 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t116 mm8 a=r7 b=r2 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t117 mm8 a=r7 b=r6 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t118 mm8 a=r0 b=r1 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t119 mm8 a=r4 b=r3 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t120 mm8 a=r5 b=r3 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t121 mm8 a=r2 b=r7 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t122 mm8 a=r0 b=r1 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t123 mm8 a=r5 b=r2 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t124 mm8 a=r5 b=r1 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t125 mm8 a=r7 b=r2 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t126 mm8 a=r6 b=r3 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t127 mm8 a=r6 b=r7 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t128 mm8 a=r6 b=r7 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t129 mm8 a=r3 b=r2 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t130 mm8 a=r0 b=r4 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t131 mm8 a=r3 b=r5 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t132 mm8 a=r0 b=r7 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t133 mm8 a=r2 b=r6 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t134 mm8 a=r5 b=r6 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t135 mm8 a=r0 b=r2 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t136 mm8 a=r3 b=r1 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t137 mm8 a=r7 b=r0 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t138 mm8 a=r2 b=r2 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t139 mm8 a=r0 b=r4 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t140 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t141 mm8 a=r5 b=r5 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t142 mm8 a=r2 b=r2 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t143 mm8 a=r4 b=r3 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t144 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t145 mm8 a=r7 b=r2 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t146 mm8 a=r2 b=r7 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t147 mm8 a=r3 b=r1 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t148 mm8 a=r2 b=r1 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t149 mm8 a=r4 b=r6 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t150 mm8 a=r3 b=r6 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t151 mm8 a=r1 b=r4 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t152 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t153 mm8 a=r0 b=r3 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t154 mm8 a=r7 b=r3 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t155 mm8 a=r1 b=r3 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t156 mm8 a=r6 b=r3 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t157 mm8 a=r6 b=r4 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t158 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t159 mm8 a=r1 b=r0 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t160 mm8 a=r0 b=r1 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t161 mm8 a=r4 b=r0 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t162 mm8 a=r6 b=r6 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t163 mm8 a=r5 b=r3 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t164 mm8 a=r0 b=r7 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t165 mm8 a=r6 b=r5 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t166 mm8 a=r6 b=r5 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t167 mm8 a=r4 b=r5 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t168 mm8 a=r3 b=r6 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t169 mm8 a=r6 b=r2 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t170 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t171 mm8 a=r6 b=r3 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t172 mm8 a=r7 b=r3 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t173 mm8 a=r0 b=r0 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t174 mm8 a=r6 b=r6 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t175 mm8 a=r1 b=r6 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t176 mm8 a=r2 b=r4 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t177 mm8 a=r1 b=r7 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t178 mm8 a=r4 b=r2 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t179 mm8 a=r5 b=r2 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t180 mm8 a=r0 b=r4 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t181 mm8 a=r3 b=r5 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t182 mm8 a=r1 b=r1 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t183 mm8 a=r3 b=r1 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t184 mm8 a=r7 b=r4 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t185 mm8 a=r5 b=r0 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t186 mm8 a=r2 b=r5 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t187 mm8 a=r3 b=r0 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t188 mm8 a=r1 b=r4 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t189 mm8 a=r7 b=r2 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t190 mm8 a=r1 b=r5 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t191 mm8 a=r2 b=r1 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t192 mm8 a=r5 b=r0 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t193 mm8 a=r2 b=r1 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t194 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t195 mm8 a=r2 b=r7 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t196 mm8 a=r5 b=r6 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t197 mm8 a=r5 b=r3 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t198 mm8 a=r1 b=r0 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t199 mm8 a=r2 b=r5 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t200 mm8 a=r5 b=r7 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t201 mm8 a=r7 b=r2 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t202 mm8 a=r2 b=r5 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t203 mm8 a=r0 b=r5 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t204 mm8 a=r4 b=r7 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t205 mm8 a=r3 b=r7 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t206 mm8 a=r0 b=r6 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t207 mm8 a=r7 b=r7 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t208 mm8 a=r0 b=r1 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t209 mm8 a=r4 b=r4 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t210 mm8 a=r7 b=r1 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t211 mm8 a=r1 b=r5 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t212 mm8 a=r2 b=r4 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t213 mm8 a=r2 b=r2 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t214 mm8 a=r4 b=r6 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t215 mm8 a=r7 b=r3 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t216 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t217 mm8 a=r5 b=r7 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t218 mm8 a=r7 b=r1 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t219 mm8 a=r2 b=r1 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t220 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t221 mm8 a=r7 b=r1 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t222 mm8 a=r0 b=r6 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t223 mm8 a=r2 b=r2 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t224 mm8 a=r1 b=r1 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t225 mm8 a=r2 b=r7 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t226 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t227 mm8 a=r3 b=r5 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t228 mm8 a=r7 b=r1 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t229 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t230 mm8 a=r0 b=r3 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t231 mm8 a=r1 b=r3 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t232 mm8 a=r5 b=r4 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t233 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t234 mm8 a=r2 b=r0 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t235 mm8 a=r3 b=r4 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t236 mm8 a=r1 b=r7 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t237 mm8 a=r3 b=r5 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t238 mm8 a=r2 b=r0 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t239 mm8 a=r5 b=r7 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t240 mm8 a=r2 b=r7 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t241 mm8 a=r0 b=r1 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t242 mm8 a=r0 b=r2 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t243 mm8 a=r4 b=r4 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t244 mm8 a=r7 b=r6 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t245 mm8 a=r4 b=r4 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t246 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t247 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t248 mm8 a=r7 b=r5 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t249 mm8 a=r4 b=r4 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t250 mm8 a=r0 b=r6 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t251 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t252 mm8 a=r0 b=r2 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t253 mm8 a=r5 b=r5 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t254 mm8 a=r6 b=r7 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t255 mm8 a=r5 b=r0 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t256 mm8 a=r2 b=r1 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t257 mm8 a=r4 b=r3 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t258 mm8 a=r4 b=r1 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t259 mm8 a=r5 b=r5 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t260 mm8 a=r3 b=r2 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t261 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t262 mm8 a=r6 b=r4 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t263 mm8 a=r6 b=r4 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t264 mm8 a=r7 b=r2 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t265 mm8 a=r6 b=r1 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t266 mm8 a=r0 b=r0 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t267 mm8 a=r6 b=r3 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t268 mm8 a=r1 b=r0 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t269 mm8 a=r7 b=r7 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t270 mm8 a=r6 b=r2 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t271 mm8 a=r0 b=r3 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t272 mm8 a=r6 b=r0 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t273 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t274 mm8 a=r1 b=r0 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t275 mm8 a=r1 b=r3 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t276 mm8 a=r1 b=r5 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t277 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t278 mm8 a=r0 b=r0 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t279 mm8 a=r2 b=r4 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t280 mm8 a=r7 b=r2 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t281 mm8 a=r1 b=r0 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t282 mm8 a=r0 b=r1 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t283 mm8 a=r0 b=r2 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t284 mm8 a=r1 b=r5 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t285 mm8 a=r4 b=r7 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t286 mm8 a=r1 b=r0 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t287 mm8 a=r7 b=r1 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t288 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t289 mm8 a=r0 b=r5 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t290 mm8 a=r3 b=r2 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t291 mm8 a=r7 b=r4 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t292 mm8 a=r6 b=r7 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t293 mm8 a=r5 b=r5 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t294 mm8 a=r5 b=r7 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t295 mm8 a=r0 b=r7 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t296 mm8 a=r6 b=r1 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t297 mm8 a=r4 b=r5 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t298 mm8 a=r6 b=r0 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t299 mm8 a=r6 b=r0 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t300 mm8 a=r6 b=r4 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t301 mm8 a=r6 b=r4 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t302 mm8 a=r7 b=r6 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t303 mm8 a=r3 b=r2 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t304 mm8 a=r6 b=r5 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t305 mm8 a=r4 b=r5 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t306 mm8 a=r3 b=r1 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t307 mm8 a=r4 b=r3 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t308 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t309 mm8 a=r2 b=r1 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t310 mm8 a=r6 b=r1 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t311 mm8 a=r4 b=r6 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t312 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t313 mm8 a=r2 b=r6 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t314 mm8 a=r0 b=r3 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t315 mm8 a=r7 b=r1 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t316 mm8 a=r2 b=r4 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t317 mm8 a=r1 b=r6 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t318 mm8 a=r2 b=r1 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t319 mm8 a=r0 b=r2 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t320 mm8 a=r5 b=r2 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t321 mm8 a=r7 b=r4 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t322 mm8 a=r2 b=r4 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t323 mm8 a=r4 b=r3 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t324 mm8 a=r5 b=r7 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t325 mm8 a=r6 b=r2 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t326 mm8 a=r6 b=r1 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t327 mm8 a=r1 b=r5 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t328 mm8 a=r0 b=r1 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t329 mm8 a=r0 b=r3 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t330 mm8 a=r6 b=r7 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t331 mm8 a=r4 b=r4 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t332 mm8 a=r4 b=r4 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t333 mm8 a=r4 b=r6 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t334 mm8 a=r2 b=r2 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t335 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t336 mm8 a=r4 b=r0 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t337 mm8 a=r1 b=r4 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t338 mm8 a=r3 b=r7 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t339 mm8 a=r5 b=r1 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t340 mm8 a=r5 b=r0 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t341 mm8 a=r6 b=r6 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t342 mm8 a=r4 b=r7 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t343 mm8 a=r0 b=r4 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t344 mm8 a=r2 b=r5 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t345 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t346 mm8 a=r6 b=r0 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t347 mm8 a=r5 b=r3 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t348 mm8 a=r0 b=r4 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t349 mm8 a=r6 b=r1 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t350 mm8 a=r1 b=r5 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t351 mm8 a=r6 b=r4 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t352 mm8 a=r4 b=r1 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t353 mm8 a=r7 b=r3 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t354 mm8 a=r4 b=r1 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t355 mm8 a=r4 b=r5 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t356 mm8 a=r7 b=r6 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t357 mm8 a=r7 b=r3 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t358 mm8 a=r4 b=r7 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t359 mm8 a=r2 b=r7 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t360 mm8 a=r4 b=r4 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t361 mm8 a=r2 b=r2 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t362 mm8 a=r2 b=r7 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t363 mm8 a=r7 b=r6 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t364 mm8 a=r6 b=r4 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t365 mm8 a=r6 b=r6 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t366 mm8 a=r7 b=r7 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t367 mm8 a=r4 b=r1 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t368 mm8 a=r7 b=r0 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t369 mm8 a=r1 b=r0 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t370 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t371 mm8 a=r5 b=r4 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t372 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t373 mm8 a=r4 b=r6 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t374 mm8 a=r1 b=r5 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t375 mm8 a=r6 b=r5 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t376 mm8 a=r3 b=r6 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t377 mm8 a=r6 b=r4 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t378 mm8 a=r6 b=r3 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t379 mm8 a=r3 b=r0 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t380 mm8 a=r2 b=r0 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t381 mm8 a=r3 b=r5 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t382 mm8 a=r5 b=r2 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t383 mm8 a=r1 b=r2 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t384 mm8 a=r3 b=r1 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t385 mm8 a=r6 b=r0 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t386 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t387 mm8 a=r3 b=r7 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t388 mm8 a=r3 b=r0 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t389 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t390 mm8 a=r5 b=r4 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t391 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t392 mm8 a=r2 b=r3 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t393 mm8 a=r3 b=r6 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t394 mm8 a=r0 b=r7 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t395 mm8 a=r0 b=r7 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t396 mm8 a=r7 b=r7 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t397 mm8 a=r5 b=r7 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t398 mm8 a=r6 b=r5 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t399 mm8 a=r6 b=r3 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t400 mm8 a=r2 b=r7 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t401 mm8 a=r7 b=r6 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t402 mm8 a=r0 b=r0 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t403 mm8 a=r7 b=r4 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t404 mm8 a=r1 b=r1 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t405 mm8 a=r6 b=r6 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t406 mm8 a=r6 b=r1 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t407 mm8 a=r5 b=r7 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t408 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t409 mm8 a=r3 b=r4 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t410 mm8 a=r0 b=r5 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t411 mm8 a=r2 b=r0 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t412 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t413 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t414 mm8 a=r7 b=r1 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t415 mm8 a=r4 b=r7 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t416 mm8 a=r5 b=r4 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t417 mm8 a=r6 b=r6 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t418 mm8 a=r1 b=r3 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t419 mm8 a=r5 b=r0 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t420 mm8 a=r4 b=r2 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t421 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t422 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t423 mm8 a=r1 b=r7 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t424 mm8 a=r5 b=r1 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t425 mm8 a=r2 b=r7 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t426 mm8 a=r1 b=r6 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t427 mm8 a=r6 b=r6 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t428 mm8 a=r6 b=r2 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t429 mm8 a=r3 b=r3 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t430 mm8 a=r7 b=r7 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t431 mm8 a=r6 b=r4 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t432 mm8 a=r3 b=r2 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t433 mm8 a=r3 b=r0 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t434 mm8 a=r0 b=r7 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t435 mm8 a=r7 b=r5 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t436 mm8 a=r6 b=r4 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t437 mm8 a=r0 b=r7 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t438 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t439 mm8 a=r4 b=r7 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t440 mm8 a=r3 b=r5 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t441 mm8 a=r5 b=r1 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t442 mm8 a=r1 b=r2 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t443 mm8 a=r1 b=r7 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t444 mm8 a=r3 b=r7 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t445 mm8 a=r7 b=r5 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t446 mm8 a=r3 b=r1 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t447 mm8 a=r6 b=r7 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t448 mm8 a=r5 b=r2 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t449 mm8 a=r7 b=r1 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t450 mm8 a=r7 b=r5 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t451 mm8 a=r0 b=r3 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t452 mm8 a=r2 b=r4 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t453 mm8 a=r5 b=r7 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t454 mm8 a=r7 b=r7 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t455 mm8 a=r7 b=r1 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t456 mm8 a=r6 b=r5 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t457 mm8 a=r7 b=r7 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t458 mm8 a=r5 b=r1 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t459 mm8 a=r6 b=r7 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t460 mm8 a=r7 b=r7 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t461 mm8 a=r0 b=r6 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t462 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t463 mm8 a=r2 b=r0 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t464 mm8 a=r0 b=r3 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t465 mm8 a=r1 b=r7 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t466 mm8 a=r1 b=r2 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t467 mm8 a=r1 b=r0 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t468 mm8 a=r4 b=r1 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t469 mm8 a=r2 b=r0 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t470 mm8 a=r3 b=r1 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t471 mm8 a=r2 b=r2 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t472 mm8 a=r1 b=r7 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t473 mm8 a=r2 b=r7 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t474 mm8 a=r5 b=r3 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t475 mm8 a=r7 b=r6 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t476 mm8 a=r3 b=r0 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t477 mm8 a=r3 b=r1 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t478 mm8 a=r0 b=r6 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t479 mm8 a=r7 b=r6 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t480 mm8 a=r6 b=r0 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t481 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t482 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t483 mm8 a=r7 b=r0 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t484 mm8 a=r3 b=r7 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t485 mm8 a=r6 b=r4 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t486 mm8 a=r6 b=r5 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t487 mm8 a=r2 b=r5 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t488 mm8 a=r1 b=r2 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t489 mm8 a=r5 b=r2 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t490 mm8 a=r7 b=r1 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t491 mm8 a=r0 b=r0 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t492 mm8 a=r3 b=r1 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t493 mm8 a=r6 b=r1 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t494 mm8 a=r5 b=r6 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t495 mm8 a=r2 b=r2 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t496 mm8 a=r3 b=r5 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t497 mm8 a=r3 b=r2 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t498 mm8 a=r3 b=r1 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t499 mm8 a=r1 b=r6 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t500 mm8 a=r7 b=r6 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t501 mm8 a=r4 b=r4 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t502 mm8 a=r5 b=r7 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t503 mm8 a=r7 b=r5 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t504 mm8 a=r2 b=r0 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t505 mm8 a=r4 b=r0 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t506 mm8 a=r0 b=r2 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t507 mm8 a=r5 b=r6 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t508 mm8 a=r7 b=r3 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t509 mm8 a=r4 b=r4 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t510 mm8 a=r0 b=r6 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t511 mm8 a=r2 b=r4 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t512 mm8 a=r2 b=r7 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t513 mm8 a=r0 b=r3 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t514 mm8 a=r6 b=r7 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t515 mm8 a=r5 b=r2 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t516 mm8 a=r0 b=r2 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t517 mm8 a=r6 b=r7 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t518 mm8 a=r1 b=r4 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t519 mm8 a=r6 b=r0 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t520 mm8 a=r1 b=r2 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t521 mm8 a=r0 b=r4 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t522 mm8 a=r7 b=r2 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t523 mm8 a=r5 b=r3 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t524 mm8 a=r0 b=r4 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t525 mm8 a=r1 b=r5 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t526 mm8 a=r7 b=r1 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t527 mm8 a=r5 b=r1 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t528 mm8 a=r4 b=r1 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t529 mm8 a=r4 b=r2 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t530 mm8 a=r0 b=r2 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t531 mm8 a=r7 b=r0 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t532 mm8 a=r7 b=r2 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t533 mm8 a=r3 b=r2 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t534 mm8 a=r4 b=r3 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t535 mm8 a=r0 b=r6 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t536 mm8 a=r3 b=r1 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t537 mm8 a=r1 b=r6 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t538 mm8 a=r6 b=r6 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t539 mm8 a=r4 b=r4 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t540 mm8 a=r1 b=r4 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t541 mm8 a=r5 b=r3 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t542 mm8 a=r7 b=r7 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t543 mm8 a=r4 b=r2 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t544 mm8 a=r5 b=r7 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t545 mm8 a=r5 b=r2 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t546 mm8 a=r7 b=r1 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t547 mm8 a=r3 b=r5 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t548 mm8 a=r6 b=r3 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t549 mm8 a=r2 b=r2 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t550 mm8 a=r5 b=r5 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t551 mm8 a=r7 b=r6 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t552 mm8 a=r6 b=r7 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t553 mm8 a=r6 b=r3 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t554 mm8 a=r4 b=r2 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t555 mm8 a=r2 b=r7 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t556 mm8 a=r3 b=r6 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t557 mm8 a=r7 b=r7 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t558 mm8 a=r5 b=r0 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t559 mm8 a=r4 b=r1 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t560 mm8 a=r5 b=r4 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t561 mm8 a=r6 b=r1 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t562 mm8 a=r1 b=r2 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t563 mm8 a=r5 b=r3 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t564 mm8 a=r4 b=r3 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t565 mm8 a=r6 b=r0 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t566 mm8 a=r1 b=r6 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t567 mm8 a=r3 b=r6 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t568 mm8 a=r6 b=r5 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t569 mm8 a=r3 b=r7 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t570 mm8 a=r5 b=r1 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t571 mm8 a=r6 b=r0 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t572 mm8 a=r5 b=r0 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t573 mm8 a=r3 b=r0 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t574 mm8 a=r2 b=r2 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t575 mm8 a=r3 b=r1 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t576 mm8 a=r2 b=r3 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t577 mm8 a=r6 b=r3 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t578 mm8 a=r5 b=r6 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t579 mm8 a=r4 b=r7 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t580 mm8 a=r7 b=r1 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t581 mm8 a=r1 b=r6 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t582 mm8 a=r2 b=r1 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t583 mm8 a=r5 b=r0 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t584 mm8 a=r1 b=r7 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t585 mm8 a=r0 b=r4 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t586 mm8 a=r4 b=r4 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t587 mm8 a=r0 b=r4 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t588 mm8 a=r5 b=r5 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t589 mm8 a=r3 b=r6 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t590 mm8 a=r3 b=r2 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t591 mm8 a=r5 b=r1 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t592 mm8 a=r5 b=r3 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t593 mm8 a=r5 b=r3 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t594 mm8 a=r6 b=r7 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t595 mm8 a=r3 b=r0 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t596 mm8 a=r7 b=r6 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t597 mm8 a=r2 b=r4 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t598 mm8 a=r7 b=r7 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t599 mm8 a=r7 b=r2 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t600 mm8 a=r7 b=r7 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t601 mm8 a=r2 b=r7 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t602 mm8 a=r0 b=r1 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t603 mm8 a=r0 b=r0 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t604 mm8 a=r5 b=r5 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t605 mm8 a=r2 b=r0 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t606 mm8 a=r7 b=r5 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t607 mm8 a=r0 b=r2 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t608 mm8 a=r7 b=r6 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t609 mm8 a=r1 b=r5 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t610 mm8 a=r3 b=r3 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t611 mm8 a=r2 b=r2 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t612 mm8 a=r1 b=r1 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t613 mm8 a=r6 b=r3 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t614 mm8 a=r3 b=r4 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t615 mm8 a=r6 b=r2 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t616 mm8 a=r2 b=r3 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t617 mm8 a=r5 b=r4 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t618 mm8 a=r3 b=r6 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t619 mm8 a=r5 b=r0 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t620 mm8 a=r3 b=r5 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t621 mm8 a=r4 b=r2 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t622 mm8 a=r7 b=r0 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t623 mm8 a=r4 b=r6 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t624 mm8 a=r6 b=r2 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t625 mm8 a=r6 b=r2 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t626 mm8 a=r5 b=r2 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t627 mm8 a=r4 b=r0 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t628 mm8 a=r1 b=r5 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t629 mm8 a=r4 b=r5 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t630 mm8 a=r4 b=r5 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t631 mm8 a=r6 b=r4 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t632 mm8 a=r2 b=r0 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t633 mm8 a=r2 b=r1 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t634 mm8 a=r1 b=r1 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t635 mm8 a=r6 b=r7 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t636 mm8 a=r2 b=r5 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t637 mm8 a=r5 b=r7 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t638 mm8 a=r1 b=r4 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t639 mm8 a=r1 b=r7 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t640 mm8 a=r0 b=r4 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t641 mm8 a=r5 b=r6 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t642 mm8 a=r7 b=r5 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t643 mm8 a=r3 b=r5 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t644 mm8 a=r1 b=r6 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t645 mm8 a=r0 b=r7 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t646 mm8 a=r5 b=r6 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t647 mm8 a=r6 b=r4 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t648 mm8 a=r3 b=r1 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t649 mm8 a=r7 b=r2 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t650 mm8 a=r7 b=r7 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t651 mm8 a=r6 b=r3 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t652 mm8 a=r1 b=r6 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t653 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t654 mm8 a=r1 b=r6 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t655 mm8 a=r7 b=r5 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t656 mm8 a=r6 b=r5 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t657 mm8 a=r7 b=r4 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t658 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t659 mm8 a=r0 b=r7 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t660 mm8 a=r6 b=r1 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t661 mm8 a=r3 b=r7 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t662 mm8 a=r2 b=r3 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t663 mm8 a=r0 b=r3 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t664 mm8 a=r2 b=r0 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t665 mm8 a=r1 b=r1 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t666 mm8 a=r7 b=r1 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t667 mm8 a=r2 b=r0 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t668 mm8 a=r3 b=r1 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t669 mm8 a=r1 b=r4 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t670 mm8 a=r7 b=r7 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t671 mm8 a=r0 b=r3 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t672 mm8 a=r0 b=r1 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t673 mm8 a=r6 b=r2 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t674 mm8 a=r3 b=r3 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t675 mm8 a=r5 b=r6 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t676 mm8 a=r4 b=r3 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t677 mm8 a=r0 b=r1 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t678 mm8 a=r4 b=r7 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t679 mm8 a=r4 b=r7 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t680 mm8 a=r5 b=r1 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t681 mm8 a=r4 b=r2 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t682 mm8 a=r3 b=r6 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t683 mm8 a=r0 b=r5 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t684 mm8 a=r6 b=r2 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t685 mm8 a=r7 b=r3 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t686 mm8 a=r4 b=r1 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t687 mm8 a=r0 b=r5 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t688 mm8 a=r3 b=r6 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t689 mm8 a=r3 b=r1 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t690 mm8 a=r2 b=r6 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t691 mm8 a=r2 b=r6 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t692 mm8 a=r1 b=r4 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t693 mm8 a=r7 b=r1 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t694 mm8 a=r3 b=r5 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t695 mm8 a=r3 b=r2 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t696 mm8 a=r2 b=r4 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t697 mm8 a=r2 b=r3 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t698 mm8 a=r3 b=r0 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t699 mm8 a=r2 b=r1 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t700 mm8 a=r0 b=r1 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t701 mm8 a=r5 b=r7 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t702 mm8 a=r5 b=r5 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t703 mm8 a=r2 b=r2 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t704 mm8 a=r6 b=r5 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t705 mm8 a=r5 b=r4 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t706 mm8 a=r6 b=r0 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t707 mm8 a=r6 b=r4 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t708 mm8 a=r2 b=r0 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t709 mm8 a=r7 b=r0 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t710 mm8 a=r4 b=r2 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t711 mm8 a=r4 b=r7 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t712 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t713 mm8 a=r5 b=r2 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t714 mm8 a=r0 b=r0 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t715 mm8 a=r1 b=r7 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t716 mm8 a=r3 b=r3 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t717 mm8 a=r3 b=r3 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t718 mm8 a=r7 b=r5 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t719 mm8 a=r4 b=r7 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t720 mm8 a=r4 b=r7 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t721 mm8 a=r6 b=r4 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t722 mm8 a=r7 b=r5 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t723 mm8 a=r5 b=r0 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t724 mm8 a=r7 b=r4 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t725 mm8 a=r6 b=r1 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t726 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t727 mm8 a=r0 b=r6 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t728 mm8 a=r5 b=r7 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t729 mm8 a=r5 b=r6 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t730 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t731 mm8 a=r5 b=r6 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t732 mm8 a=r5 b=r1 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t733 mm8 a=r3 b=r4 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t734 mm8 a=r4 b=r3 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t735 mm8 a=r7 b=r4 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t736 mm8 a=r4 b=r2 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t737 mm8 a=r4 b=r7 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t738 mm8 a=r6 b=r2 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t739 mm8 a=r7 b=r5 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t740 mm8 a=r4 b=r3 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t741 mm8 a=r7 b=r0 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t742 mm8 a=r6 b=r5 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t743 mm8 a=r5 b=r3 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t744 mm8 a=r7 b=r4 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t745 mm8 a=r7 b=r0 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t746 mm8 a=r2 b=r5 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t747 mm8 a=r0 b=r3 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t748 mm8 a=r1 b=r6 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t749 mm8 a=r2 b=r5 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t750 mm8 a=r4 b=r3 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t751 mm8 a=r6 b=r6 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t752 mm8 a=r5 b=r0 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t753 mm8 a=r6 b=r7 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t754 mm8 a=r2 b=r4 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t755 mm8 a=r5 b=r6 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t756 mm8 a=r7 b=r3 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t757 mm8 a=r6 b=r6 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t758 mm8 a=r6 b=r2 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t759 mm8 a=r2 b=r7 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t760 mm8 a=r7 b=r7 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t761 mm8 a=r3 b=r3 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t762 mm8 a=r5 b=r2 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t763 mm8 a=r1 b=r1 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t764 mm8 a=r1 b=r2 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t765 mm8 a=r2 b=r4 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t766 mm8 a=r7 b=r6 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t767 mm8 a=r7 b=r5 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t768 mm8 a=r5 b=r6 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t769 mm8 a=r4 b=r5 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t770 mm8 a=r7 b=r5 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t771 mm8 a=r2 b=r5 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t772 mm8 a=r0 b=r4 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t773 mm8 a=r2 b=r5 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t774 mm8 a=r4 b=r0 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t775 mm8 a=r4 b=r6 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t776 mm8 a=r5 b=r7 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t777 mm8 a=r1 b=r1 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t778 mm8 a=r1 b=r2 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t779 mm8 a=r7 b=r5 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t780 mm8 a=r3 b=r7 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t781 mm8 a=r6 b=r7 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t782 mm8 a=r6 b=r4 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t783 mm8 a=r6 b=r5 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t784 mm8 a=r1 b=r5 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t785 mm8 a=r7 b=r2 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t786 mm8 a=r2 b=r4 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t787 mm8 a=r7 b=r2 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t788 mm8 a=r0 b=r0 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t789 mm8 a=r1 b=r4 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t790 mm8 a=r3 b=r1 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t791 mm8 a=r3 b=r5 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t792 mm8 a=r7 b=r1 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t793 mm8 a=r7 b=r2 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t794 mm8 a=r2 b=r4 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t795 mm8 a=r1 b=r7 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t796 mm8 a=r2 b=r4 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t797 mm8 a=r6 b=r2 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t798 mm8 a=r1 b=r3 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t799 mm8 a=r1 b=r3 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t800 mm8 a=r0 b=r4 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t801 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t802 mm8 a=r4 b=r7 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t803 mm8 a=r3 b=r2 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t804 mm8 a=r0 b=r4 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t805 mm8 a=r4 b=r6 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t806 mm8 a=r0 b=r5 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t807 mm8 a=r5 b=r0 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t808 mm8 a=r2 b=r4 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t809 mm8 a=r1 b=r6 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t810 mm8 a=r1 b=r2 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t811 mm8 a=r2 b=r7 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t812 mm8 a=r0 b=r6 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t813 mm8 a=r2 b=r2 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t814 mm8 a=r3 b=r4 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t815 mm8 a=r4 b=r5 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t816 mm8 a=r3 b=r0 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t817 mm8 a=r3 b=r4 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t818 mm8 a=r2 b=r2 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t819 mm8 a=r1 b=r2 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t820 mm8 a=r3 b=r6 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t821 mm8 a=r7 b=r3 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t822 mm8 a=r1 b=r5 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t823 mm8 a=r1 b=r4 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t824 mm8 a=r7 b=r7 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t825 mm8 a=r1 b=r1 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t826 mm8 a=r0 b=r2 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t827 mm8 a=r3 b=r1 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t828 mm8 a=r7 b=r4 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t829 mm8 a=r5 b=r2 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t830 mm8 a=r6 b=r3 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t831 mm8 a=r2 b=r7 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t832 mm8 a=r1 b=r6 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t833 mm8 a=r0 b=r6 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t834 mm8 a=r2 b=r4 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t835 mm8 a=r0 b=r7 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t836 mm8 a=r3 b=r0 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t837 mm8 a=r4 b=r7 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t838 mm8 a=r5 b=r4 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t839 mm8 a=r0 b=r6 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t840 mm8 a=r1 b=r6 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t841 mm8 a=r4 b=r0 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t842 mm8 a=r3 b=r5 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t843 mm8 a=r2 b=r2 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t844 mm8 a=r3 b=r3 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t845 mm8 a=r1 b=r7 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t846 mm8 a=r1 b=r4 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t847 mm8 a=r4 b=r0 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t848 mm8 a=r6 b=r7 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t849 mm8 a=r6 b=r6 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t850 mm8 a=r6 b=r0 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t851 mm8 a=r0 b=r5 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t852 mm8 a=r2 b=r2 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t853 mm8 a=r6 b=r5 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t854 mm8 a=r1 b=r4 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t855 mm8 a=r1 b=r7 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t856 mm8 a=r2 b=r3 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t857 mm8 a=r6 b=r5 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t858 mm8 a=r0 b=r5 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t859 mm8 a=r2 b=r6 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t860 mm8 a=r7 b=r2 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t861 mm8 a=r3 b=r3 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t862 mm8 a=r0 b=r1 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t863 mm8 a=r0 b=r0 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t864 mm8 a=r5 b=r5 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t865 mm8 a=r3 b=r7 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t866 mm8 a=r6 b=r6 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t867 mm8 a=r7 b=r3 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t868 mm8 a=r4 b=r4 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t869 mm8 a=r3 b=r6 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t870 mm8 a=r0 b=r1 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t871 mm8 a=r5 b=r1 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t872 mm8 a=r0 b=r6 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t873 mm8 a=r2 b=r1 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t874 mm8 a=r5 b=r1 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t875 mm8 a=r1 b=r7 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t876 mm8 a=r7 b=r6 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t877 mm8 a=r6 b=r2 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t878 mm8 a=r5 b=r0 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t879 mm8 a=r4 b=r1 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t880 mm8 a=r3 b=r3 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t881 mm8 a=r2 b=r7 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t882 mm8 a=r7 b=r5 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t883 mm8 a=r2 b=r3 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t884 mm8 a=r2 b=r7 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t885 mm8 a=r3 b=r5 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t886 mm8 a=r5 b=r4 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t887 mm8 a=r1 b=r1 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t888 mm8 a=r6 b=r2 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t889 mm8 a=r4 b=r6 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t890 mm8 a=r0 b=r5 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t891 mm8 a=r0 b=r7 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t892 mm8 a=r5 b=r6 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t893 mm8 a=r2 b=r1 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t894 mm8 a=r0 b=r7 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t895 mm8 a=r2 b=r4 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t896 mm8 a=r0 b=r2 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t897 mm8 a=r7 b=r4 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t898 mm8 a=r1 b=r3 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t899 mm8 a=r5 b=r1 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t900 mm8 a=r7 b=r7 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t901 mm8 a=r4 b=r6 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t902 mm8 a=r2 b=r1 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t903 mm8 a=r0 b=r3 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t904 mm8 a=r6 b=r7 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t905 mm8 a=r0 b=r4 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t906 mm8 a=r3 b=r7 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t907 mm8 a=r2 b=r4 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t908 mm8 a=r7 b=r1 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t909 mm8 a=r4 b=r5 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t910 mm8 a=r3 b=r7 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t911 mm8 a=r2 b=r6 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t912 mm8 a=r6 b=r1 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t913 mm8 a=r2 b=r3 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t914 mm8 a=r7 b=r4 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t915 mm8 a=r4 b=r3 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t916 mm8 a=r4 b=r2 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t917 mm8 a=r5 b=r5 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t918 mm8 a=r7 b=r2 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t919 mm8 a=r3 b=r4 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t920 mm8 a=r6 b=r5 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t921 mm8 a=r6 b=r5 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t922 mm8 a=r1 b=r0 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t923 mm8 a=r6 b=r3 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t924 mm8 a=r2 b=r0 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t925 mm8 a=r6 b=r6 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t926 mm8 a=r5 b=r0 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t927 mm8 a=r1 b=r1 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t928 mm8 a=r5 b=r7 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t929 mm8 a=r1 b=r0 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t930 mm8 a=r7 b=r7 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t931 mm8 a=r7 b=r0 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t932 mm8 a=r1 b=r3 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t933 mm8 a=r1 b=r3 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t934 mm8 a=r6 b=r0 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t935 mm8 a=r5 b=r0 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t936 mm8 a=r3 b=r3 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t937 mm8 a=r0 b=r6 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t938 mm8 a=r1 b=r5 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t939 mm8 a=r7 b=r0 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t940 mm8 a=r1 b=r2 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t941 mm8 a=r1 b=r6 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t942 mm8 a=r0 b=r6 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t943 mm8 a=r4 b=r0 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t944 mm8 a=r6 b=r5 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t945 mm8 a=r3 b=r7 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t946 mm8 a=r2 b=r6 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t947 mm8 a=r7 b=r2 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t948 mm8 a=r6 b=r4 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t949 mm8 a=r6 b=r6 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t950 mm8 a=r7 b=r6 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t951 mm8 a=r2 b=r7 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t952 mm8 a=r3 b=r2 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t953 mm8 a=r4 b=r4 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t954 mm8 a=r2 b=r4 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t955 mm8 a=r0 b=r7 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t956 mm8 a=r7 b=r1 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t957 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t958 mm8 a=r6 b=r5 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t959 mm8 a=r4 b=r7 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t960 mm8 a=r1 b=r1 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t961 mm8 a=r1 b=r5 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t962 mm8 a=r6 b=r4 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t963 mm8 a=r7 b=r1 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t964 mm8 a=r2 b=r5 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t965 mm8 a=r3 b=r7 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t966 mm8 a=r7 b=r3 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t967 mm8 a=r0 b=r0 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t968 mm8 a=r5 b=r7 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t969 mm8 a=r4 b=r4 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t970 mm8 a=r4 b=r3 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t971 mm8 a=r3 b=r4 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t972 mm8 a=r1 b=r7 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t973 mm8 a=r4 b=r1 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t974 mm8 a=r5 b=r6 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t975 mm8 a=r1 b=r2 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t976 mm8 a=r0 b=r6 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t977 mm8 a=r2 b=r0 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t978 mm8 a=r0 b=r2 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t979 mm8 a=r4 b=r0 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t980 mm8 a=r3 b=r1 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t981 mm8 a=r2 b=r0 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t982 mm8 a=r1 b=r6 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t983 mm8 a=r2 b=r1 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t984 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t985 mm8 a=r4 b=r3 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t986 mm8 a=r2 b=r7 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t987 mm8 a=r5 b=r1 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t988 mm8 a=r6 b=r0 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t989 mm8 a=r2 b=r3 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t990 mm8 a=r1 b=r5 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t991 mm8 a=r6 b=r0 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t992 mm8 a=r2 b=r1 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t993 mm8 a=r6 b=r0 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t994 mm8 a=r6 b=r0 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t995 mm8 a=r0 b=r3 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t996 mm8 a=r7 b=r0 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t997 mm8 a=r5 b=r5 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t998 mm8 a=r3 b=r4 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t999 mm8 a=r7 b=r2 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t1000 mm8 a=r5 b=r3 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t1001 mm8 a=r5 b=r6 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t1002 mm8 a=r6 b=r2 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t1003 mm8 a=r4 b=r2 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t1004 mm8 a=r7 b=r3 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t1005 mm8 a=r1 b=r7 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t1006 mm8 a=r4 b=r2 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t1007 mm8 a=r0 b=r2 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t1008 mm8 a=r3 b=r5 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t1009 mm8 a=r0 b=r7 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t1010 mm8 a=r6 b=r7 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t1011 mm8 a=r7 b=r5 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t1012 mm8 a=r3 b=r5 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t1013 mm8 a=r6 b=r6 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t1014 mm8 a=r7 b=r3 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t1015 mm8 a=r7 b=r6 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t1016 mm8 a=r4 b=r0 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t1017 mm8 a=r5 b=r2 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t1018 mm8 a=r7 b=r6 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t1019 mm8 a=r0 b=r7 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t1020 mm8 a=r5 b=r6 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t1021 mm8 a=r7 b=r6 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t1022 mm8 a=r3 b=r5 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t1023 mm8 a=r0 b=r3 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t1024 mm8 a=r2 b=r6 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t1025 mm8 a=r4 b=r7 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t1026 mm8 a=r0 b=r1 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t1027 mm8 a=r1 b=r5 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t1028 mm8 a=r7 b=r6 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t1029 mm8 a=r2 b=r1 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t1030 mm8 a=r1 b=r7 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t1031 mm8 a=r3 b=r7 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t1032 mm8 a=r2 b=r1 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t1033 mm8 a=r2 b=r7 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t1034 mm8 a=r4 b=r3 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t1035 mm8 a=r3 b=r0 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t1036 mm8 a=r3 b=r0 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t1037 mm8 a=r7 b=r1 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t1038 mm8 a=r2 b=r1 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t1039 mm8 a=r5 b=r1 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t1040 mm8 a=r0 b=r7 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t1041 mm8 a=r2 b=r2 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t1042 mm8 a=r6 b=r6 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t1043 mm8 a=r6 b=r7 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t1044 mm8 a=r4 b=r2 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t1045 mm8 a=r0 b=r0 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t1046 mm8 a=r7 b=r1 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t1047 mm8 a=r5 b=r7 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t1048 mm8 a=r2 b=r3 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t1049 mm8 a=r5 b=r1 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t1050 mm8 a=r0 b=r5 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t1051 mm8 a=r7 b=r7 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t1052 mm8 a=r2 b=r3 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t1053 mm8 a=r6 b=r0 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t1054 mm8 a=r4 b=r2 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t1055 mm8 a=r5 b=r3 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t1056 mm8 a=r5 b=r1 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t1057 mm8 a=r1 b=r5 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t1058 mm8 a=r0 b=r5 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t1059 mm8 a=r5 b=r6 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t1060 mm8 a=r7 b=r3 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t1061 mm8 a=r5 b=r6 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t1062 mm8 a=r3 b=r0 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t1063 mm8 a=r1 b=r1 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t1064 mm8 a=r5 b=r4 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t1065 mm8 a=r3 b=r1 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t1066 mm8 a=r7 b=r7 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t1067 mm8 a=r0 b=r5 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t1068 mm8 a=r4 b=r6 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t1069 mm8 a=r2 b=r0 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t1070 mm8 a=r2 b=r3 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t1071 mm8 a=r2 b=r6 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t1072 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t1073 mm8 a=r3 b=r6 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t1074 mm8 a=r0 b=r0 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t1075 mm8 a=r4 b=r4 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t1076 mm8 a=r1 b=r7 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t1077 mm8 a=r7 b=r7 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t1078 mm8 a=r3 b=r7 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t1079 mm8 a=r6 b=r6 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t1080 mm8 a=r4 b=r5 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t1081 mm8 a=r1 b=r5 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t1082 mm8 a=r5 b=r4 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t1083 mm8 a=r2 b=r2 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t1084 mm8 a=r2 b=r0 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t1085 mm8 a=r3 b=r3 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t1086 mm8 a=r3 b=r1 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t1087 mm8 a=r1 b=r3 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t1088 mm8 a=r3 b=r5 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t1089 mm8 a=r0 b=r0 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t1090 mm8 a=r4 b=r6 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t1091 mm8 a=r4 b=r3 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t1092 mm8 a=r2 b=r6 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t1093 mm8 a=r5 b=r7 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t1094 mm8 a=r7 b=r3 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t1095 mm8 a=r0 b=r2 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t1096 mm8 a=r3 b=r3 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t1097 mm8 a=r6 b=r2 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t1098 mm8 a=r3 b=r6 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t1099 mm8 a=r2 b=r5 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t1100 mm8 a=r1 b=r4 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t1101 mm8 a=r0 b=r6 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t1102 mm8 a=r0 b=r5 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t1103 mm8 a=r7 b=r0 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t1104 mm8 a=r4 b=r2 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t1105 mm8 a=r3 b=r4 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t1106 mm8 a=r2 b=r1 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t1107 mm8 a=r3 b=r0 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t1108 mm8 a=r3 b=r5 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t1109 mm8 a=r6 b=r2 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t1110 mm8 a=r2 b=r4 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t1111 mm8 a=r0 b=r7 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t1112 mm8 a=r1 b=r0 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t1113 mm8 a=r4 b=r6 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t1114 mm8 a=r0 b=r5 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t1115 mm8 a=r4 b=r5 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t1116 mm8 a=r4 b=r6 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t1117 mm8 a=r0 b=r6 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t1118 mm8 a=r7 b=r6 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t1119 mm8 a=r2 b=r7 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t1120 mm8 a=r7 b=r5 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t1121 mm8 a=r4 b=r0 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t1122 mm8 a=r0 b=r5 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t1123 mm8 a=r0 b=r6 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t1124 mm8 a=r0 b=r1 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t1125 mm8 a=r6 b=r6 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t1126 mm8 a=r0 b=r0 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t1127 mm8 a=r1 b=r1 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t1128 mm8 a=r4 b=r1 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t1129 mm8 a=r0 b=r0 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t1130 mm8 a=r0 b=r4 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t1131 mm8 a=r3 b=r0 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t1132 mm8 a=r0 b=r6 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t1133 mm8 a=r6 b=r0 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t1134 mm8 a=r7 b=r3 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t1135 mm8 a=r6 b=r7 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t1136 mm8 a=r7 b=r4 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t1137 mm8 a=r4 b=r0 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t1138 mm8 a=r6 b=r4 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t1139 mm8 a=r6 b=r2 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t1140 mm8 a=r1 b=r2 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t1141 mm8 a=r0 b=r5 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t1142 mm8 a=r4 b=r5 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t1143 mm8 a=r5 b=r6 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t1144 mm8 a=r6 b=r4 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t1145 mm8 a=r5 b=r1 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t1146 mm8 a=r6 b=r5 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t1147 mm8 a=r7 b=r1 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t1148 mm8 a=r1 b=r6 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t1149 mm8 a=r5 b=r5 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t1150 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t1151 mm8 a=r7 b=r5 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t1152 mm8 a=r0 b=r6 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t1153 mm8 a=r2 b=r5 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t1154 mm8 a=r5 b=r3 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t1155 mm8 a=r0 b=r6 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t1156 mm8 a=r2 b=r6 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t1157 mm8 a=r6 b=r7 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t1158 mm8 a=r2 b=r3 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t1159 mm8 a=r1 b=r5 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t1160 mm8 a=r5 b=r0 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t1161 mm8 a=r7 b=r0 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t1162 mm8 a=r5 b=r3 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t1163 mm8 a=r4 b=r3 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t1164 mm8 a=r3 b=r4 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t1165 mm8 a=r1 b=r2 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t1166 mm8 a=r4 b=r7 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t1167 mm8 a=r2 b=r7 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t1168 mm8 a=r2 b=r6 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t1169 mm8 a=r0 b=r4 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t1170 mm8 a=r2 b=r1 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t1171 mm8 a=r1 b=r7 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t1172 mm8 a=r0 b=r1 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t1173 mm8 a=r6 b=r4 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t1174 mm8 a=r0 b=r1 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t1175 mm8 a=r0 b=r0 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t1176 mm8 a=r4 b=r5 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t1177 mm8 a=r7 b=r2 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t1178 mm8 a=r1 b=r5 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t1179 mm8 a=r0 b=r7 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t1180 mm8 a=r7 b=r3 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t1181 mm8 a=r7 b=r3 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t1182 mm8 a=r1 b=r2 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t1183 mm8 a=r2 b=r4 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t1184 mm8 a=r7 b=r0 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t1185 mm8 a=r5 b=r2 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t1186 mm8 a=r1 b=r0 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t1187 mm8 a=r5 b=r4 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t1188 mm8 a=r3 b=r0 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t1189 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t1190 mm8 a=r5 b=r0 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t1191 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t1192 mm8 a=r0 b=r2 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t1193 mm8 a=r3 b=r7 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t1194 mm8 a=r7 b=r2 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t1195 mm8 a=r1 b=r4 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t1196 mm8 a=r4 b=r6 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t1197 mm8 a=r5 b=r6 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t1198 mm8 a=r0 b=r6 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t1199 mm8 a=r5 b=r1 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t1200 mm8 a=r1 b=r3 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t1201 mm8 a=r6 b=r6 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t1202 mm8 a=r2 b=r4 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t1203 mm8 a=r0 b=r5 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t1204 mm8 a=r0 b=r2 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t1205 mm8 a=r4 b=r6 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t1206 mm8 a=r4 b=r6 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t1207 mm8 a=r3 b=r7 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t1208 mm8 a=r6 b=r1 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t1209 mm8 a=r5 b=r5 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t1210 mm8 a=r0 b=r0 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t1211 mm8 a=r7 b=r5 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t1212 mm8 a=r2 b=r6 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t1213 mm8 a=r6 b=r2 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t1214 mm8 a=r4 b=r2 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t1215 mm8 a=r7 b=r2 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t1216 mm8 a=r3 b=r3 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t1217 mm8 a=r5 b=r5 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t1218 mm8 a=r0 b=r7 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t1219 mm8 a=r2 b=r4 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t1220 mm8 a=r1 b=r7 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t1221 mm8 a=r5 b=r5 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t1222 mm8 a=r1 b=r3 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t1223 mm8 a=r0 b=r2 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t1224 mm8 a=r1 b=r1 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t1225 mm8 a=r4 b=r1 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t1226 mm8 a=r5 b=r6 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t1227 mm8 a=r1 b=r7 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t1228 mm8 a=r4 b=r2 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t1229 mm8 a=r5 b=r3 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t1230 mm8 a=r6 b=r2 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t1231 mm8 a=r4 b=r3 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t1232 mm8 a=r6 b=r7 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t1233 mm8 a=r0 b=r5 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t1234 mm8 a=r5 b=r4 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t1235 mm8 a=r5 b=r7 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t1236 mm8 a=r0 b=r7 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t1237 mm8 a=r7 b=r2 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t1238 mm8 a=r7 b=r0 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t1239 mm8 a=r3 b=r2 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t1240 mm8 a=r0 b=r5 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t1241 mm8 a=r5 b=r0 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t1242 mm8 a=r5 b=r7 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t1243 mm8 a=r6 b=r3 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t1244 mm8 a=r1 b=r5 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t1245 mm8 a=r0 b=r6 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t1246 mm8 a=r5 b=r5 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t1247 mm8 a=r1 b=r6 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t1248 mm8 a=r4 b=r0 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t1249 mm8 a=r7 b=r6 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t1250 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t1251 mm8 a=r3 b=r7 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t1252 mm8 a=r5 b=r5 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t1253 mm8 a=r5 b=r5 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t1254 mm8 a=r0 b=r6 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t1255 mm8 a=r1 b=r4 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t1256 mm8 a=r1 b=r3 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t1257 mm8 a=r0 b=r7 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t1258 mm8 a=r5 b=r4 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t1259 mm8 a=r0 b=r3 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t1260 mm8 a=r1 b=r5 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t1261 mm8 a=r6 b=r7 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t1262 mm8 a=r2 b=r5 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t1263 mm8 a=r3 b=r7 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t1264 mm8 a=r1 b=r6 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t1265 mm8 a=r2 b=r3 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t1266 mm8 a=r4 b=r6 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t1267 mm8 a=r6 b=r6 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t1268 mm8 a=r4 b=r5 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t1269 mm8 a=r5 b=r7 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t1270 mm8 a=r7 b=r3 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t1271 mm8 a=r7 b=r0 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t1272 mm8 a=r1 b=r3 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t1273 mm8 a=r3 b=r2 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t1274 mm8 a=r2 b=r0 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t1275 mm8 a=r7 b=r3 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t1276 mm8 a=r4 b=r3 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t1277 mm8 a=r5 b=r4 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t1278 mm8 a=r6 b=r3 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t1279 mm8 a=r0 b=r1 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t1280 mm8 a=r4 b=r1 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t1281 mm8 a=r6 b=r3 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t1282 mm8 a=r1 b=r7 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t1283 mm8 a=r6 b=r2 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t1284 mm8 a=r1 b=r6 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t1285 mm8 a=r3 b=r5 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t1286 mm8 a=r3 b=r1 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t1287 mm8 a=r4 b=r1 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t1288 mm8 a=r3 b=r6 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t1289 mm8 a=r6 b=r2 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t1290 mm8 a=r6 b=r5 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t1291 mm8 a=r5 b=r6 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t1292 mm8 a=r6 b=r6 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t1293 mm8 a=r6 b=r2 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t1294 mm8 a=r0 b=r5 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t1295 mm8 a=r5 b=r0 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t1296 mm8 a=r1 b=r0 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t1297 mm8 a=r0 b=r0 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t1298 mm8 a=r2 b=r6 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t1299 mm8 a=r7 b=r0 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t1300 mm8 a=r4 b=r1 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t1301 mm8 a=r4 b=r5 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t1302 mm8 a=r0 b=r4 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t1303 mm8 a=r4 b=r2 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t1304 mm8 a=r4 b=r7 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t1305 mm8 a=r3 b=r2 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t1306 mm8 a=r3 b=r0 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t1307 mm8 a=r0 b=r2 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t1308 mm8 a=r2 b=r5 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t1309 mm8 a=r5 b=r7 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t1310 mm8 a=r4 b=r5 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t1311 mm8 a=r4 b=r7 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t1312 mm8 a=r3 b=r7 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t1313 mm8 a=r5 b=r1 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t1314 mm8 a=r5 b=r2 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t1315 mm8 a=r1 b=r0 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t1316 mm8 a=r7 b=r3 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t1317 mm8 a=r2 b=r5 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t1318 mm8 a=r2 b=r3 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t1319 mm8 a=r7 b=r5 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t1320 mm8 a=r0 b=r3 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t1321 mm8 a=r1 b=r0 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t1322 mm8 a=r4 b=r0 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t1323 mm8 a=r5 b=r3 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t1324 mm8 a=r3 b=r0 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t1325 mm8 a=r6 b=r4 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t1326 mm8 a=r4 b=r6 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t1327 mm8 a=r1 b=r3 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t1328 mm8 a=r7 b=r3 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t1329 mm8 a=r3 b=r7 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t1330 mm8 a=r3 b=r6 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t1331 mm8 a=r0 b=r7 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t1332 mm8 a=r0 b=r7 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t1333 mm8 a=r3 b=r3 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t1334 mm8 a=r7 b=r5 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t1335 mm8 a=r5 b=r7 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t1336 mm8 a=r5 b=r5 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t1337 mm8 a=r6 b=r0 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t1338 mm8 a=r6 b=r4 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t1339 mm8 a=r4 b=r6 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t1340 mm8 a=r6 b=r1 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t1341 mm8 a=r0 b=r6 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t1342 mm8 a=r4 b=r6 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t1343 mm8 a=r5 b=r5 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t1344 mm8 a=r2 b=r6 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t1345 mm8 a=r7 b=r3 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t1346 mm8 a=r5 b=r0 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t1347 mm8 a=r2 b=r3 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t1348 mm8 a=r4 b=r3 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t1349 mm8 a=r0 b=r1 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t1350 mm8 a=r1 b=r6 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t1351 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t1352 mm8 a=r5 b=r4 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t1353 mm8 a=r2 b=r7 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t1354 mm8 a=r7 b=r4 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t1355 mm8 a=r7 b=r4 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t1356 mm8 a=r4 b=r6 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t1357 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t1358 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t1359 mm8 a=r7 b=r7 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t1360 mm8 a=r0 b=r4 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t1361 mm8 a=r0 b=r5 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t1362 mm8 a=r0 b=r2 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t1363 mm8 a=r4 b=r4 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t1364 mm8 a=r7 b=r3 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t1365 mm8 a=r2 b=r4 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t1366 mm8 a=r4 b=r3 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t1367 mm8 a=r3 b=r3 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t1368 mm8 a=r4 b=r0 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t1369 mm8 a=r5 b=r3 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t1370 mm8 a=r0 b=r1 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t1371 mm8 a=r0 b=r5 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t1372 mm8 a=r5 b=r6 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t1373 mm8 a=r7 b=r1 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t1374 mm8 a=r4 b=r6 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t1375 mm8 a=r5 b=r2 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t1376 mm8 a=r5 b=r5 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t1377 mm8 a=r7 b=r3 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t1378 mm8 a=r4 b=r1 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t1379 mm8 a=r4 b=r0 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t1380 mm8 a=r0 b=r7 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t1381 mm8 a=r2 b=r0 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t1382 mm8 a=r7 b=r5 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t1383 mm8 a=r1 b=r5 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t1384 mm8 a=r5 b=r2 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t1385 mm8 a=r0 b=r7 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t1386 mm8 a=r7 b=r0 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t1387 mm8 a=r4 b=r3 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t1388 mm8 a=r4 b=r6 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t1389 mm8 a=r4 b=r2 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t1390 mm8 a=r5 b=r1 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t1391 mm8 a=r6 b=r1 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t1392 mm8 a=r5 b=r7 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t1393 mm8 a=r3 b=r4 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t1394 mm8 a=r4 b=r7 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t1395 mm8 a=r7 b=r1 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t1396 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t1397 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t1398 mm8 a=r3 b=r0 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t1399 mm8 a=r3 b=r4 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t1400 mm8 a=r6 b=r4 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t1401 mm8 a=r0 b=r5 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t1402 mm8 a=r7 b=r5 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t1403 mm8 a=r0 b=r0 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t1404 mm8 a=r3 b=r6 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t1405 mm8 a=r0 b=r0 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t1406 mm8 a=r5 b=r3 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t1407 mm8 a=r7 b=r3 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t1408 mm8 a=r4 b=r7 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t1409 mm8 a=r4 b=r2 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t1410 mm8 a=r6 b=r0 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t1411 mm8 a=r1 b=r7 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t1412 mm8 a=r4 b=r3 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t1413 mm8 a=r4 b=r6 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t1414 mm8 a=r0 b=r7 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t1415 mm8 a=r3 b=r6 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t1416 mm8 a=r3 b=r2 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t1417 mm8 a=r2 b=r7 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t1418 mm8 a=r0 b=r6 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t1419 mm8 a=r0 b=r3 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t1420 mm8 a=r1 b=r3 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t1421 mm8 a=r4 b=r0 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t1422 mm8 a=r6 b=r4 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t1423 mm8 a=r4 b=r6 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t1424 mm8 a=r6 b=r3 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t1425 mm8 a=r7 b=r3 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t1426 mm8 a=r0 b=r3 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t1427 mm8 a=r5 b=r6 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t1428 mm8 a=r0 b=r3 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t1429 mm8 a=r7 b=r1 c=r3 c2=r7 - } - uint32_t lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint32_t hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((uint64_t)hi << 32) | (uint64_t)lo; -} - -cudaError_t igneum_launch_hash_bound(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask, - IgneumInitWords iw, uint32_t nonces, uint32_t blockWarps) { - if (blockWarps == 0u || blockWarps > 32u) return cudaErrorInvalidValue; - uint32_t block = 32u * blockWarps; - if (nonces == 0u || (nonces % block) != 0u) return cudaErrorInvalidValue; - igneum_hash_bound<<>>(ds, out, baseNonce, mask, iw); - return cudaGetLastError(); -} - -cudaError_t igneum_hash_bound_info(int* numRegs, int* blocksPerSM, uint32_t blockWarps) { - cudaFuncAttributes attr; - cudaError_t e = cudaFuncGetAttributes(&attr, igneum_hash_bound); - if (e != cudaSuccess) return e; - *numRegs = attr.numRegs; - return cudaOccupancyMaxActiveBlocksPerMultiprocessor(blocksPerSM, igneum_hash_bound, (int)(32u * blockWarps), 0); -} diff --git a/proto-cuda/packs-ca4/mx8_mm1430/memhard.h b/proto-cuda/packs-ca4/mx8_mm1430/memhard.h deleted file mode 100644 index f7f34c732..000000000 --- a/proto-cuda/packs-ca4/mx8_mm1430/memhard.h +++ /dev/null @@ -1,109 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// Memory-hard dataset core, the same text that the Mac's Metal kernels and CPU verifier were checked against. -// Included by kernel.cu (device), host.cu (host reference) and proto-opencl/host.c (C99 host reference). -// See proto-metal/MEMHARD.md for the construction. kernel.cl carries the same text in OpenCL C. -#pragma once -#ifdef __cplusplus -#include -#else -#include -#endif -#if defined(__CUDACC__) -#define IGNEUM_HD __host__ __device__ __forceinline__ -#elif defined(_MSC_VER) && !defined(__cplusplus) -#define IGNEUM_HD static __inline -#else -#define IGNEUM_HD static inline -#endif -// Memory-hard dataset core (MEMHARD.md). Cache: 2^26 words in 2^16 segments of 64 chained ChaCha12 lines. -// Item: 8 rounds of 8 x seed-parameterised mixer + one 64-byte cache read, then 8 x final mixer (class v3, mixer multiplier 8, -// docs/plans/mixer-x4.md: the round key of application j of round r is 0x9E3779B9 * (r * m + j + 1)). All parameters are literals. -#define MH_CACHE_LINE_MASK 0x003fffffu -#define MH_SEGMENT_LINES 64u -#define MH_QR(a, b, c, d, r1, r2, r3, r4) { a += b; d ^= a; d = mh_rotl(d, r1); c += d; b ^= c; b = mh_rotl(b, r2); a += b; d ^= a; d = mh_rotl(d, r3); c += d; b ^= c; b = mh_rotl(b, r4); } -IGNEUM_HD uint32_t mh_rotl(uint32_t x, uint32_t n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 at every call site - -// y = ChaCha12 core(x) + x -IGNEUM_HD void mh_chacha_block(const uint32_t* x, uint32_t* y) { - for (uint32_t i = 0u; i < 16u; ++i) y[i] = x[i]; - for (uint32_t r = 0u; r < 6u; ++r) { - MH_QR(y[0], y[4], y[8], y[12], 16u, 12u, 8u, 7u) MH_QR(y[1], y[5], y[9], y[13], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[6], y[10], y[14], 16u, 12u, 8u, 7u) MH_QR(y[3], y[7], y[11], y[15], 16u, 12u, 8u, 7u) - MH_QR(y[0], y[5], y[10], y[15], 16u, 12u, 8u, 7u) MH_QR(y[1], y[6], y[11], y[12], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[7], y[8], y[13], 16u, 12u, 8u, 7u) MH_QR(y[3], y[4], y[9], y[14], 16u, 12u, 8u, 7u) - } - for (uint32_t i = 0u; i < 16u; ++i) y[i] += x[i]; -} - -// One cache segment: 64 chained lines written at cache[seg * 1024]. in_j = prev ^ (sigma || K || seg || j || tag), prev_0 = 0. -IGNEUM_HD void mh_cache_segment(uint32_t* cache, uint32_t seg) { - uint32_t prev[16]; uint32_t x[16]; uint32_t y[16]; - for (uint32_t i = 0u; i < 16u; ++i) prev[i] = 0u; - for (uint32_t j = 0u; j < MH_SEGMENT_LINES; ++j) { - x[0] = 0x61707865u ^ prev[0]; x[1] = 0x3320646eu ^ prev[1]; x[2] = 0x79622d32u ^ prev[2]; x[3] = 0x6b206574u ^ prev[3]; - x[4] = 0x3067619fu ^ prev[4]; - x[5] = 0x3c269176u ^ prev[5]; - x[6] = 0x84a03b03u ^ prev[6]; - x[7] = 0xf8c63294u ^ prev[7]; - x[8] = 0xff977c5bu ^ prev[8]; - x[9] = 0xe60def3eu ^ prev[9]; - x[10] = 0x63630141u ^ prev[10]; - x[11] = 0xb8fbcb58u ^ prev[11]; - x[12] = seg ^ prev[12]; x[13] = j ^ prev[13]; x[14] = 0x49676e65u ^ prev[14]; x[15] = 0x756d4d48u ^ prev[15]; - mh_chacha_block(x, y); - uint32_t* line = cache + ((seg * MH_SEGMENT_LINES + j) * 16u); - for (uint32_t i = 0u; i < 16u; ++i) { line[i] = y[i]; prev[i] = y[i]; } - } -} - -// M_r: per word (s ^ (RC + rk)) * MUL, then a column round and a diagonal round with the seed-drawn rotations. -IGNEUM_HD void mh_mixer(uint32_t* s, uint32_t rk) { - s[0] = (s[0] ^ (0xbab68293u + rk)) * 0x42146205u; - s[1] = (s[1] ^ (0xcc162340u + rk)) * 0x52cbe0fbu; - s[2] = (s[2] ^ (0x6ce151ccu + rk)) * 0x7ecf4a03u; - s[3] = (s[3] ^ (0xe62b8997u + rk)) * 0x6728907fu; - s[4] = (s[4] ^ (0xc9c80297u + rk)) * 0xd81d9751u; - s[5] = (s[5] ^ (0xf74a1654u + rk)) * 0x132952c3u; - s[6] = (s[6] ^ (0x3d704af5u + rk)) * 0xf60de277u; - s[7] = (s[7] ^ (0x3cf522b7u + rk)) * 0x05358035u; - s[8] = (s[8] ^ (0x2b9cac04u + rk)) * 0xbaf6499du; - s[9] = (s[9] ^ (0xa880ac10u + rk)) * 0xe4db9667u; - s[10] = (s[10] ^ (0x13e5dd1du + rk)) * 0x3e98f45du; - s[11] = (s[11] ^ (0x6fc3e233u + rk)) * 0xd0004eddu; - s[12] = (s[12] ^ (0x2d83eeacu + rk)) * 0x2691630du; - s[13] = (s[13] ^ (0x9006e8bfu + rk)) * 0x9beb3bcfu; - s[14] = (s[14] ^ (0x2c4b5362u + rk)) * 0xab310379u; - s[15] = (s[15] ^ (0x31b49ee2u + rk)) * 0x99cfb423u; - MH_QR(s[0], s[4], s[8], s[12], 20u, 20u, 19u, 4u) MH_QR(s[1], s[5], s[9], s[13], 20u, 20u, 19u, 4u) - MH_QR(s[2], s[6], s[10], s[14], 20u, 20u, 19u, 4u) MH_QR(s[3], s[7], s[11], s[15], 20u, 20u, 19u, 4u) - MH_QR(s[0], s[5], s[10], s[15], 26u, 3u, 3u, 27u) MH_QR(s[1], s[6], s[11], s[12], 26u, 3u, 3u, 27u) - MH_QR(s[2], s[7], s[8], s[13], 26u, 3u, 3u, 27u) MH_QR(s[3], s[4], s[9], s[14], 26u, 3u, 3u, 27u) -} - -// Item t: 16 words. s = (K, t * MUL[i] + RC[i]); 8 rounds of 8 x mixer + cache line s[0] & mask; 8 x final mixer. -IGNEUM_HD void mh_item(const uint32_t* cache, uint32_t t, uint32_t* s) { - s[0] = 0x3067619fu; - s[1] = 0x3c269176u; - s[2] = 0x84a03b03u; - s[3] = 0xf8c63294u; - s[4] = 0xff977c5bu; - s[5] = 0xe60def3eu; - s[6] = 0x63630141u; - s[7] = 0xb8fbcb58u; - s[8] = t * 0x42146205u + 0xbab68293u; - s[9] = t * 0x52cbe0fbu + 0xcc162340u; - s[10] = t * 0x7ecf4a03u + 0x6ce151ccu; - s[11] = t * 0x6728907fu + 0xe62b8997u; - s[12] = t * 0xd81d9751u + 0xc9c80297u; - s[13] = t * 0x132952c3u + 0xf74a1654u; - s[14] = t * 0xf60de277u + 0x3d704af5u; - s[15] = t * 0x05358035u + 0x3cf522b7u; - for (uint32_t r = 0u; r < 8u; ++r) { - for (uint32_t j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (r * 8u + j + 1u)); - const uint32_t* line = cache + ((s[0] & MH_CACHE_LINE_MASK) * 16u); - for (uint32_t i = 0u; i < 16u; ++i) s[i] ^= line[i]; - } - for (uint32_t j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (64u + j + 1u)); -} -// dataset[w] without the dataset: derive item w >> 4 and take word w & 15. -IGNEUM_HD uint32_t mh_word(const uint32_t* cache, uint32_t w) { uint32_t s[16]; mh_item(cache, w >> 4u, s); return s[w & 15u]; } diff --git a/proto-cuda/packs-ca4/mx8_mm1430/memhard.metal b/proto-cuda/packs-ca4/mx8_mm1430/memhard.metal deleted file mode 100644 index 01b263d6d..000000000 --- a/proto-cuda/packs-ca4/mx8_mm1430/memhard.metal +++ /dev/null @@ -1,107 +0,0 @@ -#include -using namespace metal; -// Memory-hard dataset core (MEMHARD.md). Cache: 2^26 words in 2^16 segments of 64 chained ChaCha12 lines. -// Item: 8 rounds of 8 x seed-parameterised mixer + one 64-byte cache read, then 8 x final mixer (class v3, mixer multiplier 8, -// docs/plans/mixer-x4.md: the round key of application j of round r is 0x9E3779B9 * (r * m + j + 1)). All parameters are literals. -#define MH_CACHE_LINE_MASK 0x003fffffu -#define MH_SEGMENT_LINES 64u -#define MH_QR(a, b, c, d, r1, r2, r3, r4) { a += b; d ^= a; d = mh_rotl(d, r1); c += d; b ^= c; b = mh_rotl(b, r2); a += b; d ^= a; d = mh_rotl(d, r3); c += d; b ^= c; b = mh_rotl(b, r4); } -inline uint mh_rotl(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 at every call site - -// y = ChaCha12 core(x) + x -inline void mh_chacha_block(const thread uint* x, thread uint* y) { - for (uint i = 0u; i < 16u; ++i) y[i] = x[i]; - for (uint r = 0u; r < 6u; ++r) { - MH_QR(y[0], y[4], y[8], y[12], 16u, 12u, 8u, 7u) MH_QR(y[1], y[5], y[9], y[13], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[6], y[10], y[14], 16u, 12u, 8u, 7u) MH_QR(y[3], y[7], y[11], y[15], 16u, 12u, 8u, 7u) - MH_QR(y[0], y[5], y[10], y[15], 16u, 12u, 8u, 7u) MH_QR(y[1], y[6], y[11], y[12], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[7], y[8], y[13], 16u, 12u, 8u, 7u) MH_QR(y[3], y[4], y[9], y[14], 16u, 12u, 8u, 7u) - } - for (uint i = 0u; i < 16u; ++i) y[i] += x[i]; -} - -// One cache segment: 64 chained lines written at cache[seg * 1024]. in_j = prev ^ (sigma || K || seg || j || tag), prev_0 = 0. -inline void mh_cache_segment(device uint* cache, uint seg) { - uint prev[16]; uint x[16]; uint y[16]; - for (uint i = 0u; i < 16u; ++i) prev[i] = 0u; - for (uint j = 0u; j < MH_SEGMENT_LINES; ++j) { - x[0] = 0x61707865u ^ prev[0]; x[1] = 0x3320646eu ^ prev[1]; x[2] = 0x79622d32u ^ prev[2]; x[3] = 0x6b206574u ^ prev[3]; - x[4] = 0x3067619fu ^ prev[4]; - x[5] = 0x3c269176u ^ prev[5]; - x[6] = 0x84a03b03u ^ prev[6]; - x[7] = 0xf8c63294u ^ prev[7]; - x[8] = 0xff977c5bu ^ prev[8]; - x[9] = 0xe60def3eu ^ prev[9]; - x[10] = 0x63630141u ^ prev[10]; - x[11] = 0xb8fbcb58u ^ prev[11]; - x[12] = seg ^ prev[12]; x[13] = j ^ prev[13]; x[14] = 0x49676e65u ^ prev[14]; x[15] = 0x756d4d48u ^ prev[15]; - mh_chacha_block(x, y); - device uint* line = cache + ((seg * MH_SEGMENT_LINES + j) * 16u); - for (uint i = 0u; i < 16u; ++i) { line[i] = y[i]; prev[i] = y[i]; } - } -} - -// M_r: per word (s ^ (RC + rk)) * MUL, then a column round and a diagonal round with the seed-drawn rotations. -inline void mh_mixer(thread uint* s, uint rk) { - s[0] = (s[0] ^ (0xbab68293u + rk)) * 0x42146205u; - s[1] = (s[1] ^ (0xcc162340u + rk)) * 0x52cbe0fbu; - s[2] = (s[2] ^ (0x6ce151ccu + rk)) * 0x7ecf4a03u; - s[3] = (s[3] ^ (0xe62b8997u + rk)) * 0x6728907fu; - s[4] = (s[4] ^ (0xc9c80297u + rk)) * 0xd81d9751u; - s[5] = (s[5] ^ (0xf74a1654u + rk)) * 0x132952c3u; - s[6] = (s[6] ^ (0x3d704af5u + rk)) * 0xf60de277u; - s[7] = (s[7] ^ (0x3cf522b7u + rk)) * 0x05358035u; - s[8] = (s[8] ^ (0x2b9cac04u + rk)) * 0xbaf6499du; - s[9] = (s[9] ^ (0xa880ac10u + rk)) * 0xe4db9667u; - s[10] = (s[10] ^ (0x13e5dd1du + rk)) * 0x3e98f45du; - s[11] = (s[11] ^ (0x6fc3e233u + rk)) * 0xd0004eddu; - s[12] = (s[12] ^ (0x2d83eeacu + rk)) * 0x2691630du; - s[13] = (s[13] ^ (0x9006e8bfu + rk)) * 0x9beb3bcfu; - s[14] = (s[14] ^ (0x2c4b5362u + rk)) * 0xab310379u; - s[15] = (s[15] ^ (0x31b49ee2u + rk)) * 0x99cfb423u; - MH_QR(s[0], s[4], s[8], s[12], 20u, 20u, 19u, 4u) MH_QR(s[1], s[5], s[9], s[13], 20u, 20u, 19u, 4u) - MH_QR(s[2], s[6], s[10], s[14], 20u, 20u, 19u, 4u) MH_QR(s[3], s[7], s[11], s[15], 20u, 20u, 19u, 4u) - MH_QR(s[0], s[5], s[10], s[15], 26u, 3u, 3u, 27u) MH_QR(s[1], s[6], s[11], s[12], 26u, 3u, 3u, 27u) - MH_QR(s[2], s[7], s[8], s[13], 26u, 3u, 3u, 27u) MH_QR(s[3], s[4], s[9], s[14], 26u, 3u, 3u, 27u) -} - -// Item t: 16 words. s = (K, t * MUL[i] + RC[i]); 8 rounds of 8 x mixer + cache line s[0] & mask; 8 x final mixer. -inline void mh_item(device const uint* cache, uint t, thread uint* s) { - s[0] = 0x3067619fu; - s[1] = 0x3c269176u; - s[2] = 0x84a03b03u; - s[3] = 0xf8c63294u; - s[4] = 0xff977c5bu; - s[5] = 0xe60def3eu; - s[6] = 0x63630141u; - s[7] = 0xb8fbcb58u; - s[8] = t * 0x42146205u + 0xbab68293u; - s[9] = t * 0x52cbe0fbu + 0xcc162340u; - s[10] = t * 0x7ecf4a03u + 0x6ce151ccu; - s[11] = t * 0x6728907fu + 0xe62b8997u; - s[12] = t * 0xd81d9751u + 0xc9c80297u; - s[13] = t * 0x132952c3u + 0xf74a1654u; - s[14] = t * 0xf60de277u + 0x3d704af5u; - s[15] = t * 0x05358035u + 0x3cf522b7u; - for (uint r = 0u; r < 8u; ++r) { - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (r * 8u + j + 1u)); - device const uint* line = cache + ((s[0] & MH_CACHE_LINE_MASK) * 16u); - for (uint i = 0u; i < 16u; ++i) s[i] ^= line[i]; - } - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (64u + j + 1u)); -} -// dataset[w] without the dataset: derive item w >> 4 and take word w & 15. -inline uint mh_word(device const uint* cache, uint w) { uint s[16]; mh_item(cache, w >> 4u, s); return s[w & 15u]; } - -// One thread per segment (2^16 threads). -kernel void igneum_cache_fill(device uint* cache [[buffer(0)]], uint gid [[thread_position_in_grid]]) { - mh_cache_segment(cache, gid); -} -// One thread per 64-byte item (dataset words / 16 threads). -kernel void igneum_build(device const uint* cache [[buffer(0)]], device uint* dataset [[buffer(1)]], - uint gid [[thread_position_in_grid]]) { - uint s[16]; - mh_item(cache, gid, s); - device uint* d = dataset + gid * 16u; - for (uint i = 0u; i < 16u; ++i) d[i] = s[i]; -} diff --git a/proto-cuda/packs-ca4/mx8_mm1430/program.h b/proto-cuda/packs-ca4/mx8_mm1430/program.h deleted file mode 100644 index eaa7df57e..000000000 --- a/proto-cuda/packs-ca4/mx8_mm1430/program.h +++ /dev/null @@ -1,73 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// Program metadata for host.cu plus the launch wrappers defined in kernel.cu. -// Also included by proto-opencl/host.c (C99), which defines IGNEUM_NO_CUDA first and reads only the macros. -#pragma once -#ifdef __cplusplus -#include -#else -#include -#endif -#ifndef IGNEUM_NO_CUDA -#include -#endif - -#define IGNEUM_SEED_STRING "igneum-genesis" -#define IGNEUM_SEED_BYTES_HEX "69676e65756d2d67656e65736973" -#define IGNEUM_GENERATOR 2 -#define IGNEUM_PROGRAM_ATTEMPT 0 -#define IGNEUM_PROGRAM_ID 0xd746ef93184dc1f8ull -#define IGNEUM_DAY_STRING "2026-10-03" -#define IGNEUM_DAY_BYTES_HEX "6461792f323032362d31302d3033" -#define IGNEUM_DAY0 0x3067619fu -#define IGNEUM_DAY1 0x3c269176u -#define IGNEUM_DATASET_LOG2 28 -#define IGNEUM_MASK 0x0fffffffu -#define IGNEUM_LANES 32 -#define IGNEUM_ITERATIONS 8 -#define IGNEUM_INSTR_COUNT 64 -#define IGNEUM_LOADS_PER_HASH 128 -#define IGNEUM_WIDE_LOADS_PER_HASH 0 -#define IGNEUM_OP_MIX "load=16 add=8 shfl=8 xor=6 mad=5 mul=5 mulhi=5 sub=4 rotl=3 rotr=3 or=1" -// Class v3 construction (Counter ASIC 2.0, 5 October 2026, docs/plans/mixer-x4.md): version 2 loads; the dataset item -// derivation applies the mixer IGNEUM_MIXER_MULT times per round (memhard.h), and the cache follows the growth rule. -#define IGNEUM_LOAD_CLASS "mx8+mm1430" -#define IGNEUM_CLASS_MIXER_MULT 8 -#define IGNEUM_CACHE_GROWTH 1 // 1: cache words = 2^(26 + doublings(day)), doublings = floor(log2(1 + day / 1460)) -#define IGNEUM_LOAD_SLOTS 16 -#define IGNEUM_LOAD_MIX { 100, 0, 0 } -#define IGNEUM_LOAD_WIDTH_COUNTS { 16, 0, 0 } // loads of 4, 16, 64 bytes per program -#define IGNEUM_BYTES_PER_HASH 512 -#define IGNEUM_FOLD_ROT 11 -#define IGNEUM_FOLD_MUL 0x9e3779b1u -// Latency-shadow block (Counter ASIC 3.0 item 8, docs/analysis/latency-shadow-2026-10-06.md): NOT the lottery hash. A block of -// IGNEUM_SHADOW_INSTRS ALU instructions (the ten non-load families) runs IGNEUM_SHADOW_REPS times at the end of every -// iteration; the 16 loads, the acceptance rule and the base program are the class's without the shadow. -#define IGNEUM_SHADOW_INSTRS 0 -#define IGNEUM_SHADOW_REPS 0 -#define IGNEUM_SHADOW_INSTRS_PER_HASH 0 -#define IGNEUM_SHADOW_OP_MIX "" -// Counter ASIC 4.0 research (experimental): IGNEUM_MM8_TILES int8 mma.m8n8k16 u8 tiles per iteration after the shadow block. -#define IGNEUM_MM8_TILES 1430 -#define IGNEUM_MM8_TILES_PER_HASH 11440 -// 0 = closed-form dataset (ds_elem), 1 = memory-hard cache construction (MEMHARD.md, memhard.h) -#define IGNEUM_DATASET_MODE 1 - -#define IGNEUM_SEEDW_INIT { 0x67a9a7beu, 0x1a155b25u, 0xfddfb732u, 0x4b5af2e8u, 0xc55caf33u, 0xa27c13b7u, 0x06628a48u, 0x03852469u } -#define IGNEUM_KEY_INIT { 0x3067619fu, 0x3c269176u, 0x84a03b03u, 0xf8c63294u, 0xff977c5bu, 0xe60def3eu, 0x63630141u, 0xb8fbcb58u } -#define IGNEUM_CACHE_LOG2_WORDS 26 -#define IGNEUM_CACHE_SEGMENT_LOG2_LINES 6 -#define IGNEUM_CACHE_SEGMENTS 65536u -#define IGNEUM_ITEM_ROUNDS 8 -#define IGNEUM_MIXER_MULT 8 // mixer applications per round and after the last read (class v3, docs/plans/mixer-x4.md) -#define IGNEUM_MIX_ROT_INIT { 20u, 20u, 19u, 4u, 26u, 3u, 3u, 27u } -#define IGNEUM_MIX_MUL_INIT { 0x42146205u, 0x52cbe0fbu, 0x7ecf4a03u, 0x6728907fu, 0xd81d9751u, 0x132952c3u, 0xf60de277u, 0x05358035u, 0xbaf6499du, 0xe4db9667u, 0x3e98f45du, 0xd0004eddu, 0x2691630du, 0x9beb3bcfu, 0xab310379u, 0x99cfb423u } -#define IGNEUM_MIX_RC_INIT { 0xbab68293u, 0xcc162340u, 0x6ce151ccu, 0xe62b8997u, 0xc9c80297u, 0xf74a1654u, 0x3d704af5u, 0x3cf522b7u, 0x2b9cac04u, 0xa880ac10u, 0x13e5dd1du, 0x6fc3e233u, 0x2d83eeacu, 0x9006e8bfu, 0x2c4b5362u, 0x31b49ee2u } - -#ifndef IGNEUM_NO_CUDA -// Defined in kernel.cu. All launch on the default stream and return cudaGetLastError(). -cudaError_t igneum_launch_cache_fill(uint32_t* cache, uint32_t nSegments); -cudaError_t igneum_launch_build(uint32_t* ds, const uint32_t* cache, uint32_t nItems); -cudaError_t igneum_launch_hash(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask, - uint32_t nonces, uint32_t blockWarps); -cudaError_t igneum_hash_info(int* numRegs, int* blocksPerSM, uint32_t blockWarps); -#endif diff --git a/proto-cuda/packs-ca4/mx8_mm1430/program.json b/proto-cuda/packs-ca4/mx8_mm1430/program.json deleted file mode 100644 index 04d40f8b0..000000000 --- a/proto-cuda/packs-ca4/mx8_mm1430/program.json +++ /dev/null @@ -1,134 +0,0 @@ -{ - "format": "igneum-program-pack-3", - "generator": 2, - "attempt": 0, - "program_id": "0xd746ef93184dc1f8", - "program_id_derivation": "FNV-1a 64 over 'igneum-program/' || generator_le32 || seed_words as little-endian bytes || attempt_le32", - "dataset_mode": "memory-hard", - "seed": "igneum-genesis", - "seed_bytes": "69676e65756d2d67656e65736973", - "seed_words": ["0x67a9a7be", "0x1a155b25", "0xfddfb732", "0x4b5af2e8", "0xc55caf33", "0xa27c13b7", "0x06628a48", "0x03852469"], - "seed_derivation": "seed_words = FNV-1a 64 over seed_bytes (attempt 0) or seed_bytes || attempt_le32 (attempt k >= 1), basis ^ (salt * 0x9E3779B97F4A7C15) for salt 0..3, then h ^= h>>33; h *= 0xff51afd7ed558ccd; h ^= h>>33; words[2*salt] = low 32, words[2*salt+1] = high 32", - "generator_rule": "version 2: exactly 16 load slots drawn first from instructions 1..63 (partial Fisher-Yates), the other 48 ops from the ten non-load weights (sum 75); a load's source is drawn from the registers other than dst written by an earlier instruction and not read by a load since; the candidate must pass the acceptance rule of spec 01 section 1.4.6 (static: no cyclically stale load source, every register has an injecting write; dynamic: 64 units on the seed-keyed closed-form dataset with no constant register bit, no lane-constant load site, under 164 saturated final values, every output bit within 136 of 1024, distinct addresses above 245760), else the next attempt of the seed is tried", - "lanes": 32, - "registers": 8, - "iterations": 8, - "instruction_count": 64, - "loads_per_hash": 128, - "load_class": "mx8+mm1430", - "mixer_mult": 8, - "cache_growth": true, - "mixer": "class v3 (Counter ASIC 2.0, 5 October 2026, docs/plans/mixer-x4.md): every mixer application of the item derivation is 8 applications with round keys (r * 8 + j + 1) * 0x9E3779B9, the 8 dependent cache reads per item unchanged; cache growth rule option C: cache words = 2^(26 + doublings(day)), dataset words = 2^(genesis_log2 + doublings(day)), doublings(day) = floor(log2(1 + day / 1460)) for day = days since genesis", - "load_slots": 16, - "load_mix_percent_4_16_64": [100, 0, 0], - "load_width_counts_4_16_64": [16, 0, 0], - "bytes_per_hash": 512, - "wide_load": "read-width experiment (5 October 2026, docs/plans/read-width.md), NOT the lottery hash: a load of W words (width field, 4 or 16) reads dataset[b .. b + W) with b = (src & mask) & ~(W - 1) and folds every word into dst: x = dst ^ w[0]; for j in 1..W: x = (rotl(x, 11) * 0x9e3779b1) ^ w[j]; dst = x; width 1 is the plain load; the width is drawn per instruction from the class mix with one extra below(100) draw after the nine of version 2, and the program id is FNV-1a 64 over 'igneum-program-rw/' || generator_le32 || seed words || attempt_le32 || mix[3] || load_slots", - "op_mix": {"load": 16, "add": 8, "shfl": 8, "xor": 6, "mad": 5, "mul": 5, "mulhi": 5, "sub": 4, "rotl": 3, "rotr": 3, "or": 1}, - "register_init": "for i in 0..7: x = nonce ^ seed_words[i]; x += 0x9e3779b9 * (i+1) (mod 2^32); x = splitmix32(x); r[i] = x ^ seed_words[(i+1) & 7]", - "splitmix32": "x ^= x>>16; x *= 0x7feb352d; x ^= x>>15; x *= 0x846ca68b; x ^= x>>16", - "iteration": "sel = r0 sampled once at the top of each iteration, then all instructions in order", - "output": "lo = r0 ^ rotl(r1,7) ^ rotl(r2,14) ^ rotl(r3,21); hi = r4 ^ rotl(r5,9) ^ rotl(r6,18) ^ rotl(r7,27); out = (hi << 32) | lo", - "op_semantics": { - "add": "dst = dst + src + (bit `bit` of sel ? imm2 : imm)", - "sub": "dst = dst - src", - "mul": "dst = dst * src (low 32)", - "mulhi": "dst = high 32 bits of dst * src", - "xor": "dst = dst ^ src", - "or": "dst = dst | src", - "rotl": "dst = rotl(dst, rot), rot in 1..31", - "rotr": "dst = rotr(dst, src & 31)", - "mad": "dst = src * src2 + dst", - "shfl": "dst = dst ^ (src of lane (lane ^ mask)), mask in {1,2,4,8,16}, within the 32-lane warp", - "load": "dst = dst ^ dataset[src & dataset.mask]", - "wload": "base = (src of lane 0 & dataset.mask) & ~31; dst = dst ^ dataset[base + lane] (warp-coalesced 128-byte load, lever b, only when --wide-frac > 0)" - }, - "dataset": { - "log2_words": 28, - "bytes": 1073741824, - "mask": "0x0fffffff", - "day": "2026-10-03", - "day_bytes": "6461792f323032362d31302d3033", - "day_words_from": "seed_words_from_bytes(day_bytes)", - "d0": "0x3067619f", - "d1": "0x3c269176", - "mode": "memory-hard", - "spec": "proto-metal/MEMHARD.md", - "key": ["0x3067619f", "0x3c269176", "0x84a03b03", "0xf8c63294", "0xff977c5b", "0xe60def3e", "0x63630141", "0xb8fbcb58"], - "key_derivation": "the 8 words of seed_words_from_bytes(day_bytes); d0, d1 are key[0], key[1]", - "cache": {"log2_words": 26, "bytes": 268435456, "line_words": 16, "segment_lines": 64, "segments": 65536, "block": "ChaCha12 core + feed-forward, rotations 16 12 8 7", "sigma": ["0x61707865", "0x3320646e", "0x79622d32", "0x6b206574"], "tag": ["0x49676e65", "0x756d4d48"], "chain": "in_j = prev_line ^ (sigma[0..3] || key[0..7] || seg || j || tag[0..1]); line_j = block(in_j); prev_0 = 0"}, - "mixer": {"draw": "SplitMix64 seeded with key[0] | key[1] << 32: rot[0..7] = 1 + next() % 31, mul[0..15] = low32(next()) | 1, rc[0..15] = low32(next())", "rot": [20, 20, 19, 4, 26, 3, 3, 27], "mul": ["0x42146205", "0x52cbe0fb", "0x7ecf4a03", "0x6728907f", "0xd81d9751", "0x132952c3", "0xf60de277", "0x05358035", "0xbaf6499d", "0xe4db9667", "0x3e98f45d", "0xd0004edd", "0x2691630d", "0x9beb3bcf", "0xab310379", "0x99cfb423"], "rc": ["0xbab68293", "0xcc162340", "0x6ce151cc", "0xe62b8997", "0xc9c80297", "0xf74a1654", "0x3d704af5", "0x3cf522b7", "0x2b9cac04", "0xa880ac10", "0x13e5dd1d", "0x6fc3e233", "0x2d83eeac", "0x9006e8bf", "0x2c4b5362", "0x31b49ee2"], "round": "for i in 0..15: s[i] = (s[i] ^ (rc[i] + (r+1) * 0x9E3779B9)) * mul[i]; then quarter rounds on columns (0,4,8,12) (1,5,9,13) (2,6,10,14) (3,7,11,15) with rot[0..3] and diagonals (0,5,10,15) (1,6,11,12) (2,7,8,13) (3,4,9,14) with rot[4..7]", "quarter_round": "a += b; d ^= a; d = rotl(d, r1); c += d; b ^= c; b = rotl(b, r2); a += b; d ^= a; d = rotl(d, r3); c += d; b ^= c; b = rotl(b, r4)"}, - "mixer_mult": 8, - "item": "s[0..7] = key; s[8+i] = t * mul[i] + rc[i] for i in 0..7; for r in 0..7: for j in 0..7: s = M(s, rk = (r * 8 + j + 1) * 0x9E3779B9); line = s[0] & 0x003fffff; s[i] ^= cache[line * 16 + i]; then for j in 0..7: s = M(s, rk = (64 + j + 1) * 0x9E3779B9); item(t) = s", - "word": "dataset[w] = item(w >> 4)[w & 15]" - }, - "shadow": {"instrs": 0, "reps": 0, "instrs_per_hash": 0, "op_mix": {}, "rule": "Counter ASIC 3.0 item 8 (docs/analysis/latency-shadow-2026-10-06.md): after the 64 base instructions the program stream draws instrs more ALU instructions (the non-load table, nine draws each, the source as on an ALU slot); the block runs reps times at the end of every iteration with the iteration's sel; the acceptance rule interprets the base program only", "program_id_suffix": "'shadow/' || instrs_le16 || reps_le16", "instructions": [ - ]}, - "mm8_tiles": {"tiles": 1430, "tiles_per_hash": 11440, "rule": "Counter ASIC 4.0 research (docs/analysis/counter-asic-4-research.md 15.2): after the shadow draws the stream draws tiles descriptors a, b, c (below 8 each) and c2 (below 7, skipping c); each tile is the PTX mma.m8n8k16 u8 product of the 32 lanes' r[a] (8 x 16 bytes) and r[b] (16 x 8), lane l adds C[l >> 2][2 (l & 3)] into r[c] and C[l >> 2][2 (l & 3) + 1] into r[c2] modulo 2^32; the block runs once per iteration after the shadow", "program_id_suffix": "'mm8/' || tiles_le16", "descriptors": [[6,5,2,3],[1,1,4,5],[2,5,0,7],[1,5,2,3],[2,0,2,6],[2,6,7,3],[1,4,1,3],[4,6,0,4],[1,1,1,5],[3,7,4,6],[3,0,0,4],[2,3,0,6],[3,1,6,0],[1,5,2,5],[3,3,3,4],[1,0,5,4],[7,2,0,4],[0,5,1,7],[1,1,2,0],[3,3,2,4],[3,0,6,1],[0,3,6,0],[1,7,6,4],[1,2,0,3],[3,4,0,2],[5,0,3,7],[1,0,7,3],[3,2,0,5],[2,4,6,5],[3,4,1,5],[6,5,6,3],[1,7,4,3],[7,6,3,0],[7,1,4,6],[1,0,5,1],[2,4,1,3],[1,7,7,1],[1,7,0,2],[3,6,2,4],[6,2,2,5],[6,1,4,1],[6,0,6,2],[7,4,5,1],[6,1,6,0],[0,6,4,1],[6,5,6,5],[6,6,1,0],[7,3,3,1],[7,7,6,3],[4,5,1,7],[1,7,4,5],[3,4,0,3],[2,5,7,4],[7,7,1,3],[7,6,6,5],[1,7,3,7],[4,0,7,3],[4,3,6,7],[2,3,6,0],[6,7,5,7],[6,0,2,4],[2,1,2,6],[4,6,1,6],[1,6,0,3],[0,5,5,3],[0,1,2,4],[3,5,1,3],[6,2,3,4],[7,1,1,2],[6,7,5,4],[1,7,1,5],[3,0,1,0],[4,0,3,6],[4,1,0,2],[5,1,3,4],[4,4,7,3],[5,6,1,7],[0,6,3,2],[2,5,0,5],[7,4,4,1],[6,7,3,4],[4,3,1,6],[6,0,1,7],[2,2,1,6],[0,6,0,3],[1,3,4,1],[3,4,2,3],[4,4,3,7],[6,4,5,3],[4,2,2,1],[6,5,3,6],[6,5,5,4],[2,6,2,3],[2,2,5,7],[2,4,7,3],[5,4,1,6],[0,6,3,1],[0,5,1,2],[2,7,3,2],[1,4,0,2],[2,4,2,5],[2,4,6,7],[5,7,6,3],[4,7,4,3],[2,0,3,7],[6,1,0,5],[4,1,0,5],[2,1,4,2],[4,6,1,2],[2,7,5,3],[1,0,0,2],[6,5,4,6],[5,7,4,2],[6,3,4,1],[2,1,7,5],[1,4,1,4],[7,2,5,0],[7,6,7,6],[0,1,2,1],[4,3,5,3],[5,3,7,2],[2,7,2,1],[0,1,3,0],[5,2,3,5],[5,1,2,5],[7,2,7,2],[6,3,3,2],[6,7,5,1],[6,7,4,1],[3,2,6,3],[0,4,2,1],[3,5,3,1],[0,7,0,6],[2,6,0,1],[5,6,3,1],[0,2,4,5],[3,1,1,5],[7,0,4,6],[2,2,4,2],[0,4,2,7],[4,6,1,6],[5,5,1,2],[2,2,6,2],[4,3,7,1],[2,0,1,7],[7,2,6,5],[2,7,2,7],[3,1,3,0],[2,1,0,6],[4,6,7,1],[3,6,7,0],[1,4,2,4],[3,1,5,1],[0,3,2,3],[7,3,0,4],[1,3,7,5],[6,3,2,3],[6,4,4,5],[6,5,0,4],[1,0,5,2],[0,1,5,7],[4,0,7,0],[6,6,6,1],[5,3,7,3],[0,7,3,2],[6,5,1,5],[6,5,4,0],[4,5,3,7],[3,6,0,7],[6,2,4,3],[6,5,0,4],[6,3,2,3],[7,3,5,2],[0,0,2,4],[6,6,5,6],[1,6,3,4],[2,4,7,5],[1,7,7,2],[4,2,7,3],[5,2,5,1],[0,4,2,1],[3,5,2,4],[1,1,4,3],[3,1,0,7],[7,4,0,7],[5,0,6,3],[2,5,7,6],[3,0,0,7],[1,4,5,0],[7,2,5,4],[1,5,2,0],[2,1,2,5],[5,0,2,4],[2,1,4,6],[4,1,0,5],[2,7,2,4],[5,6,2,5],[5,3,6,5],[1,0,7,1],[2,5,0,4],[5,7,3,2],[7,2,2,1],[2,5,5,6],[0,5,1,6],[4,7,3,7],[3,7,1,3],[0,6,7,3],[7,7,1,4],[0,1,6,5],[4,4,6,2],[7,1,2,5],[1,5,5,6],[2,4,5,3],[2,2,3,5],[4,6,5,7],[7,3,2,6],[6,2,3,4],[5,7,0,6],[7,1,1,0],[2,1,3,1],[1,7,4,5],[7,1,2,1],[0,6,2,1],[2,2,7,0],[1,1,6,2],[2,7,6,4],[2,1,7,5],[3,5,5,6],[7,1,3,1],[1,4,1,4],[0,3,5,6],[1,3,4,0],[5,4,0,1],[7,2,0,4],[2,0,2,1],[3,4,4,1],[1,7,0,5],[3,5,6,3],[2,0,0,2],[5,7,1,0],[2,7,4,6],[0,1,3,7],[0,2,2,7],[4,4,1,5],[7,6,0,7],[4,4,2,0],[0,6,1,2],[3,6,2,4],[7,5,7,3],[4,4,0,5],[0,6,0,1],[0,7,4,3],[0,2,5,4],[5,5,7,4],[6,7,6,3],[5,0,0,3],[2,1,3,4],[4,3,6,0],[4,1,7,3],[5,5,3,0],[3,2,3,0],[0,6,3,2],[6,4,2,5],[6,4,2,1],[7,2,1,4],[6,1,3,1],[0,0,7,0],[6,3,3,4],[1,0,6,5],[7,7,7,3],[6,2,7,0],[0,3,5,3],[6,0,0,3],[2,4,6,7],[1,0,7,6],[1,3,1,4],[1,5,6,2],[5,1,4,7],[0,0,6,7],[2,4,2,4],[7,2,3,5],[1,0,4,2],[0,1,5,1],[0,2,7,2],[1,5,0,2],[4,7,5,0],[1,0,2,0],[7,1,7,4],[0,7,4,3],[0,5,5,4],[3,2,6,7],[7,4,7,1],[6,7,6,3],[5,5,6,4],[5,7,1,6],[0,7,1,3],[6,1,5,6],[4,5,3,5],[6,0,3,1],[6,0,2,3],[6,4,4,2],[6,4,6,3],[7,6,6,4],[3,2,3,6],[6,5,2,4],[4,5,0,2],[3,1,0,2],[4,3,5,0],[2,1,2,6],[2,1,1,4],[6,1,2,3],[4,6,3,2],[1,3,4,1],[2,6,7,1],[0,3,2,1],[7,1,6,2],[2,4,1,6],[1,6,3,6],[2,1,1,0],[0,2,6,3],[5,2,6,2],[7,4,1,3],[2,4,7,1],[4,3,2,3],[5,7,2,0],[6,2,7,2],[6,1,7,6],[1,5,3,0],[0,1,1,6],[0,3,7,1],[6,7,4,0],[4,4,4,1],[4,4,6,3],[4,6,4,2],[2,2,0,1],[7,1,1,2],[4,0,2,3],[1,4,0,3],[3,7,2,7],[5,1,7,6],[5,0,7,5],[6,6,7,5],[4,7,0,4],[0,4,4,6],[2,5,4,6],[7,4,3,1],[6,0,7,3],[5,3,3,5],[0,4,4,5],[6,1,4,7],[1,5,3,5],[6,4,4,5],[4,1,6,0],[7,3,1,4],[4,1,1,5],[4,5,0,1],[7,6,0,4],[7,3,7,5],[4,7,5,7],[2,7,3,5],[4,4,0,2],[2,2,5,6],[2,7,3,0],[7,6,5,0],[6,4,0,5],[6,6,5,4],[7,7,2,5],[4,1,7,5],[7,0,3,5],[1,0,6,0],[2,5,7,4],[5,4,2,6],[2,7,6,2],[4,6,2,4],[1,5,0,4],[6,5,6,2],[3,6,0,3],[6,4,6,4],[6,3,2,4],[3,0,2,4],[2,0,6,2],[3,5,1,2],[5,2,2,4],[1,2,0,2],[3,1,1,0],[6,0,7,5],[2,0,1,7],[3,7,2,4],[3,0,1,2],[6,4,5,3],[5,4,1,4],[6,3,4,1],[2,3,3,0],[3,6,5,0],[0,7,1,2],[0,7,2,4],[7,7,1,4],[5,7,3,2],[6,5,7,0],[6,3,3,5],[2,7,0,4],[7,6,2,1],[0,0,7,1],[7,4,1,0],[1,1,1,4],[6,6,3,6],[6,1,4,3],[5,7,2,4],[7,4,4,1],[3,4,4,7],[0,5,5,2],[2,0,5,1],[7,4,3,1],[2,7,6,2],[7,1,0,3],[4,7,5,1],[5,4,3,5],[6,6,1,2],[1,3,1,5],[5,0,0,6],[4,2,7,6],[4,4,3,7],[0,7,4,3],[1,7,6,0],[5,1,6,5],[2,7,6,5],[1,6,5,0],[6,6,6,5],[6,2,7,4],[3,3,7,0],[7,7,3,5],[6,4,1,0],[3,2,3,4],[3,0,5,0],[0,7,5,3],[7,5,6,3],[6,4,0,5],[0,7,3,4],[5,1,4,7],[4,7,3,2],[3,5,3,2],[5,1,1,4],[1,2,7,5],[1,7,5,2],[3,7,1,0],[7,5,7,0],[3,1,2,1],[6,7,4,7],[5,2,4,5],[7,1,1,3],[7,5,7,3],[0,3,1,7],[2,4,7,1],[5,7,7,1],[7,7,7,1],[7,1,7,5],[6,5,5,6],[7,7,1,5],[5,1,4,1],[6,7,1,0],[7,7,6,5],[0,6,6,2],[0,6,1,2],[2,0,3,2],[0,3,0,1],[1,7,0,5],[1,2,4,1],[1,0,1,6],[4,1,6,2],[2,0,0,1],[3,1,2,0],[2,2,7,1],[1,7,6,5],[2,7,6,1],[5,3,6,3],[7,6,6,0],[3,0,5,6],[3,1,4,6],[0,6,2,5],[7,6,7,1],[6,0,1,0],[2,4,6,7],[2,3,0,6],[7,0,7,6],[3,7,6,4],[6,4,1,2],[6,5,2,5],[2,5,4,5],[1,2,1,2],[5,2,6,4],[7,1,2,6],[0,0,0,6],[3,1,6,4],[6,1,7,6],[5,6,3,1],[2,2,7,2],[3,5,5,0],[3,2,2,0],[3,1,6,2],[1,6,1,5],[7,6,1,5],[4,4,1,0],[5,7,7,6],[7,5,3,4],[2,0,4,2],[4,0,0,1],[0,2,7,0],[5,6,4,7],[7,3,4,7],[4,4,6,0],[0,6,6,5],[2,4,1,3],[2,7,4,1],[0,3,5,7],[6,7,4,2],[5,2,0,4],[0,2,2,4],[6,7,6,7],[1,4,0,6],[6,0,7,4],[1,2,0,2],[0,4,0,2],[7,2,6,7],[5,3,7,0],[0,4,7,0],[1,5,5,1],[7,1,1,6],[5,1,6,7],[4,1,3,2],[4,2,2,1],[0,2,6,4],[7,0,3,4],[7,2,2,7],[3,2,6,4],[4,3,3,6],[0,6,1,6],[3,1,4,6],[1,6,4,5],[6,6,6,7],[4,4,3,1],[1,4,2,4],[5,3,3,6],[7,7,0,7],[4,2,3,7],[5,7,3,6],[5,2,2,1],[7,1,5,7],[3,5,2,1],[6,3,5,4],[2,2,3,7],[5,5,5,2],[7,6,4,0],[6,7,1,2],[6,3,6,7],[4,2,7,0],[2,7,0,1],[3,6,0,7],[7,7,7,0],[5,0,5,2],[4,1,5,1],[5,4,5,4],[6,1,2,5],[1,2,7,0],[5,3,3,2],[4,3,4,6],[6,0,2,7],[1,6,5,0],[3,6,2,0],[6,5,3,1],[3,7,2,5],[5,1,3,2],[6,0,2,6],[5,0,3,1],[3,0,4,3],[2,2,0,5],[3,1,6,0],[2,3,0,4],[6,3,6,1],[5,6,2,4],[4,7,7,4],[7,1,5,7],[1,6,3,4],[2,1,1,3],[5,0,7,3],[1,7,0,4],[0,4,5,2],[4,4,1,2],[0,4,6,3],[5,5,7,0],[3,6,3,5],[3,2,5,0],[5,1,0,5],[5,3,7,4],[5,3,6,4],[6,7,4,0],[3,0,6,4],[7,6,3,2],[2,4,5,1],[7,7,4,0],[7,2,2,6],[7,7,5,4],[2,7,7,5],[0,1,7,1],[0,0,7,4],[5,5,2,1],[2,0,4,2],[7,5,1,7],[0,2,4,5],[7,6,6,4],[1,5,7,1],[3,3,1,6],[2,2,5,7],[1,1,1,2],[6,3,3,4],[3,4,1,3],[6,2,7,6],[2,3,1,7],[5,4,3,2],[3,6,5,0],[5,0,0,7],[3,5,6,4],[4,2,7,2],[7,0,4,3],[4,6,2,1],[6,2,1,0],[6,2,7,5],[5,2,5,6],[4,0,2,3],[1,5,2,0],[4,5,7,3],[4,5,6,4],[6,4,7,4],[2,0,7,1],[2,1,2,5],[1,1,2,5],[6,7,2,4],[2,5,4,1],[5,7,4,3],[1,4,5,1],[1,7,5,1],[0,4,4,0],[5,6,5,4],[7,5,1,6],[3,5,5,6],[1,6,2,5],[0,7,4,1],[5,6,4,3],[6,4,6,4],[3,1,6,2],[7,2,6,7],[7,7,3,5],[6,3,3,4],[1,6,3,2],[4,1,0,5],[1,6,7,4],[7,5,3,1],[6,5,1,6],[7,4,7,2],[3,0,6,1],[0,7,3,0],[6,1,4,5],[3,7,1,4],[2,3,5,3],[0,3,0,4],[2,0,1,3],[1,1,1,0],[7,1,2,7],[2,0,3,6],[3,1,3,6],[1,4,4,3],[7,7,5,0],[0,3,0,3],[0,1,4,3],[6,2,0,2],[3,3,0,5],[5,6,3,6],[4,3,1,0],[0,1,7,5],[4,7,7,1],[4,7,1,4],[5,1,5,2],[4,2,4,2],[3,6,6,5],[0,5,4,7],[6,2,3,7],[7,3,0,5],[4,1,1,7],[0,5,3,0],[3,6,4,1],[3,1,7,1],[2,6,0,1],[2,6,0,6],[1,4,0,7],[7,1,2,5],[3,5,7,3],[3,2,2,4],[2,4,3,0],[2,3,3,4],[3,0,4,7],[2,1,4,2],[0,1,1,3],[5,7,1,3],[5,5,4,6],[2,2,1,2],[6,5,7,2],[5,4,7,3],[6,0,4,1],[6,4,7,6],[2,0,7,4],[7,0,7,5],[4,2,5,6],[4,7,6,1],[2,7,6,2],[5,2,5,1],[0,0,5,1],[1,7,6,5],[3,3,5,6],[3,3,7,1],[7,5,2,7],[4,7,3,0],[4,7,5,3],[6,4,0,3],[7,5,7,6],[5,0,7,2],[7,4,4,0],[6,1,3,5],[3,1,5,1],[0,6,4,6],[5,7,6,5],[5,6,2,3],[5,1,4,7],[5,6,6,7],[5,1,0,6],[3,4,4,1],[4,3,0,3],[7,4,4,3],[4,2,3,0],[4,7,1,2],[6,2,1,2],[7,5,6,4],[4,3,2,7],[7,0,7,3],[6,5,4,1],[5,3,6,4],[7,4,3,6],[7,0,0,4],[2,5,1,3],[0,3,1,0],[1,6,5,7],[2,5,7,5],[4,3,7,0],[6,6,5,3],[5,0,5,3],[6,7,6,0],[2,4,5,2],[5,6,4,0],[7,3,5,4],[6,6,3,5],[6,2,4,2],[2,7,2,0],[7,7,1,0],[3,3,2,4],[5,2,5,6],[1,1,6,0],[1,2,0,2],[2,4,6,3],[7,6,6,0],[7,5,5,7],[5,6,7,3],[4,5,1,5],[7,5,2,6],[2,5,3,6],[0,4,1,6],[2,5,1,3],[4,0,2,0],[4,6,1,2],[5,7,1,0],[1,1,2,0],[1,2,5,7],[7,5,0,7],[3,7,0,3],[6,7,7,3],[6,4,6,4],[6,5,6,0],[1,5,2,6],[7,2,5,3],[2,4,5,2],[7,2,4,6],[0,0,7,6],[1,4,3,5],[3,1,2,3],[3,5,7,4],[7,1,6,4],[7,2,1,0],[2,4,3,0],[1,7,6,2],[2,4,3,6],[6,2,2,7],[1,3,2,1],[1,3,4,2],[0,4,6,4],[2,0,1,7],[4,7,6,0],[3,2,7,5],[0,4,7,2],[4,6,5,2],[0,5,2,7],[5,0,2,3],[2,4,7,4],[1,6,4,6],[1,2,5,4],[2,7,6,3],[0,6,6,1],[2,2,2,3],[3,4,2,6],[4,5,5,7],[3,0,1,7],[3,4,5,7],[2,2,5,2],[1,2,7,4],[3,6,2,6],[7,3,5,1],[1,5,4,3],[1,4,2,5],[7,7,0,5],[1,1,1,2],[0,2,7,1],[3,1,6,7],[7,4,2,4],[5,2,0,5],[6,3,4,0],[2,7,2,5],[1,6,6,1],[0,6,5,2],[2,4,3,6],[0,7,5,0],[3,0,7,3],[4,7,6,4],[5,4,4,3],[0,6,7,3],[1,6,5,3],[4,0,4,2],[3,5,3,7],[2,2,2,4],[3,3,2,1],[1,7,6,3],[1,4,2,3],[4,0,2,4],[6,7,1,0],[6,6,5,4],[6,0,6,3],[0,5,0,3],[2,2,2,1],[6,5,6,7],[1,4,5,7],[1,7,0,1],[2,3,1,7],[6,5,4,0],[0,5,1,7],[2,6,0,3],[7,2,4,7],[3,3,7,4],[0,1,4,5],[0,0,4,1],[5,5,2,1],[3,7,3,2],[6,6,1,5],[7,3,4,1],[4,4,1,3],[3,6,2,7],[0,1,6,5],[5,1,3,1],[0,6,1,4],[2,1,2,3],[5,1,5,6],[1,7,2,5],[7,6,7,1],[6,2,6,1],[5,0,7,4],[4,1,7,2],[3,3,7,1],[2,7,0,5],[7,5,1,4],[2,3,1,2],[2,7,5,2],[3,5,4,0],[5,4,5,3],[1,1,4,7],[6,2,7,3],[4,6,4,5],[0,5,6,2],[0,7,2,0],[5,6,7,4],[2,1,0,1],[0,7,3,7],[2,4,5,0],[0,2,2,5],[7,4,6,0],[1,3,3,0],[5,1,1,3],[7,7,7,0],[4,6,1,5],[2,1,5,6],[0,3,1,6],[6,7,5,6],[0,4,6,1],[3,7,2,3],[2,4,0,1],[7,1,4,3],[4,5,7,5],[3,7,4,5],[2,6,3,5],[6,1,4,5],[2,3,2,6],[7,4,5,4],[4,3,3,5],[4,2,3,6],[5,5,6,7],[7,2,3,7],[3,4,6,7],[6,5,2,6],[6,5,0,2],[1,0,4,7],[6,3,6,7],[2,0,7,6],[6,6,3,4],[5,0,2,0],[1,1,7,5],[5,7,7,5],[1,0,7,5],[7,7,3,2],[7,0,7,3],[1,3,6,4],[1,3,0,3],[6,0,6,0],[5,0,6,7],[3,3,1,6],[0,6,7,5],[1,5,1,3],[7,0,3,0],[1,2,4,5],[1,6,2,7],[0,6,1,3],[4,0,1,4],[6,5,2,1],[3,7,4,2],[2,6,6,5],[7,2,5,3],[6,4,3,1],[6,6,0,6],[7,6,6,5],[2,7,6,5],[3,2,1,2],[4,4,0,5],[2,4,6,2],[0,7,0,7],[7,1,7,5],[7,4,3,1],[6,5,7,4],[4,7,1,5],[1,1,4,2],[1,5,6,3],[6,4,0,2],[7,1,4,7],[2,5,1,0],[3,7,6,5],[7,3,6,0],[0,0,7,1],[5,7,5,7],[4,4,6,1],[4,3,0,7],[3,4,7,3],[1,7,4,7],[4,1,1,7],[5,6,3,2],[1,2,6,4],[0,6,7,6],[2,0,4,0],[0,2,2,6],[4,0,3,6],[3,1,2,7],[2,0,2,1],[1,6,0,3],[2,1,7,4],[1,7,7,1],[4,3,1,5],[2,7,2,5],[5,1,7,3],[6,0,3,7],[2,3,1,3],[1,5,4,3],[6,0,1,0],[2,1,0,4],[6,0,1,7],[6,0,6,2],[0,3,1,3],[7,0,6,0],[5,5,0,6],[3,4,1,7],[7,2,4,7],[5,3,5,3],[5,6,0,5],[6,2,1,4],[4,2,1,5],[7,3,4,5],[1,7,4,1],[4,2,2,7],[0,2,4,5],[3,5,5,7],[0,7,6,7],[6,7,2,5],[7,5,5,2],[3,5,1,5],[6,6,6,1],[7,3,2,3],[7,6,2,6],[4,0,1,6],[5,2,6,3],[7,6,2,6],[0,7,5,4],[5,6,5,4],[7,6,3,4],[3,5,2,6],[0,3,6,7],[2,6,2,1],[4,7,2,0],[0,1,2,4],[1,5,5,0],[7,6,3,6],[2,1,0,7],[1,7,0,2],[3,7,0,3],[2,1,7,2],[2,7,5,6],[4,3,4,3],[3,0,5,6],[3,0,7,5],[7,1,4,2],[2,1,1,3],[5,1,7,5],[0,7,3,7],[2,2,7,1],[6,6,2,4],[6,7,3,6],[4,2,7,1],[0,0,1,6],[7,1,3,2],[5,7,5,6],[2,3,6,0],[5,1,6,7],[0,5,1,0],[7,7,4,2],[2,3,4,1],[6,0,2,0],[4,2,4,5],[5,3,7,2],[5,1,5,6],[1,5,5,4],[0,5,4,3],[5,6,4,5],[7,3,5,0],[5,6,6,1],[3,0,2,0],[1,1,7,0],[5,4,2,1],[3,1,4,5],[7,7,4,5],[0,5,6,1],[4,6,4,6],[2,0,6,5],[2,3,2,1],[2,6,1,7],[3,6,2,4],[3,6,2,0],[0,0,7,1],[4,4,7,1],[1,7,3,6],[7,7,6,5],[3,7,1,3],[6,6,7,4],[4,5,1,4],[1,5,4,5],[5,4,3,6],[2,2,6,7],[2,0,2,1],[3,3,0,4],[3,1,3,2],[1,3,6,5],[3,5,5,7],[0,0,2,3],[4,6,7,2],[4,3,6,4],[2,6,1,6],[5,7,0,4],[7,3,2,6],[0,2,4,6],[3,3,6,1],[6,2,5,4],[3,6,1,6],[2,5,4,0],[1,4,1,0],[0,6,5,7],[0,5,6,5],[7,0,7,3],[4,2,7,4],[3,4,2,0],[2,1,7,0],[3,0,6,5],[3,5,4,2],[6,2,4,3],[2,4,4,7],[0,7,2,1],[1,0,3,0],[4,6,5,6],[0,5,0,6],[4,5,3,4],[4,6,7,5],[0,6,0,7],[7,6,5,6],[2,7,0,5],[7,5,0,5],[4,0,6,7],[0,5,3,2],[0,6,0,3],[0,1,4,0],[6,6,2,0],[0,0,0,3],[1,1,7,1],[4,1,7,3],[0,0,6,2],[0,4,7,5],[3,0,3,0],[0,6,5,7],[6,0,2,1],[7,3,1,4],[6,7,6,4],[7,4,7,2],[4,0,5,7],[6,4,3,4],[6,2,1,7],[1,2,6,3],[0,5,1,6],[4,5,2,5],[5,6,6,1],[6,4,7,3],[5,1,2,7],[6,5,5,3],[7,1,4,6],[1,6,4,6],[5,5,5,7],[3,1,5,1],[7,5,7,5],[0,6,5,3],[2,5,0,3],[5,3,7,0],[0,6,0,4],[2,6,4,7],[6,7,1,2],[2,3,3,7],[1,5,0,4],[5,0,2,7],[7,0,3,4],[5,3,0,4],[4,3,2,1],[3,4,2,0],[1,2,7,1],[4,7,7,4],[2,7,2,5],[2,6,4,0],[0,4,3,7],[2,1,3,4],[1,7,6,7],[0,1,0,4],[6,4,4,1],[0,1,2,6],[0,0,3,1],[4,5,5,6],[7,2,5,2],[1,5,7,0],[0,7,6,5],[7,3,5,2],[7,3,0,7],[1,2,7,3],[2,4,3,1],[7,0,2,7],[5,2,4,5],[1,0,2,3],[5,4,2,7],[3,0,4,6],[6,5,0,4],[5,0,2,7],[3,1,5,1],[0,2,2,1],[3,7,5,1],[7,2,1,2],[1,4,0,4],[4,6,5,0],[5,6,3,4],[0,6,2,5],[5,1,7,2],[1,3,1,2],[6,6,2,0],[2,4,4,7],[0,5,4,2],[0,2,7,4],[4,6,7,3],[4,6,4,6],[3,7,7,6],[6,1,2,4],[5,5,2,4],[0,0,3,0],[7,5,2,3],[2,6,7,1],[6,2,0,1],[4,2,3,5],[7,2,1,7],[3,3,7,0],[5,5,3,5],[0,7,3,6],[2,4,0,4],[1,7,7,4],[5,5,3,2],[1,3,2,0],[0,2,7,5],[1,1,6,3],[4,1,6,3],[5,6,4,5],[1,7,1,6],[4,2,4,5],[5,3,0,6],[6,2,2,7],[4,3,5,3],[6,7,5,4],[0,5,4,2],[5,4,7,1],[5,7,3,5],[0,7,1,3],[7,2,4,1],[7,0,5,0],[3,2,3,2],[0,5,2,4],[5,0,3,7],[5,7,7,4],[6,3,6,7],[1,5,0,4],[0,6,5,1],[5,5,1,4],[1,6,7,1],[4,0,2,7],[7,6,1,7],[3,0,6,1],[3,7,1,0],[5,5,2,7],[5,5,6,2],[0,6,3,5],[1,4,3,1],[1,3,5,6],[0,7,7,3],[5,4,4,3],[0,3,5,2],[1,5,5,7],[6,7,4,2],[2,5,5,1],[3,7,4,1],[1,6,6,7],[2,3,3,6],[4,6,1,3],[6,6,7,3],[4,5,5,7],[5,7,7,4],[7,3,7,3],[7,0,6,0],[1,3,1,3],[3,2,7,5],[2,0,6,3],[7,3,0,5],[4,3,5,2],[5,4,2,4],[6,3,1,6],[0,1,7,1],[4,1,4,2],[6,3,2,4],[1,7,2,5],[6,2,7,1],[1,6,7,2],[3,5,1,4],[3,1,2,4],[4,1,5,3],[3,6,6,0],[6,2,2,5],[6,5,1,5],[5,6,7,6],[6,6,1,6],[6,2,0,4],[0,5,2,3],[5,0,2,6],[1,0,3,5],[0,0,6,1],[2,6,0,4],[7,0,7,4],[4,1,4,5],[4,5,2,5],[0,4,7,2],[4,2,7,1],[4,7,3,7],[3,2,2,4],[3,0,2,4],[0,2,6,7],[2,5,7,1],[5,7,5,4],[4,5,4,7],[4,7,1,4],[3,7,6,3],[5,1,7,0],[5,2,0,2],[1,0,3,7],[7,3,6,7],[2,5,0,3],[2,3,5,4],[7,5,0,5],[0,3,4,6],[1,0,6,1],[4,0,6,3],[5,3,0,3],[3,0,4,6],[6,4,1,0],[4,6,1,0],[1,3,0,4],[7,3,7,6],[3,7,6,2],[3,6,5,3],[0,7,2,3],[0,7,5,4],[3,3,5,1],[7,5,0,3],[5,7,7,6],[5,5,1,2],[6,0,6,7],[6,4,7,6],[4,6,6,0],[6,1,7,0],[0,6,1,4],[4,6,0,1],[5,5,2,1],[2,6,3,1],[7,3,4,2],[5,0,0,2],[2,3,4,2],[4,3,2,1],[0,1,3,1],[1,6,0,4],[7,1,1,2],[5,4,5,0],[2,7,3,2],[7,4,7,0],[7,4,3,0],[4,6,5,3],[4,6,1,6],[1,7,7,1],[7,7,3,2],[0,4,3,1],[0,5,7,4],[0,2,4,0],[4,4,5,6],[7,3,0,2],[2,4,3,4],[4,3,1,5],[3,3,6,2],[4,0,5,6],[5,3,2,7],[0,1,2,5],[0,5,0,3],[5,6,7,2],[7,1,3,2],[4,6,6,5],[5,2,2,5],[5,5,5,2],[7,3,7,5],[4,1,2,6],[4,0,7,2],[0,7,3,6],[2,0,0,7],[7,5,4,7],[1,5,2,1],[5,2,2,6],[0,7,5,1],[7,0,0,1],[4,3,0,2],[4,6,7,5],[4,2,3,2],[5,1,0,3],[6,1,4,0],[5,7,0,6],[3,4,7,4],[4,7,3,5],[7,1,6,5],[3,0,6,1],[0,6,1,2],[3,0,4,3],[3,4,3,1],[6,4,2,5],[0,5,3,5],[7,5,6,2],[0,0,4,7],[3,6,6,2],[0,0,2,3],[5,3,4,5],[7,3,3,1],[4,7,0,1],[4,2,6,2],[6,0,6,3],[1,7,6,4],[4,3,3,2],[4,6,4,2],[0,7,2,5],[3,6,7,5],[3,2,4,7],[2,7,0,2],[0,6,2,7],[0,3,4,1],[1,3,5,1],[4,0,3,2],[6,4,6,2],[4,6,3,1],[6,3,2,5],[7,3,0,1],[0,3,5,4],[5,6,3,6],[0,3,5,7],[7,1,3,7]]}, - "instructions": [ - {"i": 0, "op": "mad", "dst": 2, "src": 3, "src2": 4, "imm": "0xbf7b174d", "imm2": "0x337b762e", "rot": 17, "bit": 2, "mask": 2, "width": 1}, - {"i": 1, "op": "mad", "dst": 2, "src": 1, "src2": 1, "imm": "0xdd04a5da", "imm2": "0x42da7657", "rot": 15, "bit": 30, "mask": 16, "width": 1}, - {"i": 2, "op": "mad", "dst": 2, "src": 3, "src2": 2, "imm": "0x734003fa", "imm2": "0x5bb67700", "rot": 3, "bit": 20, "mask": 1, "width": 1}, - {"i": 3, "op": "xor", "dst": 3, "src": 5, "src2": 5, "imm": "0xc55a1b1c", "imm2": "0xa19720f3", "rot": 7, "bit": 8, "mask": 1, "width": 1}, - {"i": 4, "op": "load", "dst": 7, "src": 2, "src2": 5, "imm": "0xad572dd7", "imm2": "0x9ceb3ea7", "rot": 18, "bit": 30, "mask": 2, "width": 1}, - {"i": 5, "op": "load", "dst": 5, "src": 7, "src2": 3, "imm": "0x769a53be", "imm2": "0x80f9067e", "rot": 12, "bit": 22, "mask": 1, "width": 1}, - {"i": 6, "op": "shfl", "dst": 1, "src": 4, "src2": 0, "imm": "0xd3613d88", "imm2": "0x262fb219", "rot": 10, "bit": 30, "mask": 8, "width": 1}, - {"i": 7, "op": "shfl", "dst": 7, "src": 3, "src2": 4, "imm": "0xce38e42f", "imm2": "0xb868b818", "rot": 11, "bit": 8, "mask": 8, "width": 1}, - {"i": 8, "op": "mulhi", "dst": 1, "src": 5, "src2": 2, "imm": "0xa5eebca5", "imm2": "0x5703a72b", "rot": 13, "bit": 13, "mask": 16, "width": 1}, - {"i": 9, "op": "rotr", "dst": 6, "src": 3, "src2": 4, "imm": "0x17a5a9c7", "imm2": "0xdcfb93a1", "rot": 20, "bit": 27, "mask": 2, "width": 1}, - {"i": 10, "op": "or", "dst": 3, "src": 4, "src2": 1, "imm": "0xccb7d785", "imm2": "0xc335364c", "rot": 14, "bit": 12, "mask": 4, "width": 1}, - {"i": 11, "op": "load", "dst": 4, "src": 3, "src2": 2, "imm": "0x88cb9af3", "imm2": "0x4e7dc10d", "rot": 24, "bit": 17, "mask": 4, "width": 1}, - {"i": 12, "op": "mulhi", "dst": 0, "src": 4, "src2": 4, "imm": "0x45374321", "imm2": "0x3cd91989", "rot": 11, "bit": 4, "mask": 2, "width": 1}, - {"i": 13, "op": "add", "dst": 5, "src": 1, "src2": 2, "imm": "0xc7934706", "imm2": "0xd3177981", "rot": 16, "bit": 30, "mask": 2, "width": 1}, - {"i": 14, "op": "load", "dst": 0, "src": 4, "src2": 3, "imm": "0xfd7f56bb", "imm2": "0x65e14f52", "rot": 13, "bit": 22, "mask": 2, "width": 1}, - {"i": 15, "op": "sub", "dst": 2, "src": 4, "src2": 4, "imm": "0x35a80b49", "imm2": "0x060f2d13", "rot": 16, "bit": 20, "mask": 16, "width": 1}, - {"i": 16, "op": "load", "dst": 2, "src": 0, "src2": 2, "imm": "0xae0a32c2", "imm2": "0x4c2a4cfe", "rot": 8, "bit": 31, "mask": 16, "width": 1}, - {"i": 17, "op": "load", "dst": 7, "src": 2, "src2": 6, "imm": "0x82a84cc3", "imm2": "0x21a38d68", "rot": 15, "bit": 21, "mask": 2, "width": 1}, - {"i": 18, "op": "shfl", "dst": 7, "src": 3, "src2": 3, "imm": "0xa3818806", "imm2": "0x8f66b5c8", "rot": 14, "bit": 6, "mask": 4, "width": 1}, - {"i": 19, "op": "mul", "dst": 5, "src": 0, "src2": 1, "imm": "0xa00de107", "imm2": "0x77bfcaa5", "rot": 3, "bit": 10, "mask": 2, "width": 1}, - {"i": 20, "op": "shfl", "dst": 3, "src": 4, "src2": 7, "imm": "0x1d2b8cab", "imm2": "0x80b4f9a2", "rot": 14, "bit": 25, "mask": 2, "width": 1}, - {"i": 21, "op": "shfl", "dst": 2, "src": 4, "src2": 5, "imm": "0x3ac915d2", "imm2": "0x5fba7bc2", "rot": 16, "bit": 1, "mask": 16, "width": 1}, - {"i": 22, "op": "mulhi", "dst": 6, "src": 2, "src2": 0, "imm": "0xdc3ec8fd", "imm2": "0x599e2fa3", "rot": 22, "bit": 3, "mask": 2, "width": 1}, - {"i": 23, "op": "load", "dst": 6, "src": 1, "src2": 5, "imm": "0x2a6b16d5", "imm2": "0xd73e396f", "rot": 28, "bit": 29, "mask": 2, "width": 1}, - {"i": 24, "op": "mul", "dst": 5, "src": 0, "src2": 7, "imm": "0x376d0223", "imm2": "0xe1c2169a", "rot": 4, "bit": 16, "mask": 16, "width": 1}, - {"i": 25, "op": "rotl", "dst": 5, "src": 7, "src2": 3, "imm": "0x78ad8c60", "imm2": "0x6f5b77d5", "rot": 19, "bit": 11, "mask": 16, "width": 1}, - {"i": 26, "op": "shfl", "dst": 7, "src": 6, "src2": 5, "imm": "0x93915b9f", "imm2": "0x1e61fb6b", "rot": 28, "bit": 23, "mask": 2, "width": 1}, - {"i": 27, "op": "xor", "dst": 0, "src": 5, "src2": 7, "imm": "0x6378fe15", "imm2": "0x66c78f42", "rot": 12, "bit": 31, "mask": 8, "width": 1}, - {"i": 28, "op": "xor", "dst": 0, "src": 4, "src2": 7, "imm": "0x20a57fda", "imm2": "0x088c848e", "rot": 16, "bit": 13, "mask": 4, "width": 1}, - {"i": 29, "op": "sub", "dst": 3, "src": 0, "src2": 2, "imm": "0x49d95fd5", "imm2": "0x1a5f946a", "rot": 6, "bit": 12, "mask": 1, "width": 1}, - {"i": 30, "op": "mul", "dst": 5, "src": 1, "src2": 7, "imm": "0x0a816217", "imm2": "0x405c4f73", "rot": 13, "bit": 27, "mask": 4, "width": 1}, - {"i": 31, "op": "load", "dst": 7, "src": 2, "src2": 2, "imm": "0x09ed045e", "imm2": "0xd69c4715", "rot": 5, "bit": 9, "mask": 2, "width": 1}, - {"i": 32, "op": "load", "dst": 1, "src": 0, "src2": 6, "imm": "0xeb79ea49", "imm2": "0xcc587f5a", "rot": 6, "bit": 8, "mask": 16, "width": 1}, - {"i": 33, "op": "xor", "dst": 5, "src": 6, "src2": 1, "imm": "0x3027401e", "imm2": "0x5f20c27e", "rot": 18, "bit": 9, "mask": 2, "width": 1}, - {"i": 34, "op": "load", "dst": 5, "src": 1, "src2": 3, "imm": "0x0e1cab07", "imm2": "0x09356c5b", "rot": 19, "bit": 31, "mask": 1, "width": 1}, - {"i": 35, "op": "mulhi", "dst": 0, "src": 5, "src2": 2, "imm": "0x90e31357", "imm2": "0xabd32484", "rot": 26, "bit": 5, "mask": 8, "width": 1}, - {"i": 36, "op": "shfl", "dst": 5, "src": 2, "src2": 5, "imm": "0xee9a955f", "imm2": "0x31b3faed", "rot": 8, "bit": 24, "mask": 4, "width": 1}, - {"i": 37, "op": "load", "dst": 7, "src": 0, "src2": 7, "imm": "0x3ba2f832", "imm2": "0x1160dcd3", "rot": 4, "bit": 29, "mask": 1, "width": 1}, - {"i": 38, "op": "add", "dst": 3, "src": 1, "src2": 7, "imm": "0x75ba2fad", "imm2": "0x230c005c", "rot": 4, "bit": 27, "mask": 1, "width": 1}, - {"i": 39, "op": "shfl", "dst": 1, "src": 5, "src2": 3, "imm": "0xdbf37e75", "imm2": "0xb5ac1969", "rot": 30, "bit": 13, "mask": 4, "width": 1}, - {"i": 40, "op": "xor", "dst": 2, "src": 5, "src2": 5, "imm": "0x47f136c5", "imm2": "0x06ce9153", "rot": 19, "bit": 10, "mask": 2, "width": 1}, - {"i": 41, "op": "mad", "dst": 3, "src": 6, "src2": 3, "imm": "0xce13eff8", "imm2": "0x04cc1d55", "rot": 3, "bit": 1, "mask": 4, "width": 1}, - {"i": 42, "op": "sub", "dst": 6, "src": 7, "src2": 1, "imm": "0x6a65ab71", "imm2": "0x8fbc1bcd", "rot": 4, "bit": 1, "mask": 8, "width": 1}, - {"i": 43, "op": "xor", "dst": 7, "src": 0, "src2": 7, "imm": "0xdaeb4928", "imm2": "0xc0423027", "rot": 24, "bit": 11, "mask": 8, "width": 1}, - {"i": 44, "op": "load", "dst": 1, "src": 7, "src2": 7, "imm": "0x778f01c9", "imm2": "0x28cedcea", "rot": 12, "bit": 4, "mask": 16, "width": 1}, - {"i": 45, "op": "mul", "dst": 2, "src": 3, "src2": 2, "imm": "0xf4264f1b", "imm2": "0x0f627d56", "rot": 5, "bit": 28, "mask": 8, "width": 1}, - {"i": 46, "op": "mulhi", "dst": 1, "src": 5, "src2": 1, "imm": "0xffb2147a", "imm2": "0xccde9b05", "rot": 13, "bit": 9, "mask": 2, "width": 1}, - {"i": 47, "op": "sub", "dst": 4, "src": 3, "src2": 5, "imm": "0x73b36234", "imm2": "0x3f5d5997", "rot": 7, "bit": 18, "mask": 2, "width": 1}, - {"i": 48, "op": "rotr", "dst": 2, "src": 6, "src2": 3, "imm": "0x3a4d9aa9", "imm2": "0x212bec7b", "rot": 4, "bit": 29, "mask": 16, "width": 1}, - {"i": 49, "op": "load", "dst": 3, "src": 5, "src2": 2, "imm": "0x626f11df", "imm2": "0x56cd5bfd", "rot": 7, "bit": 1, "mask": 1, "width": 1}, - {"i": 50, "op": "add", "dst": 1, "src": 5, "src2": 6, "imm": "0x81b8bc2c", "imm2": "0x1907970c", "rot": 28, "bit": 7, "mask": 4, "width": 1}, - {"i": 51, "op": "mul", "dst": 0, "src": 2, "src2": 2, "imm": "0xa8848b30", "imm2": "0xef6ac348", "rot": 9, "bit": 15, "mask": 8, "width": 1}, - {"i": 52, "op": "add", "dst": 0, "src": 2, "src2": 0, "imm": "0x4f92b968", "imm2": "0x699fd448", "rot": 22, "bit": 6, "mask": 4, "width": 1}, - {"i": 53, "op": "add", "dst": 1, "src": 0, "src2": 2, "imm": "0x2bb965af", "imm2": "0x77b1520d", "rot": 2, "bit": 12, "mask": 8, "width": 1}, - {"i": 54, "op": "rotl", "dst": 7, "src": 1, "src2": 0, "imm": "0x553e678b", "imm2": "0x3cc8eae0", "rot": 14, "bit": 20, "mask": 2, "width": 1}, - {"i": 55, "op": "add", "dst": 3, "src": 7, "src2": 2, "imm": "0x7b0fe07a", "imm2": "0xa54c55a0", "rot": 10, "bit": 1, "mask": 1, "width": 1}, - {"i": 56, "op": "load", "dst": 6, "src": 7, "src2": 4, "imm": "0x01eba9aa", "imm2": "0x2758c0f7", "rot": 14, "bit": 15, "mask": 4, "width": 1}, - {"i": 57, "op": "rotr", "dst": 1, "src": 5, "src2": 2, "imm": "0x1f5267b3", "imm2": "0x236f5a27", "rot": 2, "bit": 31, "mask": 16, "width": 1}, - {"i": 58, "op": "load", "dst": 5, "src": 4, "src2": 3, "imm": "0xa9954a9b", "imm2": "0x6a54d4e8", "rot": 11, "bit": 10, "mask": 16, "width": 1}, - {"i": 59, "op": "load", "dst": 6, "src": 2, "src2": 4, "imm": "0x9923ff88", "imm2": "0x9357254e", "rot": 16, "bit": 1, "mask": 16, "width": 1}, - {"i": 60, "op": "mad", "dst": 3, "src": 5, "src2": 0, "imm": "0xc0cc51a6", "imm2": "0x3fd7701b", "rot": 20, "bit": 1, "mask": 4, "width": 1}, - {"i": 61, "op": "add", "dst": 5, "src": 7, "src2": 7, "imm": "0xaf9dd72d", "imm2": "0xad7493e7", "rot": 7, "bit": 31, "mask": 16, "width": 1}, - {"i": 62, "op": "add", "dst": 4, "src": 6, "src2": 2, "imm": "0x89841d87", "imm2": "0x1e07c3d9", "rot": 6, "bit": 27, "mask": 1, "width": 1}, - {"i": 63, "op": "rotl", "dst": 5, "src": 4, "src2": 2, "imm": "0xae210f8d", "imm2": "0x8e499ba4", "rot": 19, "bit": 9, "mask": 1, "width": 1} - ] -} diff --git a/proto-cuda/packs-ca4/mx8_mm1430/program.metal b/proto-cuda/packs-ca4/mx8_mm1430/program.metal deleted file mode 100644 index 4767a6cd2..000000000 --- a/proto-cuda/packs-ca4/mx8_mm1430/program.metal +++ /dev/null @@ -1,1559 +0,0 @@ -#include -using namespace metal; -// int8 tile reference (Counter ASIC 4.0 research): A = the 32 lanes' av as 8 x 16 bytes (lane l holds row l >> 2, bytes 4 (l & 3)..), -// B = bv as 16 x 8 (lane l holds column l >> 2); lane l gets C[l >> 2][2 (l & 3)] and C[l >> 2][2 (l & 3) + 1]; the PTX mma.m8n8k16 u8 layout. -static inline void mm8_ref(uint av, uint bv, uint lane, thread uint& d0, thread uint& d1) { - uint row = lane >> 2, col0 = 2u * (lane & 3u); - uint acc0 = 0u, acc1 = 0u; - for (uint j = 0u; j < 4u; ++j) { - uint aw = simd_shuffle(av, (ushort)(4u * row + j)); - uint bw0 = simd_shuffle(bv, (ushort)(4u * col0 + j)); - uint bw1 = simd_shuffle(bv, (ushort)(4u * (col0 + 1u) + j)); - for (uint t = 0u; t < 4u; ++t) { - uint ab = (aw >> (8u * t)) & 0xffu; - acc0 += ab * ((bw0 >> (8u * t)) & 0xffu); - acc1 += ab * ((bw1 >> (8u * t)) & 0xffu); - } - } - d0 = acc0; d1 = acc1; -} - - -#define MASK 0x0fffffffu -constant uint SEEDW[8] = { 0x67a9a7beu, 0x1a155b25u, 0xfddfb732u, 0x4b5af2e8u, 0xc55caf33u, 0xa27c13b7u, 0x06628a48u, 0x03852469u }; - -inline uint splitmix32(uint x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -inline uint rotl_imm(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 -inline uint rotr_var(uint x, uint n) { n &= 31u; return (x >> n) | (x << ((32u - n) & 31u)); } -inline uint ds_elem(uint i, uint d0, uint d1) { - uint x = i ^ d0; - x *= 0x9E3779B1u; x ^= x >> 15; - x += d1; - x *= 0x85EBCA77u; x ^= x >> 13; - x *= 0xC2B2AE3Du; x ^= x >> 16; - return x; -} - -kernel void igneum_hash(device const uint* dataset [[buffer(0)]], - device ulong* out [[buffer(1)]], - constant uint& baseNonce [[buffer(2)]], - uint gid [[thread_position_in_grid]]) { - uint nonce = baseNonce + gid; - uint r0, r1, r2, r3, r4, r5, r6, r7; - uint lane = gid & 31u; - { uint x = nonce ^ SEEDW[0]; x += 0x9e3779b9u * 1u; x = splitmix32(x); r0 = x ^ SEEDW[1]; } - { uint x = nonce ^ SEEDW[1]; x += 0x9e3779b9u * 2u; x = splitmix32(x); r1 = x ^ SEEDW[2]; } - { uint x = nonce ^ SEEDW[2]; x += 0x9e3779b9u * 3u; x = splitmix32(x); r2 = x ^ SEEDW[3]; } - { uint x = nonce ^ SEEDW[3]; x += 0x9e3779b9u * 4u; x = splitmix32(x); r3 = x ^ SEEDW[4]; } - { uint x = nonce ^ SEEDW[4]; x += 0x9e3779b9u * 5u; x = splitmix32(x); r4 = x ^ SEEDW[5]; } - { uint x = nonce ^ SEEDW[5]; x += 0x9e3779b9u * 6u; x = splitmix32(x); r5 = x ^ SEEDW[6]; } - { uint x = nonce ^ SEEDW[6]; x += 0x9e3779b9u * 7u; x = splitmix32(x); r6 = x ^ SEEDW[7]; } - { uint x = nonce ^ SEEDW[7]; x += 0x9e3779b9u * 8u; x = splitmix32(x); r7 = x ^ SEEDW[0]; } - - for (uint it = 0u; it < 8u; ++it) { - uint sel = r0; - r2 = r3 * r4 + r2; // 0 - r2 = r1 * r1 + r2; // 1 - r2 = r3 * r2 + r2; // 2 - r3 = r3 ^ r5; // 3 - r7 = r7 ^ dataset[r2 & MASK]; // 4 - r5 = r5 ^ dataset[r7 & MASK]; // 5 - r1 = r1 ^ simd_shuffle_xor(r4, (ushort)8); // 6 - r7 = r7 ^ simd_shuffle_xor(r3, (ushort)8); // 7 - r1 = mulhi(r1, r5); // 8 - r6 = rotr_var(r6, r3); // 9 - r3 = r3 | r4; // 10 - r4 = r4 ^ dataset[r3 & MASK]; // 11 - r0 = mulhi(r0, r4); // 12 - r5 = r5 + r1 + select(0xc7934706u, 0xd3177981u, ((sel >> 30u) & 1u) != 0u); // 13 - r0 = r0 ^ dataset[r4 & MASK]; // 14 - r2 = r2 - r4; // 15 - r2 = r2 ^ dataset[r0 & MASK]; // 16 - r7 = r7 ^ dataset[r2 & MASK]; // 17 - r7 = r7 ^ simd_shuffle_xor(r3, (ushort)4); // 18 - r5 = r5 * r0; // 19 - r3 = r3 ^ simd_shuffle_xor(r4, (ushort)2); // 20 - r2 = r2 ^ simd_shuffle_xor(r4, (ushort)16); // 21 - r6 = mulhi(r6, r2); // 22 - r6 = r6 ^ dataset[r1 & MASK]; // 23 - r5 = r5 * r0; // 24 - r5 = rotl_imm(r5, 19u); // 25 - r7 = r7 ^ simd_shuffle_xor(r6, (ushort)2); // 26 - r0 = r0 ^ r5; // 27 - r0 = r0 ^ r4; // 28 - r3 = r3 - r0; // 29 - r5 = r5 * r1; // 30 - r7 = r7 ^ dataset[r2 & MASK]; // 31 - r1 = r1 ^ dataset[r0 & MASK]; // 32 - r5 = r5 ^ r6; // 33 - r5 = r5 ^ dataset[r1 & MASK]; // 34 - r0 = mulhi(r0, r5); // 35 - r5 = r5 ^ simd_shuffle_xor(r2, (ushort)4); // 36 - r7 = r7 ^ dataset[r0 & MASK]; // 37 - r3 = r3 + r1 + select(0x75ba2fadu, 0x230c005cu, ((sel >> 27u) & 1u) != 0u); // 38 - r1 = r1 ^ simd_shuffle_xor(r5, (ushort)4); // 39 - r2 = r2 ^ r5; // 40 - r3 = r6 * r3 + r3; // 41 - r6 = r6 - r7; // 42 - r7 = r7 ^ r0; // 43 - r1 = r1 ^ dataset[r7 & MASK]; // 44 - r2 = r2 * r3; // 45 - r1 = mulhi(r1, r5); // 46 - r4 = r4 - r3; // 47 - r2 = rotr_var(r2, r6); // 48 - r3 = r3 ^ dataset[r5 & MASK]; // 49 - r1 = r1 + r5 + select(0x81b8bc2cu, 0x1907970cu, ((sel >> 7u) & 1u) != 0u); // 50 - r0 = r0 * r2; // 51 - r0 = r0 + r2 + select(0x4f92b968u, 0x699fd448u, ((sel >> 6u) & 1u) != 0u); // 52 - r1 = r1 + r0 + select(0x2bb965afu, 0x77b1520du, ((sel >> 12u) & 1u) != 0u); // 53 - r7 = rotl_imm(r7, 14u); // 54 - r3 = r3 + r7 + select(0x7b0fe07au, 0xa54c55a0u, ((sel >> 1u) & 1u) != 0u); // 55 - r6 = r6 ^ dataset[r7 & MASK]; // 56 - r1 = rotr_var(r1, r5); // 57 - r5 = r5 ^ dataset[r4 & MASK]; // 58 - r6 = r6 ^ dataset[r2 & MASK]; // 59 - r3 = r5 * r0 + r3; // 60 - r5 = r5 + r7 + select(0xaf9dd72du, 0xad7493e7u, ((sel >> 31u) & 1u) != 0u); // 61 - r4 = r4 + r6 + select(0x89841d87u, 0x1e07c3d9u, ((sel >> 27u) & 1u) != 0u); // 62 - r5 = rotl_imm(r5, 19u); // 63 - // int8 tile block (Counter ASIC 4.0 research, experimental): 1430 mma.m8n8k16 u8 tiles per iteration, both outputs consumed - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t0 mm8 a=r6 b=r5 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t1 mm8 a=r1 b=r1 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t2 mm8 a=r2 b=r5 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t3 mm8 a=r1 b=r5 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t4 mm8 a=r2 b=r0 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t5 mm8 a=r2 b=r6 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t6 mm8 a=r1 b=r4 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t7 mm8 a=r4 b=r6 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t8 mm8 a=r1 b=r1 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t9 mm8 a=r3 b=r7 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t10 mm8 a=r3 b=r0 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t11 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t12 mm8 a=r3 b=r1 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t13 mm8 a=r1 b=r5 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t14 mm8 a=r3 b=r3 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t15 mm8 a=r1 b=r0 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t16 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t17 mm8 a=r0 b=r5 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t18 mm8 a=r1 b=r1 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t19 mm8 a=r3 b=r3 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t20 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t21 mm8 a=r0 b=r3 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t22 mm8 a=r1 b=r7 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t23 mm8 a=r1 b=r2 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t24 mm8 a=r3 b=r4 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t25 mm8 a=r5 b=r0 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t26 mm8 a=r1 b=r0 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t27 mm8 a=r3 b=r2 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t28 mm8 a=r2 b=r4 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t29 mm8 a=r3 b=r4 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t30 mm8 a=r6 b=r5 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t31 mm8 a=r1 b=r7 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t32 mm8 a=r7 b=r6 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t33 mm8 a=r7 b=r1 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t34 mm8 a=r1 b=r0 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t35 mm8 a=r2 b=r4 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t36 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t37 mm8 a=r1 b=r7 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t38 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t39 mm8 a=r6 b=r2 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t40 mm8 a=r6 b=r1 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t41 mm8 a=r6 b=r0 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t42 mm8 a=r7 b=r4 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t43 mm8 a=r6 b=r1 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t44 mm8 a=r0 b=r6 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t45 mm8 a=r6 b=r5 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t46 mm8 a=r6 b=r6 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t47 mm8 a=r7 b=r3 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t48 mm8 a=r7 b=r7 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t49 mm8 a=r4 b=r5 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t50 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t51 mm8 a=r3 b=r4 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t52 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t53 mm8 a=r7 b=r7 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t54 mm8 a=r7 b=r6 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t55 mm8 a=r1 b=r7 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t56 mm8 a=r4 b=r0 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t57 mm8 a=r4 b=r3 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t58 mm8 a=r2 b=r3 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t59 mm8 a=r6 b=r7 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t60 mm8 a=r6 b=r0 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t61 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t62 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t63 mm8 a=r1 b=r6 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t64 mm8 a=r0 b=r5 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t65 mm8 a=r0 b=r1 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t66 mm8 a=r3 b=r5 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t67 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t68 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t69 mm8 a=r6 b=r7 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t70 mm8 a=r1 b=r7 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t71 mm8 a=r3 b=r0 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t72 mm8 a=r4 b=r0 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t73 mm8 a=r4 b=r1 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t74 mm8 a=r5 b=r1 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t75 mm8 a=r4 b=r4 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t76 mm8 a=r5 b=r6 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t77 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t78 mm8 a=r2 b=r5 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t79 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t80 mm8 a=r6 b=r7 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t81 mm8 a=r4 b=r3 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t82 mm8 a=r6 b=r0 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t83 mm8 a=r2 b=r2 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t84 mm8 a=r0 b=r6 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t85 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t86 mm8 a=r3 b=r4 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t87 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t88 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t89 mm8 a=r4 b=r2 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t90 mm8 a=r6 b=r5 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t91 mm8 a=r6 b=r5 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t92 mm8 a=r2 b=r6 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t93 mm8 a=r2 b=r2 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t94 mm8 a=r2 b=r4 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t95 mm8 a=r5 b=r4 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t96 mm8 a=r0 b=r6 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t97 mm8 a=r0 b=r5 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t98 mm8 a=r2 b=r7 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t99 mm8 a=r1 b=r4 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t100 mm8 a=r2 b=r4 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t101 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t102 mm8 a=r5 b=r7 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t103 mm8 a=r4 b=r7 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t104 mm8 a=r2 b=r0 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t105 mm8 a=r6 b=r1 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t106 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t107 mm8 a=r2 b=r1 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t108 mm8 a=r4 b=r6 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t109 mm8 a=r2 b=r7 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t110 mm8 a=r1 b=r0 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t111 mm8 a=r6 b=r5 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t112 mm8 a=r5 b=r7 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t113 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t114 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t115 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t116 mm8 a=r7 b=r2 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t117 mm8 a=r7 b=r6 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t118 mm8 a=r0 b=r1 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t119 mm8 a=r4 b=r3 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t120 mm8 a=r5 b=r3 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t121 mm8 a=r2 b=r7 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t122 mm8 a=r0 b=r1 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t123 mm8 a=r5 b=r2 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t124 mm8 a=r5 b=r1 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t125 mm8 a=r7 b=r2 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t126 mm8 a=r6 b=r3 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t127 mm8 a=r6 b=r7 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t128 mm8 a=r6 b=r7 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t129 mm8 a=r3 b=r2 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t130 mm8 a=r0 b=r4 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t131 mm8 a=r3 b=r5 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t132 mm8 a=r0 b=r7 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t133 mm8 a=r2 b=r6 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t134 mm8 a=r5 b=r6 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t135 mm8 a=r0 b=r2 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t136 mm8 a=r3 b=r1 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t137 mm8 a=r7 b=r0 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t138 mm8 a=r2 b=r2 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t139 mm8 a=r0 b=r4 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t140 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t141 mm8 a=r5 b=r5 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t142 mm8 a=r2 b=r2 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t143 mm8 a=r4 b=r3 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t144 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t145 mm8 a=r7 b=r2 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t146 mm8 a=r2 b=r7 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t147 mm8 a=r3 b=r1 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t148 mm8 a=r2 b=r1 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t149 mm8 a=r4 b=r6 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t150 mm8 a=r3 b=r6 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t151 mm8 a=r1 b=r4 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t152 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t153 mm8 a=r0 b=r3 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t154 mm8 a=r7 b=r3 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t155 mm8 a=r1 b=r3 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t156 mm8 a=r6 b=r3 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t157 mm8 a=r6 b=r4 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t158 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t159 mm8 a=r1 b=r0 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t160 mm8 a=r0 b=r1 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t161 mm8 a=r4 b=r0 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t162 mm8 a=r6 b=r6 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t163 mm8 a=r5 b=r3 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t164 mm8 a=r0 b=r7 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t165 mm8 a=r6 b=r5 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t166 mm8 a=r6 b=r5 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t167 mm8 a=r4 b=r5 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t168 mm8 a=r3 b=r6 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t169 mm8 a=r6 b=r2 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t170 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t171 mm8 a=r6 b=r3 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t172 mm8 a=r7 b=r3 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t173 mm8 a=r0 b=r0 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t174 mm8 a=r6 b=r6 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t175 mm8 a=r1 b=r6 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t176 mm8 a=r2 b=r4 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t177 mm8 a=r1 b=r7 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t178 mm8 a=r4 b=r2 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t179 mm8 a=r5 b=r2 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t180 mm8 a=r0 b=r4 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t181 mm8 a=r3 b=r5 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t182 mm8 a=r1 b=r1 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t183 mm8 a=r3 b=r1 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t184 mm8 a=r7 b=r4 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t185 mm8 a=r5 b=r0 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t186 mm8 a=r2 b=r5 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t187 mm8 a=r3 b=r0 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t188 mm8 a=r1 b=r4 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t189 mm8 a=r7 b=r2 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t190 mm8 a=r1 b=r5 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t191 mm8 a=r2 b=r1 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t192 mm8 a=r5 b=r0 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t193 mm8 a=r2 b=r1 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t194 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t195 mm8 a=r2 b=r7 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t196 mm8 a=r5 b=r6 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t197 mm8 a=r5 b=r3 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t198 mm8 a=r1 b=r0 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t199 mm8 a=r2 b=r5 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t200 mm8 a=r5 b=r7 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t201 mm8 a=r7 b=r2 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t202 mm8 a=r2 b=r5 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t203 mm8 a=r0 b=r5 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t204 mm8 a=r4 b=r7 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t205 mm8 a=r3 b=r7 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t206 mm8 a=r0 b=r6 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t207 mm8 a=r7 b=r7 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t208 mm8 a=r0 b=r1 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t209 mm8 a=r4 b=r4 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t210 mm8 a=r7 b=r1 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t211 mm8 a=r1 b=r5 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t212 mm8 a=r2 b=r4 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t213 mm8 a=r2 b=r2 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t214 mm8 a=r4 b=r6 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t215 mm8 a=r7 b=r3 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t216 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t217 mm8 a=r5 b=r7 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t218 mm8 a=r7 b=r1 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t219 mm8 a=r2 b=r1 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t220 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t221 mm8 a=r7 b=r1 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t222 mm8 a=r0 b=r6 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t223 mm8 a=r2 b=r2 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t224 mm8 a=r1 b=r1 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t225 mm8 a=r2 b=r7 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t226 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t227 mm8 a=r3 b=r5 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t228 mm8 a=r7 b=r1 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t229 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t230 mm8 a=r0 b=r3 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t231 mm8 a=r1 b=r3 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t232 mm8 a=r5 b=r4 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t233 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t234 mm8 a=r2 b=r0 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t235 mm8 a=r3 b=r4 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t236 mm8 a=r1 b=r7 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t237 mm8 a=r3 b=r5 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t238 mm8 a=r2 b=r0 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t239 mm8 a=r5 b=r7 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t240 mm8 a=r2 b=r7 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t241 mm8 a=r0 b=r1 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t242 mm8 a=r0 b=r2 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t243 mm8 a=r4 b=r4 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t244 mm8 a=r7 b=r6 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t245 mm8 a=r4 b=r4 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t246 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t247 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t248 mm8 a=r7 b=r5 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t249 mm8 a=r4 b=r4 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t250 mm8 a=r0 b=r6 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t251 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t252 mm8 a=r0 b=r2 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t253 mm8 a=r5 b=r5 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t254 mm8 a=r6 b=r7 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t255 mm8 a=r5 b=r0 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t256 mm8 a=r2 b=r1 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t257 mm8 a=r4 b=r3 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t258 mm8 a=r4 b=r1 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t259 mm8 a=r5 b=r5 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t260 mm8 a=r3 b=r2 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t261 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t262 mm8 a=r6 b=r4 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t263 mm8 a=r6 b=r4 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t264 mm8 a=r7 b=r2 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t265 mm8 a=r6 b=r1 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t266 mm8 a=r0 b=r0 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t267 mm8 a=r6 b=r3 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t268 mm8 a=r1 b=r0 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t269 mm8 a=r7 b=r7 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t270 mm8 a=r6 b=r2 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t271 mm8 a=r0 b=r3 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t272 mm8 a=r6 b=r0 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t273 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t274 mm8 a=r1 b=r0 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t275 mm8 a=r1 b=r3 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t276 mm8 a=r1 b=r5 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t277 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t278 mm8 a=r0 b=r0 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t279 mm8 a=r2 b=r4 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t280 mm8 a=r7 b=r2 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t281 mm8 a=r1 b=r0 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t282 mm8 a=r0 b=r1 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t283 mm8 a=r0 b=r2 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t284 mm8 a=r1 b=r5 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t285 mm8 a=r4 b=r7 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t286 mm8 a=r1 b=r0 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t287 mm8 a=r7 b=r1 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t288 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t289 mm8 a=r0 b=r5 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t290 mm8 a=r3 b=r2 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t291 mm8 a=r7 b=r4 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t292 mm8 a=r6 b=r7 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t293 mm8 a=r5 b=r5 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t294 mm8 a=r5 b=r7 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t295 mm8 a=r0 b=r7 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t296 mm8 a=r6 b=r1 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t297 mm8 a=r4 b=r5 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t298 mm8 a=r6 b=r0 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t299 mm8 a=r6 b=r0 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t300 mm8 a=r6 b=r4 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t301 mm8 a=r6 b=r4 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t302 mm8 a=r7 b=r6 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t303 mm8 a=r3 b=r2 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t304 mm8 a=r6 b=r5 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t305 mm8 a=r4 b=r5 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t306 mm8 a=r3 b=r1 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t307 mm8 a=r4 b=r3 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t308 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t309 mm8 a=r2 b=r1 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t310 mm8 a=r6 b=r1 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t311 mm8 a=r4 b=r6 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t312 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t313 mm8 a=r2 b=r6 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t314 mm8 a=r0 b=r3 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t315 mm8 a=r7 b=r1 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t316 mm8 a=r2 b=r4 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t317 mm8 a=r1 b=r6 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t318 mm8 a=r2 b=r1 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t319 mm8 a=r0 b=r2 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t320 mm8 a=r5 b=r2 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t321 mm8 a=r7 b=r4 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t322 mm8 a=r2 b=r4 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t323 mm8 a=r4 b=r3 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t324 mm8 a=r5 b=r7 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t325 mm8 a=r6 b=r2 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t326 mm8 a=r6 b=r1 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t327 mm8 a=r1 b=r5 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t328 mm8 a=r0 b=r1 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t329 mm8 a=r0 b=r3 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t330 mm8 a=r6 b=r7 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t331 mm8 a=r4 b=r4 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t332 mm8 a=r4 b=r4 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t333 mm8 a=r4 b=r6 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t334 mm8 a=r2 b=r2 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t335 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t336 mm8 a=r4 b=r0 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t337 mm8 a=r1 b=r4 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t338 mm8 a=r3 b=r7 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t339 mm8 a=r5 b=r1 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t340 mm8 a=r5 b=r0 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t341 mm8 a=r6 b=r6 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t342 mm8 a=r4 b=r7 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t343 mm8 a=r0 b=r4 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t344 mm8 a=r2 b=r5 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t345 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t346 mm8 a=r6 b=r0 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t347 mm8 a=r5 b=r3 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t348 mm8 a=r0 b=r4 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t349 mm8 a=r6 b=r1 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t350 mm8 a=r1 b=r5 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t351 mm8 a=r6 b=r4 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t352 mm8 a=r4 b=r1 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t353 mm8 a=r7 b=r3 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t354 mm8 a=r4 b=r1 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t355 mm8 a=r4 b=r5 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t356 mm8 a=r7 b=r6 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t357 mm8 a=r7 b=r3 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t358 mm8 a=r4 b=r7 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t359 mm8 a=r2 b=r7 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t360 mm8 a=r4 b=r4 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t361 mm8 a=r2 b=r2 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t362 mm8 a=r2 b=r7 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t363 mm8 a=r7 b=r6 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t364 mm8 a=r6 b=r4 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t365 mm8 a=r6 b=r6 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t366 mm8 a=r7 b=r7 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t367 mm8 a=r4 b=r1 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t368 mm8 a=r7 b=r0 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t369 mm8 a=r1 b=r0 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t370 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t371 mm8 a=r5 b=r4 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t372 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t373 mm8 a=r4 b=r6 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t374 mm8 a=r1 b=r5 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t375 mm8 a=r6 b=r5 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t376 mm8 a=r3 b=r6 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t377 mm8 a=r6 b=r4 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t378 mm8 a=r6 b=r3 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t379 mm8 a=r3 b=r0 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t380 mm8 a=r2 b=r0 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t381 mm8 a=r3 b=r5 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t382 mm8 a=r5 b=r2 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t383 mm8 a=r1 b=r2 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t384 mm8 a=r3 b=r1 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t385 mm8 a=r6 b=r0 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t386 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t387 mm8 a=r3 b=r7 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t388 mm8 a=r3 b=r0 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t389 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t390 mm8 a=r5 b=r4 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t391 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t392 mm8 a=r2 b=r3 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t393 mm8 a=r3 b=r6 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t394 mm8 a=r0 b=r7 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t395 mm8 a=r0 b=r7 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t396 mm8 a=r7 b=r7 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t397 mm8 a=r5 b=r7 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t398 mm8 a=r6 b=r5 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t399 mm8 a=r6 b=r3 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t400 mm8 a=r2 b=r7 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t401 mm8 a=r7 b=r6 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t402 mm8 a=r0 b=r0 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t403 mm8 a=r7 b=r4 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t404 mm8 a=r1 b=r1 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t405 mm8 a=r6 b=r6 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t406 mm8 a=r6 b=r1 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t407 mm8 a=r5 b=r7 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t408 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t409 mm8 a=r3 b=r4 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t410 mm8 a=r0 b=r5 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t411 mm8 a=r2 b=r0 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t412 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t413 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t414 mm8 a=r7 b=r1 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t415 mm8 a=r4 b=r7 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t416 mm8 a=r5 b=r4 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t417 mm8 a=r6 b=r6 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t418 mm8 a=r1 b=r3 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t419 mm8 a=r5 b=r0 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t420 mm8 a=r4 b=r2 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t421 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t422 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t423 mm8 a=r1 b=r7 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t424 mm8 a=r5 b=r1 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t425 mm8 a=r2 b=r7 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t426 mm8 a=r1 b=r6 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t427 mm8 a=r6 b=r6 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t428 mm8 a=r6 b=r2 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t429 mm8 a=r3 b=r3 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t430 mm8 a=r7 b=r7 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t431 mm8 a=r6 b=r4 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t432 mm8 a=r3 b=r2 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t433 mm8 a=r3 b=r0 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t434 mm8 a=r0 b=r7 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t435 mm8 a=r7 b=r5 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t436 mm8 a=r6 b=r4 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t437 mm8 a=r0 b=r7 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t438 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t439 mm8 a=r4 b=r7 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t440 mm8 a=r3 b=r5 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t441 mm8 a=r5 b=r1 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t442 mm8 a=r1 b=r2 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t443 mm8 a=r1 b=r7 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t444 mm8 a=r3 b=r7 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t445 mm8 a=r7 b=r5 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t446 mm8 a=r3 b=r1 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t447 mm8 a=r6 b=r7 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t448 mm8 a=r5 b=r2 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t449 mm8 a=r7 b=r1 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t450 mm8 a=r7 b=r5 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t451 mm8 a=r0 b=r3 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t452 mm8 a=r2 b=r4 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t453 mm8 a=r5 b=r7 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t454 mm8 a=r7 b=r7 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t455 mm8 a=r7 b=r1 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t456 mm8 a=r6 b=r5 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t457 mm8 a=r7 b=r7 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t458 mm8 a=r5 b=r1 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t459 mm8 a=r6 b=r7 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t460 mm8 a=r7 b=r7 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t461 mm8 a=r0 b=r6 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t462 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t463 mm8 a=r2 b=r0 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t464 mm8 a=r0 b=r3 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t465 mm8 a=r1 b=r7 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t466 mm8 a=r1 b=r2 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t467 mm8 a=r1 b=r0 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t468 mm8 a=r4 b=r1 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t469 mm8 a=r2 b=r0 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t470 mm8 a=r3 b=r1 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t471 mm8 a=r2 b=r2 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t472 mm8 a=r1 b=r7 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t473 mm8 a=r2 b=r7 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t474 mm8 a=r5 b=r3 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t475 mm8 a=r7 b=r6 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t476 mm8 a=r3 b=r0 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t477 mm8 a=r3 b=r1 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t478 mm8 a=r0 b=r6 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t479 mm8 a=r7 b=r6 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t480 mm8 a=r6 b=r0 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t481 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t482 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t483 mm8 a=r7 b=r0 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t484 mm8 a=r3 b=r7 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t485 mm8 a=r6 b=r4 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t486 mm8 a=r6 b=r5 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t487 mm8 a=r2 b=r5 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t488 mm8 a=r1 b=r2 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t489 mm8 a=r5 b=r2 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t490 mm8 a=r7 b=r1 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t491 mm8 a=r0 b=r0 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t492 mm8 a=r3 b=r1 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t493 mm8 a=r6 b=r1 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t494 mm8 a=r5 b=r6 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t495 mm8 a=r2 b=r2 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t496 mm8 a=r3 b=r5 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t497 mm8 a=r3 b=r2 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t498 mm8 a=r3 b=r1 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t499 mm8 a=r1 b=r6 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t500 mm8 a=r7 b=r6 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t501 mm8 a=r4 b=r4 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t502 mm8 a=r5 b=r7 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t503 mm8 a=r7 b=r5 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t504 mm8 a=r2 b=r0 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t505 mm8 a=r4 b=r0 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t506 mm8 a=r0 b=r2 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t507 mm8 a=r5 b=r6 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t508 mm8 a=r7 b=r3 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t509 mm8 a=r4 b=r4 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t510 mm8 a=r0 b=r6 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t511 mm8 a=r2 b=r4 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t512 mm8 a=r2 b=r7 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t513 mm8 a=r0 b=r3 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t514 mm8 a=r6 b=r7 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t515 mm8 a=r5 b=r2 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t516 mm8 a=r0 b=r2 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t517 mm8 a=r6 b=r7 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t518 mm8 a=r1 b=r4 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t519 mm8 a=r6 b=r0 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t520 mm8 a=r1 b=r2 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t521 mm8 a=r0 b=r4 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t522 mm8 a=r7 b=r2 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t523 mm8 a=r5 b=r3 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t524 mm8 a=r0 b=r4 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t525 mm8 a=r1 b=r5 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t526 mm8 a=r7 b=r1 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t527 mm8 a=r5 b=r1 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t528 mm8 a=r4 b=r1 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t529 mm8 a=r4 b=r2 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t530 mm8 a=r0 b=r2 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t531 mm8 a=r7 b=r0 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t532 mm8 a=r7 b=r2 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t533 mm8 a=r3 b=r2 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t534 mm8 a=r4 b=r3 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t535 mm8 a=r0 b=r6 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t536 mm8 a=r3 b=r1 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t537 mm8 a=r1 b=r6 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t538 mm8 a=r6 b=r6 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t539 mm8 a=r4 b=r4 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t540 mm8 a=r1 b=r4 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t541 mm8 a=r5 b=r3 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t542 mm8 a=r7 b=r7 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t543 mm8 a=r4 b=r2 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t544 mm8 a=r5 b=r7 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t545 mm8 a=r5 b=r2 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t546 mm8 a=r7 b=r1 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t547 mm8 a=r3 b=r5 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t548 mm8 a=r6 b=r3 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t549 mm8 a=r2 b=r2 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t550 mm8 a=r5 b=r5 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t551 mm8 a=r7 b=r6 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t552 mm8 a=r6 b=r7 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t553 mm8 a=r6 b=r3 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t554 mm8 a=r4 b=r2 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t555 mm8 a=r2 b=r7 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t556 mm8 a=r3 b=r6 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t557 mm8 a=r7 b=r7 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t558 mm8 a=r5 b=r0 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t559 mm8 a=r4 b=r1 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t560 mm8 a=r5 b=r4 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t561 mm8 a=r6 b=r1 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t562 mm8 a=r1 b=r2 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t563 mm8 a=r5 b=r3 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t564 mm8 a=r4 b=r3 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t565 mm8 a=r6 b=r0 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t566 mm8 a=r1 b=r6 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t567 mm8 a=r3 b=r6 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t568 mm8 a=r6 b=r5 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t569 mm8 a=r3 b=r7 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t570 mm8 a=r5 b=r1 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t571 mm8 a=r6 b=r0 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t572 mm8 a=r5 b=r0 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t573 mm8 a=r3 b=r0 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t574 mm8 a=r2 b=r2 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t575 mm8 a=r3 b=r1 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t576 mm8 a=r2 b=r3 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t577 mm8 a=r6 b=r3 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t578 mm8 a=r5 b=r6 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t579 mm8 a=r4 b=r7 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t580 mm8 a=r7 b=r1 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t581 mm8 a=r1 b=r6 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t582 mm8 a=r2 b=r1 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t583 mm8 a=r5 b=r0 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t584 mm8 a=r1 b=r7 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t585 mm8 a=r0 b=r4 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t586 mm8 a=r4 b=r4 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t587 mm8 a=r0 b=r4 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t588 mm8 a=r5 b=r5 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t589 mm8 a=r3 b=r6 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t590 mm8 a=r3 b=r2 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t591 mm8 a=r5 b=r1 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t592 mm8 a=r5 b=r3 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t593 mm8 a=r5 b=r3 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t594 mm8 a=r6 b=r7 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t595 mm8 a=r3 b=r0 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t596 mm8 a=r7 b=r6 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t597 mm8 a=r2 b=r4 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t598 mm8 a=r7 b=r7 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t599 mm8 a=r7 b=r2 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t600 mm8 a=r7 b=r7 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t601 mm8 a=r2 b=r7 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t602 mm8 a=r0 b=r1 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t603 mm8 a=r0 b=r0 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t604 mm8 a=r5 b=r5 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t605 mm8 a=r2 b=r0 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t606 mm8 a=r7 b=r5 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t607 mm8 a=r0 b=r2 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t608 mm8 a=r7 b=r6 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t609 mm8 a=r1 b=r5 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t610 mm8 a=r3 b=r3 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t611 mm8 a=r2 b=r2 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t612 mm8 a=r1 b=r1 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t613 mm8 a=r6 b=r3 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t614 mm8 a=r3 b=r4 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t615 mm8 a=r6 b=r2 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t616 mm8 a=r2 b=r3 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t617 mm8 a=r5 b=r4 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t618 mm8 a=r3 b=r6 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t619 mm8 a=r5 b=r0 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t620 mm8 a=r3 b=r5 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t621 mm8 a=r4 b=r2 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t622 mm8 a=r7 b=r0 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t623 mm8 a=r4 b=r6 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t624 mm8 a=r6 b=r2 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t625 mm8 a=r6 b=r2 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t626 mm8 a=r5 b=r2 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t627 mm8 a=r4 b=r0 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t628 mm8 a=r1 b=r5 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t629 mm8 a=r4 b=r5 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t630 mm8 a=r4 b=r5 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t631 mm8 a=r6 b=r4 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t632 mm8 a=r2 b=r0 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t633 mm8 a=r2 b=r1 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t634 mm8 a=r1 b=r1 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t635 mm8 a=r6 b=r7 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t636 mm8 a=r2 b=r5 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t637 mm8 a=r5 b=r7 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t638 mm8 a=r1 b=r4 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t639 mm8 a=r1 b=r7 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t640 mm8 a=r0 b=r4 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t641 mm8 a=r5 b=r6 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t642 mm8 a=r7 b=r5 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t643 mm8 a=r3 b=r5 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t644 mm8 a=r1 b=r6 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t645 mm8 a=r0 b=r7 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t646 mm8 a=r5 b=r6 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t647 mm8 a=r6 b=r4 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t648 mm8 a=r3 b=r1 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t649 mm8 a=r7 b=r2 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t650 mm8 a=r7 b=r7 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t651 mm8 a=r6 b=r3 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t652 mm8 a=r1 b=r6 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t653 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t654 mm8 a=r1 b=r6 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t655 mm8 a=r7 b=r5 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t656 mm8 a=r6 b=r5 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t657 mm8 a=r7 b=r4 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t658 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t659 mm8 a=r0 b=r7 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t660 mm8 a=r6 b=r1 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t661 mm8 a=r3 b=r7 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t662 mm8 a=r2 b=r3 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t663 mm8 a=r0 b=r3 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t664 mm8 a=r2 b=r0 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t665 mm8 a=r1 b=r1 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t666 mm8 a=r7 b=r1 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t667 mm8 a=r2 b=r0 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t668 mm8 a=r3 b=r1 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t669 mm8 a=r1 b=r4 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t670 mm8 a=r7 b=r7 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t671 mm8 a=r0 b=r3 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t672 mm8 a=r0 b=r1 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t673 mm8 a=r6 b=r2 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t674 mm8 a=r3 b=r3 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t675 mm8 a=r5 b=r6 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t676 mm8 a=r4 b=r3 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t677 mm8 a=r0 b=r1 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t678 mm8 a=r4 b=r7 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t679 mm8 a=r4 b=r7 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t680 mm8 a=r5 b=r1 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t681 mm8 a=r4 b=r2 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t682 mm8 a=r3 b=r6 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t683 mm8 a=r0 b=r5 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t684 mm8 a=r6 b=r2 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t685 mm8 a=r7 b=r3 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t686 mm8 a=r4 b=r1 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t687 mm8 a=r0 b=r5 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t688 mm8 a=r3 b=r6 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t689 mm8 a=r3 b=r1 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t690 mm8 a=r2 b=r6 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t691 mm8 a=r2 b=r6 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t692 mm8 a=r1 b=r4 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t693 mm8 a=r7 b=r1 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t694 mm8 a=r3 b=r5 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t695 mm8 a=r3 b=r2 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t696 mm8 a=r2 b=r4 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t697 mm8 a=r2 b=r3 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t698 mm8 a=r3 b=r0 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t699 mm8 a=r2 b=r1 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t700 mm8 a=r0 b=r1 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t701 mm8 a=r5 b=r7 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t702 mm8 a=r5 b=r5 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t703 mm8 a=r2 b=r2 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t704 mm8 a=r6 b=r5 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t705 mm8 a=r5 b=r4 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t706 mm8 a=r6 b=r0 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t707 mm8 a=r6 b=r4 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t708 mm8 a=r2 b=r0 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t709 mm8 a=r7 b=r0 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t710 mm8 a=r4 b=r2 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t711 mm8 a=r4 b=r7 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t712 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t713 mm8 a=r5 b=r2 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t714 mm8 a=r0 b=r0 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t715 mm8 a=r1 b=r7 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t716 mm8 a=r3 b=r3 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t717 mm8 a=r3 b=r3 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t718 mm8 a=r7 b=r5 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t719 mm8 a=r4 b=r7 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t720 mm8 a=r4 b=r7 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t721 mm8 a=r6 b=r4 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t722 mm8 a=r7 b=r5 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t723 mm8 a=r5 b=r0 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t724 mm8 a=r7 b=r4 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t725 mm8 a=r6 b=r1 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t726 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t727 mm8 a=r0 b=r6 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t728 mm8 a=r5 b=r7 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t729 mm8 a=r5 b=r6 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t730 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t731 mm8 a=r5 b=r6 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t732 mm8 a=r5 b=r1 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t733 mm8 a=r3 b=r4 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t734 mm8 a=r4 b=r3 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t735 mm8 a=r7 b=r4 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t736 mm8 a=r4 b=r2 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t737 mm8 a=r4 b=r7 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t738 mm8 a=r6 b=r2 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t739 mm8 a=r7 b=r5 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t740 mm8 a=r4 b=r3 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t741 mm8 a=r7 b=r0 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t742 mm8 a=r6 b=r5 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t743 mm8 a=r5 b=r3 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t744 mm8 a=r7 b=r4 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t745 mm8 a=r7 b=r0 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t746 mm8 a=r2 b=r5 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t747 mm8 a=r0 b=r3 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t748 mm8 a=r1 b=r6 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t749 mm8 a=r2 b=r5 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t750 mm8 a=r4 b=r3 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t751 mm8 a=r6 b=r6 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t752 mm8 a=r5 b=r0 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t753 mm8 a=r6 b=r7 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t754 mm8 a=r2 b=r4 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t755 mm8 a=r5 b=r6 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t756 mm8 a=r7 b=r3 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t757 mm8 a=r6 b=r6 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t758 mm8 a=r6 b=r2 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t759 mm8 a=r2 b=r7 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t760 mm8 a=r7 b=r7 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t761 mm8 a=r3 b=r3 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t762 mm8 a=r5 b=r2 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t763 mm8 a=r1 b=r1 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t764 mm8 a=r1 b=r2 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t765 mm8 a=r2 b=r4 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t766 mm8 a=r7 b=r6 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t767 mm8 a=r7 b=r5 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t768 mm8 a=r5 b=r6 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t769 mm8 a=r4 b=r5 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t770 mm8 a=r7 b=r5 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t771 mm8 a=r2 b=r5 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t772 mm8 a=r0 b=r4 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t773 mm8 a=r2 b=r5 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t774 mm8 a=r4 b=r0 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t775 mm8 a=r4 b=r6 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t776 mm8 a=r5 b=r7 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t777 mm8 a=r1 b=r1 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t778 mm8 a=r1 b=r2 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t779 mm8 a=r7 b=r5 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t780 mm8 a=r3 b=r7 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t781 mm8 a=r6 b=r7 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t782 mm8 a=r6 b=r4 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t783 mm8 a=r6 b=r5 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t784 mm8 a=r1 b=r5 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t785 mm8 a=r7 b=r2 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t786 mm8 a=r2 b=r4 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t787 mm8 a=r7 b=r2 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t788 mm8 a=r0 b=r0 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t789 mm8 a=r1 b=r4 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t790 mm8 a=r3 b=r1 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t791 mm8 a=r3 b=r5 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t792 mm8 a=r7 b=r1 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t793 mm8 a=r7 b=r2 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t794 mm8 a=r2 b=r4 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t795 mm8 a=r1 b=r7 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t796 mm8 a=r2 b=r4 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t797 mm8 a=r6 b=r2 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t798 mm8 a=r1 b=r3 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t799 mm8 a=r1 b=r3 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t800 mm8 a=r0 b=r4 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t801 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t802 mm8 a=r4 b=r7 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t803 mm8 a=r3 b=r2 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t804 mm8 a=r0 b=r4 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t805 mm8 a=r4 b=r6 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t806 mm8 a=r0 b=r5 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t807 mm8 a=r5 b=r0 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t808 mm8 a=r2 b=r4 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t809 mm8 a=r1 b=r6 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t810 mm8 a=r1 b=r2 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t811 mm8 a=r2 b=r7 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t812 mm8 a=r0 b=r6 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t813 mm8 a=r2 b=r2 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t814 mm8 a=r3 b=r4 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t815 mm8 a=r4 b=r5 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t816 mm8 a=r3 b=r0 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t817 mm8 a=r3 b=r4 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t818 mm8 a=r2 b=r2 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t819 mm8 a=r1 b=r2 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t820 mm8 a=r3 b=r6 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t821 mm8 a=r7 b=r3 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t822 mm8 a=r1 b=r5 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t823 mm8 a=r1 b=r4 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t824 mm8 a=r7 b=r7 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t825 mm8 a=r1 b=r1 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t826 mm8 a=r0 b=r2 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t827 mm8 a=r3 b=r1 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t828 mm8 a=r7 b=r4 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t829 mm8 a=r5 b=r2 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t830 mm8 a=r6 b=r3 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t831 mm8 a=r2 b=r7 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t832 mm8 a=r1 b=r6 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t833 mm8 a=r0 b=r6 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t834 mm8 a=r2 b=r4 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t835 mm8 a=r0 b=r7 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t836 mm8 a=r3 b=r0 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t837 mm8 a=r4 b=r7 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t838 mm8 a=r5 b=r4 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t839 mm8 a=r0 b=r6 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t840 mm8 a=r1 b=r6 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t841 mm8 a=r4 b=r0 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t842 mm8 a=r3 b=r5 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t843 mm8 a=r2 b=r2 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t844 mm8 a=r3 b=r3 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t845 mm8 a=r1 b=r7 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t846 mm8 a=r1 b=r4 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t847 mm8 a=r4 b=r0 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t848 mm8 a=r6 b=r7 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t849 mm8 a=r6 b=r6 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t850 mm8 a=r6 b=r0 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t851 mm8 a=r0 b=r5 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t852 mm8 a=r2 b=r2 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t853 mm8 a=r6 b=r5 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t854 mm8 a=r1 b=r4 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t855 mm8 a=r1 b=r7 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t856 mm8 a=r2 b=r3 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t857 mm8 a=r6 b=r5 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t858 mm8 a=r0 b=r5 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t859 mm8 a=r2 b=r6 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t860 mm8 a=r7 b=r2 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t861 mm8 a=r3 b=r3 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t862 mm8 a=r0 b=r1 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t863 mm8 a=r0 b=r0 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t864 mm8 a=r5 b=r5 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t865 mm8 a=r3 b=r7 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t866 mm8 a=r6 b=r6 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t867 mm8 a=r7 b=r3 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t868 mm8 a=r4 b=r4 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t869 mm8 a=r3 b=r6 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t870 mm8 a=r0 b=r1 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t871 mm8 a=r5 b=r1 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t872 mm8 a=r0 b=r6 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t873 mm8 a=r2 b=r1 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t874 mm8 a=r5 b=r1 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t875 mm8 a=r1 b=r7 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t876 mm8 a=r7 b=r6 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t877 mm8 a=r6 b=r2 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t878 mm8 a=r5 b=r0 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t879 mm8 a=r4 b=r1 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t880 mm8 a=r3 b=r3 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t881 mm8 a=r2 b=r7 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t882 mm8 a=r7 b=r5 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t883 mm8 a=r2 b=r3 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t884 mm8 a=r2 b=r7 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t885 mm8 a=r3 b=r5 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t886 mm8 a=r5 b=r4 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t887 mm8 a=r1 b=r1 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t888 mm8 a=r6 b=r2 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t889 mm8 a=r4 b=r6 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t890 mm8 a=r0 b=r5 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t891 mm8 a=r0 b=r7 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t892 mm8 a=r5 b=r6 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t893 mm8 a=r2 b=r1 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t894 mm8 a=r0 b=r7 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t895 mm8 a=r2 b=r4 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t896 mm8 a=r0 b=r2 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t897 mm8 a=r7 b=r4 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t898 mm8 a=r1 b=r3 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t899 mm8 a=r5 b=r1 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t900 mm8 a=r7 b=r7 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t901 mm8 a=r4 b=r6 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t902 mm8 a=r2 b=r1 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t903 mm8 a=r0 b=r3 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t904 mm8 a=r6 b=r7 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t905 mm8 a=r0 b=r4 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t906 mm8 a=r3 b=r7 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t907 mm8 a=r2 b=r4 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t908 mm8 a=r7 b=r1 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t909 mm8 a=r4 b=r5 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t910 mm8 a=r3 b=r7 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t911 mm8 a=r2 b=r6 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t912 mm8 a=r6 b=r1 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t913 mm8 a=r2 b=r3 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t914 mm8 a=r7 b=r4 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t915 mm8 a=r4 b=r3 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t916 mm8 a=r4 b=r2 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t917 mm8 a=r5 b=r5 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t918 mm8 a=r7 b=r2 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t919 mm8 a=r3 b=r4 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t920 mm8 a=r6 b=r5 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t921 mm8 a=r6 b=r5 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t922 mm8 a=r1 b=r0 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t923 mm8 a=r6 b=r3 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t924 mm8 a=r2 b=r0 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t925 mm8 a=r6 b=r6 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t926 mm8 a=r5 b=r0 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t927 mm8 a=r1 b=r1 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t928 mm8 a=r5 b=r7 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t929 mm8 a=r1 b=r0 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t930 mm8 a=r7 b=r7 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t931 mm8 a=r7 b=r0 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t932 mm8 a=r1 b=r3 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t933 mm8 a=r1 b=r3 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t934 mm8 a=r6 b=r0 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t935 mm8 a=r5 b=r0 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t936 mm8 a=r3 b=r3 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t937 mm8 a=r0 b=r6 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t938 mm8 a=r1 b=r5 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t939 mm8 a=r7 b=r0 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t940 mm8 a=r1 b=r2 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t941 mm8 a=r1 b=r6 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t942 mm8 a=r0 b=r6 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t943 mm8 a=r4 b=r0 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t944 mm8 a=r6 b=r5 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t945 mm8 a=r3 b=r7 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t946 mm8 a=r2 b=r6 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t947 mm8 a=r7 b=r2 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t948 mm8 a=r6 b=r4 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t949 mm8 a=r6 b=r6 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t950 mm8 a=r7 b=r6 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t951 mm8 a=r2 b=r7 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t952 mm8 a=r3 b=r2 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t953 mm8 a=r4 b=r4 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t954 mm8 a=r2 b=r4 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t955 mm8 a=r0 b=r7 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t956 mm8 a=r7 b=r1 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t957 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t958 mm8 a=r6 b=r5 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t959 mm8 a=r4 b=r7 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t960 mm8 a=r1 b=r1 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t961 mm8 a=r1 b=r5 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t962 mm8 a=r6 b=r4 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t963 mm8 a=r7 b=r1 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t964 mm8 a=r2 b=r5 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t965 mm8 a=r3 b=r7 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t966 mm8 a=r7 b=r3 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t967 mm8 a=r0 b=r0 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t968 mm8 a=r5 b=r7 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t969 mm8 a=r4 b=r4 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t970 mm8 a=r4 b=r3 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t971 mm8 a=r3 b=r4 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t972 mm8 a=r1 b=r7 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t973 mm8 a=r4 b=r1 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t974 mm8 a=r5 b=r6 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t975 mm8 a=r1 b=r2 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t976 mm8 a=r0 b=r6 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t977 mm8 a=r2 b=r0 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t978 mm8 a=r0 b=r2 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t979 mm8 a=r4 b=r0 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t980 mm8 a=r3 b=r1 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t981 mm8 a=r2 b=r0 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t982 mm8 a=r1 b=r6 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t983 mm8 a=r2 b=r1 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t984 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t985 mm8 a=r4 b=r3 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t986 mm8 a=r2 b=r7 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t987 mm8 a=r5 b=r1 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t988 mm8 a=r6 b=r0 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t989 mm8 a=r2 b=r3 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t990 mm8 a=r1 b=r5 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t991 mm8 a=r6 b=r0 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t992 mm8 a=r2 b=r1 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t993 mm8 a=r6 b=r0 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t994 mm8 a=r6 b=r0 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t995 mm8 a=r0 b=r3 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t996 mm8 a=r7 b=r0 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t997 mm8 a=r5 b=r5 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t998 mm8 a=r3 b=r4 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t999 mm8 a=r7 b=r2 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t1000 mm8 a=r5 b=r3 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t1001 mm8 a=r5 b=r6 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t1002 mm8 a=r6 b=r2 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t1003 mm8 a=r4 b=r2 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t1004 mm8 a=r7 b=r3 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t1005 mm8 a=r1 b=r7 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t1006 mm8 a=r4 b=r2 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t1007 mm8 a=r0 b=r2 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t1008 mm8 a=r3 b=r5 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t1009 mm8 a=r0 b=r7 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t1010 mm8 a=r6 b=r7 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t1011 mm8 a=r7 b=r5 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t1012 mm8 a=r3 b=r5 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t1013 mm8 a=r6 b=r6 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t1014 mm8 a=r7 b=r3 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t1015 mm8 a=r7 b=r6 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t1016 mm8 a=r4 b=r0 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t1017 mm8 a=r5 b=r2 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t1018 mm8 a=r7 b=r6 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t1019 mm8 a=r0 b=r7 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t1020 mm8 a=r5 b=r6 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t1021 mm8 a=r7 b=r6 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t1022 mm8 a=r3 b=r5 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t1023 mm8 a=r0 b=r3 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t1024 mm8 a=r2 b=r6 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t1025 mm8 a=r4 b=r7 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t1026 mm8 a=r0 b=r1 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t1027 mm8 a=r1 b=r5 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t1028 mm8 a=r7 b=r6 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t1029 mm8 a=r2 b=r1 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t1030 mm8 a=r1 b=r7 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t1031 mm8 a=r3 b=r7 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t1032 mm8 a=r2 b=r1 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t1033 mm8 a=r2 b=r7 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t1034 mm8 a=r4 b=r3 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t1035 mm8 a=r3 b=r0 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t1036 mm8 a=r3 b=r0 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t1037 mm8 a=r7 b=r1 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t1038 mm8 a=r2 b=r1 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t1039 mm8 a=r5 b=r1 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t1040 mm8 a=r0 b=r7 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t1041 mm8 a=r2 b=r2 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t1042 mm8 a=r6 b=r6 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t1043 mm8 a=r6 b=r7 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t1044 mm8 a=r4 b=r2 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t1045 mm8 a=r0 b=r0 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t1046 mm8 a=r7 b=r1 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t1047 mm8 a=r5 b=r7 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t1048 mm8 a=r2 b=r3 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t1049 mm8 a=r5 b=r1 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t1050 mm8 a=r0 b=r5 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t1051 mm8 a=r7 b=r7 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t1052 mm8 a=r2 b=r3 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t1053 mm8 a=r6 b=r0 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t1054 mm8 a=r4 b=r2 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t1055 mm8 a=r5 b=r3 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t1056 mm8 a=r5 b=r1 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t1057 mm8 a=r1 b=r5 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t1058 mm8 a=r0 b=r5 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t1059 mm8 a=r5 b=r6 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t1060 mm8 a=r7 b=r3 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t1061 mm8 a=r5 b=r6 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t1062 mm8 a=r3 b=r0 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t1063 mm8 a=r1 b=r1 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t1064 mm8 a=r5 b=r4 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t1065 mm8 a=r3 b=r1 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t1066 mm8 a=r7 b=r7 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t1067 mm8 a=r0 b=r5 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t1068 mm8 a=r4 b=r6 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t1069 mm8 a=r2 b=r0 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t1070 mm8 a=r2 b=r3 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t1071 mm8 a=r2 b=r6 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t1072 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t1073 mm8 a=r3 b=r6 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t1074 mm8 a=r0 b=r0 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t1075 mm8 a=r4 b=r4 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t1076 mm8 a=r1 b=r7 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t1077 mm8 a=r7 b=r7 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t1078 mm8 a=r3 b=r7 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t1079 mm8 a=r6 b=r6 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t1080 mm8 a=r4 b=r5 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t1081 mm8 a=r1 b=r5 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t1082 mm8 a=r5 b=r4 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t1083 mm8 a=r2 b=r2 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t1084 mm8 a=r2 b=r0 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t1085 mm8 a=r3 b=r3 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t1086 mm8 a=r3 b=r1 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t1087 mm8 a=r1 b=r3 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t1088 mm8 a=r3 b=r5 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t1089 mm8 a=r0 b=r0 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t1090 mm8 a=r4 b=r6 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t1091 mm8 a=r4 b=r3 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t1092 mm8 a=r2 b=r6 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t1093 mm8 a=r5 b=r7 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t1094 mm8 a=r7 b=r3 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t1095 mm8 a=r0 b=r2 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t1096 mm8 a=r3 b=r3 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t1097 mm8 a=r6 b=r2 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t1098 mm8 a=r3 b=r6 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t1099 mm8 a=r2 b=r5 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t1100 mm8 a=r1 b=r4 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t1101 mm8 a=r0 b=r6 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t1102 mm8 a=r0 b=r5 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t1103 mm8 a=r7 b=r0 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t1104 mm8 a=r4 b=r2 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t1105 mm8 a=r3 b=r4 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t1106 mm8 a=r2 b=r1 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t1107 mm8 a=r3 b=r0 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t1108 mm8 a=r3 b=r5 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t1109 mm8 a=r6 b=r2 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t1110 mm8 a=r2 b=r4 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t1111 mm8 a=r0 b=r7 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t1112 mm8 a=r1 b=r0 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t1113 mm8 a=r4 b=r6 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t1114 mm8 a=r0 b=r5 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t1115 mm8 a=r4 b=r5 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t1116 mm8 a=r4 b=r6 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t1117 mm8 a=r0 b=r6 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t1118 mm8 a=r7 b=r6 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t1119 mm8 a=r2 b=r7 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t1120 mm8 a=r7 b=r5 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t1121 mm8 a=r4 b=r0 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t1122 mm8 a=r0 b=r5 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t1123 mm8 a=r0 b=r6 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t1124 mm8 a=r0 b=r1 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t1125 mm8 a=r6 b=r6 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t1126 mm8 a=r0 b=r0 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t1127 mm8 a=r1 b=r1 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t1128 mm8 a=r4 b=r1 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t1129 mm8 a=r0 b=r0 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t1130 mm8 a=r0 b=r4 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t1131 mm8 a=r3 b=r0 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t1132 mm8 a=r0 b=r6 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t1133 mm8 a=r6 b=r0 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t1134 mm8 a=r7 b=r3 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t1135 mm8 a=r6 b=r7 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t1136 mm8 a=r7 b=r4 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t1137 mm8 a=r4 b=r0 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t1138 mm8 a=r6 b=r4 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t1139 mm8 a=r6 b=r2 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t1140 mm8 a=r1 b=r2 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t1141 mm8 a=r0 b=r5 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t1142 mm8 a=r4 b=r5 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t1143 mm8 a=r5 b=r6 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t1144 mm8 a=r6 b=r4 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t1145 mm8 a=r5 b=r1 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t1146 mm8 a=r6 b=r5 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t1147 mm8 a=r7 b=r1 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t1148 mm8 a=r1 b=r6 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t1149 mm8 a=r5 b=r5 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t1150 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t1151 mm8 a=r7 b=r5 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t1152 mm8 a=r0 b=r6 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t1153 mm8 a=r2 b=r5 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t1154 mm8 a=r5 b=r3 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t1155 mm8 a=r0 b=r6 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t1156 mm8 a=r2 b=r6 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t1157 mm8 a=r6 b=r7 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t1158 mm8 a=r2 b=r3 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t1159 mm8 a=r1 b=r5 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t1160 mm8 a=r5 b=r0 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t1161 mm8 a=r7 b=r0 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t1162 mm8 a=r5 b=r3 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t1163 mm8 a=r4 b=r3 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t1164 mm8 a=r3 b=r4 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t1165 mm8 a=r1 b=r2 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t1166 mm8 a=r4 b=r7 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t1167 mm8 a=r2 b=r7 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t1168 mm8 a=r2 b=r6 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t1169 mm8 a=r0 b=r4 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t1170 mm8 a=r2 b=r1 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t1171 mm8 a=r1 b=r7 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t1172 mm8 a=r0 b=r1 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t1173 mm8 a=r6 b=r4 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t1174 mm8 a=r0 b=r1 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t1175 mm8 a=r0 b=r0 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t1176 mm8 a=r4 b=r5 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t1177 mm8 a=r7 b=r2 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t1178 mm8 a=r1 b=r5 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t1179 mm8 a=r0 b=r7 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t1180 mm8 a=r7 b=r3 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t1181 mm8 a=r7 b=r3 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t1182 mm8 a=r1 b=r2 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t1183 mm8 a=r2 b=r4 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t1184 mm8 a=r7 b=r0 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t1185 mm8 a=r5 b=r2 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t1186 mm8 a=r1 b=r0 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t1187 mm8 a=r5 b=r4 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t1188 mm8 a=r3 b=r0 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t1189 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t1190 mm8 a=r5 b=r0 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t1191 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t1192 mm8 a=r0 b=r2 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t1193 mm8 a=r3 b=r7 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t1194 mm8 a=r7 b=r2 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t1195 mm8 a=r1 b=r4 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t1196 mm8 a=r4 b=r6 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t1197 mm8 a=r5 b=r6 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t1198 mm8 a=r0 b=r6 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t1199 mm8 a=r5 b=r1 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t1200 mm8 a=r1 b=r3 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t1201 mm8 a=r6 b=r6 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t1202 mm8 a=r2 b=r4 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t1203 mm8 a=r0 b=r5 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t1204 mm8 a=r0 b=r2 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t1205 mm8 a=r4 b=r6 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t1206 mm8 a=r4 b=r6 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t1207 mm8 a=r3 b=r7 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t1208 mm8 a=r6 b=r1 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t1209 mm8 a=r5 b=r5 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t1210 mm8 a=r0 b=r0 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t1211 mm8 a=r7 b=r5 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t1212 mm8 a=r2 b=r6 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t1213 mm8 a=r6 b=r2 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t1214 mm8 a=r4 b=r2 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t1215 mm8 a=r7 b=r2 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t1216 mm8 a=r3 b=r3 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t1217 mm8 a=r5 b=r5 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t1218 mm8 a=r0 b=r7 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t1219 mm8 a=r2 b=r4 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t1220 mm8 a=r1 b=r7 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t1221 mm8 a=r5 b=r5 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t1222 mm8 a=r1 b=r3 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t1223 mm8 a=r0 b=r2 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t1224 mm8 a=r1 b=r1 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t1225 mm8 a=r4 b=r1 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t1226 mm8 a=r5 b=r6 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t1227 mm8 a=r1 b=r7 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t1228 mm8 a=r4 b=r2 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t1229 mm8 a=r5 b=r3 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t1230 mm8 a=r6 b=r2 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t1231 mm8 a=r4 b=r3 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t1232 mm8 a=r6 b=r7 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t1233 mm8 a=r0 b=r5 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t1234 mm8 a=r5 b=r4 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t1235 mm8 a=r5 b=r7 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t1236 mm8 a=r0 b=r7 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t1237 mm8 a=r7 b=r2 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t1238 mm8 a=r7 b=r0 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t1239 mm8 a=r3 b=r2 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t1240 mm8 a=r0 b=r5 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t1241 mm8 a=r5 b=r0 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t1242 mm8 a=r5 b=r7 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t1243 mm8 a=r6 b=r3 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t1244 mm8 a=r1 b=r5 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t1245 mm8 a=r0 b=r6 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t1246 mm8 a=r5 b=r5 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t1247 mm8 a=r1 b=r6 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t1248 mm8 a=r4 b=r0 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t1249 mm8 a=r7 b=r6 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t1250 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t1251 mm8 a=r3 b=r7 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t1252 mm8 a=r5 b=r5 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t1253 mm8 a=r5 b=r5 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t1254 mm8 a=r0 b=r6 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t1255 mm8 a=r1 b=r4 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t1256 mm8 a=r1 b=r3 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t1257 mm8 a=r0 b=r7 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t1258 mm8 a=r5 b=r4 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t1259 mm8 a=r0 b=r3 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t1260 mm8 a=r1 b=r5 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t1261 mm8 a=r6 b=r7 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t1262 mm8 a=r2 b=r5 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t1263 mm8 a=r3 b=r7 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t1264 mm8 a=r1 b=r6 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t1265 mm8 a=r2 b=r3 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t1266 mm8 a=r4 b=r6 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t1267 mm8 a=r6 b=r6 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t1268 mm8 a=r4 b=r5 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t1269 mm8 a=r5 b=r7 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t1270 mm8 a=r7 b=r3 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t1271 mm8 a=r7 b=r0 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t1272 mm8 a=r1 b=r3 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t1273 mm8 a=r3 b=r2 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t1274 mm8 a=r2 b=r0 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t1275 mm8 a=r7 b=r3 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t1276 mm8 a=r4 b=r3 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t1277 mm8 a=r5 b=r4 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t1278 mm8 a=r6 b=r3 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t1279 mm8 a=r0 b=r1 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t1280 mm8 a=r4 b=r1 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t1281 mm8 a=r6 b=r3 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t1282 mm8 a=r1 b=r7 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t1283 mm8 a=r6 b=r2 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t1284 mm8 a=r1 b=r6 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t1285 mm8 a=r3 b=r5 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t1286 mm8 a=r3 b=r1 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t1287 mm8 a=r4 b=r1 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t1288 mm8 a=r3 b=r6 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t1289 mm8 a=r6 b=r2 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t1290 mm8 a=r6 b=r5 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t1291 mm8 a=r5 b=r6 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t1292 mm8 a=r6 b=r6 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t1293 mm8 a=r6 b=r2 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t1294 mm8 a=r0 b=r5 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t1295 mm8 a=r5 b=r0 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t1296 mm8 a=r1 b=r0 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t1297 mm8 a=r0 b=r0 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t1298 mm8 a=r2 b=r6 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t1299 mm8 a=r7 b=r0 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t1300 mm8 a=r4 b=r1 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t1301 mm8 a=r4 b=r5 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t1302 mm8 a=r0 b=r4 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t1303 mm8 a=r4 b=r2 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t1304 mm8 a=r4 b=r7 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t1305 mm8 a=r3 b=r2 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t1306 mm8 a=r3 b=r0 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t1307 mm8 a=r0 b=r2 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t1308 mm8 a=r2 b=r5 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t1309 mm8 a=r5 b=r7 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t1310 mm8 a=r4 b=r5 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t1311 mm8 a=r4 b=r7 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t1312 mm8 a=r3 b=r7 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t1313 mm8 a=r5 b=r1 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t1314 mm8 a=r5 b=r2 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t1315 mm8 a=r1 b=r0 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t1316 mm8 a=r7 b=r3 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t1317 mm8 a=r2 b=r5 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t1318 mm8 a=r2 b=r3 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t1319 mm8 a=r7 b=r5 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t1320 mm8 a=r0 b=r3 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t1321 mm8 a=r1 b=r0 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t1322 mm8 a=r4 b=r0 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t1323 mm8 a=r5 b=r3 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t1324 mm8 a=r3 b=r0 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t1325 mm8 a=r6 b=r4 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t1326 mm8 a=r4 b=r6 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t1327 mm8 a=r1 b=r3 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t1328 mm8 a=r7 b=r3 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t1329 mm8 a=r3 b=r7 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t1330 mm8 a=r3 b=r6 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t1331 mm8 a=r0 b=r7 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t1332 mm8 a=r0 b=r7 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t1333 mm8 a=r3 b=r3 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t1334 mm8 a=r7 b=r5 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t1335 mm8 a=r5 b=r7 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t1336 mm8 a=r5 b=r5 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t1337 mm8 a=r6 b=r0 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t1338 mm8 a=r6 b=r4 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t1339 mm8 a=r4 b=r6 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t1340 mm8 a=r6 b=r1 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t1341 mm8 a=r0 b=r6 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t1342 mm8 a=r4 b=r6 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t1343 mm8 a=r5 b=r5 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t1344 mm8 a=r2 b=r6 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t1345 mm8 a=r7 b=r3 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t1346 mm8 a=r5 b=r0 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t1347 mm8 a=r2 b=r3 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t1348 mm8 a=r4 b=r3 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t1349 mm8 a=r0 b=r1 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t1350 mm8 a=r1 b=r6 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t1351 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t1352 mm8 a=r5 b=r4 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t1353 mm8 a=r2 b=r7 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t1354 mm8 a=r7 b=r4 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t1355 mm8 a=r7 b=r4 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t1356 mm8 a=r4 b=r6 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t1357 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t1358 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t1359 mm8 a=r7 b=r7 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t1360 mm8 a=r0 b=r4 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t1361 mm8 a=r0 b=r5 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t1362 mm8 a=r0 b=r2 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t1363 mm8 a=r4 b=r4 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t1364 mm8 a=r7 b=r3 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t1365 mm8 a=r2 b=r4 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t1366 mm8 a=r4 b=r3 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t1367 mm8 a=r3 b=r3 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t1368 mm8 a=r4 b=r0 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t1369 mm8 a=r5 b=r3 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t1370 mm8 a=r0 b=r1 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t1371 mm8 a=r0 b=r5 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t1372 mm8 a=r5 b=r6 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t1373 mm8 a=r7 b=r1 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t1374 mm8 a=r4 b=r6 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t1375 mm8 a=r5 b=r2 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t1376 mm8 a=r5 b=r5 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t1377 mm8 a=r7 b=r3 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t1378 mm8 a=r4 b=r1 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t1379 mm8 a=r4 b=r0 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t1380 mm8 a=r0 b=r7 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t1381 mm8 a=r2 b=r0 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t1382 mm8 a=r7 b=r5 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t1383 mm8 a=r1 b=r5 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t1384 mm8 a=r5 b=r2 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t1385 mm8 a=r0 b=r7 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t1386 mm8 a=r7 b=r0 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t1387 mm8 a=r4 b=r3 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t1388 mm8 a=r4 b=r6 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t1389 mm8 a=r4 b=r2 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t1390 mm8 a=r5 b=r1 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t1391 mm8 a=r6 b=r1 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t1392 mm8 a=r5 b=r7 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t1393 mm8 a=r3 b=r4 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t1394 mm8 a=r4 b=r7 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t1395 mm8 a=r7 b=r1 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t1396 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t1397 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t1398 mm8 a=r3 b=r0 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t1399 mm8 a=r3 b=r4 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t1400 mm8 a=r6 b=r4 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t1401 mm8 a=r0 b=r5 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t1402 mm8 a=r7 b=r5 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t1403 mm8 a=r0 b=r0 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t1404 mm8 a=r3 b=r6 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t1405 mm8 a=r0 b=r0 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t1406 mm8 a=r5 b=r3 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t1407 mm8 a=r7 b=r3 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t1408 mm8 a=r4 b=r7 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t1409 mm8 a=r4 b=r2 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t1410 mm8 a=r6 b=r0 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t1411 mm8 a=r1 b=r7 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t1412 mm8 a=r4 b=r3 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t1413 mm8 a=r4 b=r6 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t1414 mm8 a=r0 b=r7 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t1415 mm8 a=r3 b=r6 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t1416 mm8 a=r3 b=r2 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t1417 mm8 a=r2 b=r7 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t1418 mm8 a=r0 b=r6 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t1419 mm8 a=r0 b=r3 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t1420 mm8 a=r1 b=r3 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t1421 mm8 a=r4 b=r0 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t1422 mm8 a=r6 b=r4 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t1423 mm8 a=r4 b=r6 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t1424 mm8 a=r6 b=r3 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t1425 mm8 a=r7 b=r3 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t1426 mm8 a=r0 b=r3 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t1427 mm8 a=r5 b=r6 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t1428 mm8 a=r0 b=r3 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t1429 mm8 a=r7 b=r1 c=r3 c2=r7 - } - uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((ulong)hi << 32) | (ulong)lo; -} diff --git a/proto-cuda/packs-ca4/mx8_mm1430/program_bound.metal b/proto-cuda/packs-ca4/mx8_mm1430/program_bound.metal deleted file mode 100644 index 3311ad43a..000000000 --- a/proto-cuda/packs-ca4/mx8_mm1430/program_bound.metal +++ /dev/null @@ -1,1561 +0,0 @@ -#include -using namespace metal; -// int8 tile reference (Counter ASIC 4.0 research): A = the 32 lanes' av as 8 x 16 bytes (lane l holds row l >> 2, bytes 4 (l & 3)..), -// B = bv as 16 x 8 (lane l holds column l >> 2); lane l gets C[l >> 2][2 (l & 3)] and C[l >> 2][2 (l & 3) + 1]; the PTX mma.m8n8k16 u8 layout. -static inline void mm8_ref(uint av, uint bv, uint lane, thread uint& d0, thread uint& d1) { - uint row = lane >> 2, col0 = 2u * (lane & 3u); - uint acc0 = 0u, acc1 = 0u; - for (uint j = 0u; j < 4u; ++j) { - uint aw = simd_shuffle(av, (ushort)(4u * row + j)); - uint bw0 = simd_shuffle(bv, (ushort)(4u * col0 + j)); - uint bw1 = simd_shuffle(bv, (ushort)(4u * (col0 + 1u) + j)); - for (uint t = 0u; t < 4u; ++t) { - uint ab = (aw >> (8u * t)) & 0xffu; - acc0 += ab * ((bw0 >> (8u * t)) & 0xffu); - acc1 += ab * ((bw1 >> (8u * t)) & 0xffu); - } - } - d0 = acc0; d1 = acc1; -} - - -#define MASK 0x0fffffffu -constant uint SEEDW[8] = { 0x67a9a7beu, 0x1a155b25u, 0xfddfb732u, 0x4b5af2e8u, 0xc55caf33u, 0xa27c13b7u, 0x06628a48u, 0x03852469u }; - -inline uint splitmix32(uint x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -inline uint rotl_imm(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 -inline uint rotr_var(uint x, uint n) { n &= 31u; return (x >> n) | (x << ((32u - n) & 31u)); } -inline uint ds_elem(uint i, uint d0, uint d1) { - uint x = i ^ d0; - x *= 0x9E3779B1u; x ^= x >> 15; - x += d1; - x *= 0x85EBCA77u; x ^= x >> 13; - x *= 0xC2B2AE3Du; x ^= x >> 16; - return x; -} - -// Header-bound variant: the init words come from buffer 3 (bind.rs), not from SEEDW. -kernel void igneum_hash_bound(device const uint* dataset [[buffer(0)]], - device ulong* out [[buffer(1)]], - constant uint& baseNonce [[buffer(2)]], - constant uint* initw [[buffer(3)]], - uint gid [[thread_position_in_grid]]) { - uint nonce = baseNonce + gid; - uint r0, r1, r2, r3, r4, r5, r6, r7; - uint lane = gid & 31u; - { uint x = nonce ^ initw[0]; x += 0x9e3779b9u * 1u; x = splitmix32(x); r0 = x ^ initw[1]; } - { uint x = nonce ^ initw[1]; x += 0x9e3779b9u * 2u; x = splitmix32(x); r1 = x ^ initw[2]; } - { uint x = nonce ^ initw[2]; x += 0x9e3779b9u * 3u; x = splitmix32(x); r2 = x ^ initw[3]; } - { uint x = nonce ^ initw[3]; x += 0x9e3779b9u * 4u; x = splitmix32(x); r3 = x ^ initw[4]; } - { uint x = nonce ^ initw[4]; x += 0x9e3779b9u * 5u; x = splitmix32(x); r4 = x ^ initw[5]; } - { uint x = nonce ^ initw[5]; x += 0x9e3779b9u * 6u; x = splitmix32(x); r5 = x ^ initw[6]; } - { uint x = nonce ^ initw[6]; x += 0x9e3779b9u * 7u; x = splitmix32(x); r6 = x ^ initw[7]; } - { uint x = nonce ^ initw[7]; x += 0x9e3779b9u * 8u; x = splitmix32(x); r7 = x ^ initw[0]; } - - for (uint it = 0u; it < 8u; ++it) { - uint sel = r0; - r2 = r3 * r4 + r2; // 0 - r2 = r1 * r1 + r2; // 1 - r2 = r3 * r2 + r2; // 2 - r3 = r3 ^ r5; // 3 - r7 = r7 ^ dataset[r2 & MASK]; // 4 - r5 = r5 ^ dataset[r7 & MASK]; // 5 - r1 = r1 ^ simd_shuffle_xor(r4, (ushort)8); // 6 - r7 = r7 ^ simd_shuffle_xor(r3, (ushort)8); // 7 - r1 = mulhi(r1, r5); // 8 - r6 = rotr_var(r6, r3); // 9 - r3 = r3 | r4; // 10 - r4 = r4 ^ dataset[r3 & MASK]; // 11 - r0 = mulhi(r0, r4); // 12 - r5 = r5 + r1 + select(0xc7934706u, 0xd3177981u, ((sel >> 30u) & 1u) != 0u); // 13 - r0 = r0 ^ dataset[r4 & MASK]; // 14 - r2 = r2 - r4; // 15 - r2 = r2 ^ dataset[r0 & MASK]; // 16 - r7 = r7 ^ dataset[r2 & MASK]; // 17 - r7 = r7 ^ simd_shuffle_xor(r3, (ushort)4); // 18 - r5 = r5 * r0; // 19 - r3 = r3 ^ simd_shuffle_xor(r4, (ushort)2); // 20 - r2 = r2 ^ simd_shuffle_xor(r4, (ushort)16); // 21 - r6 = mulhi(r6, r2); // 22 - r6 = r6 ^ dataset[r1 & MASK]; // 23 - r5 = r5 * r0; // 24 - r5 = rotl_imm(r5, 19u); // 25 - r7 = r7 ^ simd_shuffle_xor(r6, (ushort)2); // 26 - r0 = r0 ^ r5; // 27 - r0 = r0 ^ r4; // 28 - r3 = r3 - r0; // 29 - r5 = r5 * r1; // 30 - r7 = r7 ^ dataset[r2 & MASK]; // 31 - r1 = r1 ^ dataset[r0 & MASK]; // 32 - r5 = r5 ^ r6; // 33 - r5 = r5 ^ dataset[r1 & MASK]; // 34 - r0 = mulhi(r0, r5); // 35 - r5 = r5 ^ simd_shuffle_xor(r2, (ushort)4); // 36 - r7 = r7 ^ dataset[r0 & MASK]; // 37 - r3 = r3 + r1 + select(0x75ba2fadu, 0x230c005cu, ((sel >> 27u) & 1u) != 0u); // 38 - r1 = r1 ^ simd_shuffle_xor(r5, (ushort)4); // 39 - r2 = r2 ^ r5; // 40 - r3 = r6 * r3 + r3; // 41 - r6 = r6 - r7; // 42 - r7 = r7 ^ r0; // 43 - r1 = r1 ^ dataset[r7 & MASK]; // 44 - r2 = r2 * r3; // 45 - r1 = mulhi(r1, r5); // 46 - r4 = r4 - r3; // 47 - r2 = rotr_var(r2, r6); // 48 - r3 = r3 ^ dataset[r5 & MASK]; // 49 - r1 = r1 + r5 + select(0x81b8bc2cu, 0x1907970cu, ((sel >> 7u) & 1u) != 0u); // 50 - r0 = r0 * r2; // 51 - r0 = r0 + r2 + select(0x4f92b968u, 0x699fd448u, ((sel >> 6u) & 1u) != 0u); // 52 - r1 = r1 + r0 + select(0x2bb965afu, 0x77b1520du, ((sel >> 12u) & 1u) != 0u); // 53 - r7 = rotl_imm(r7, 14u); // 54 - r3 = r3 + r7 + select(0x7b0fe07au, 0xa54c55a0u, ((sel >> 1u) & 1u) != 0u); // 55 - r6 = r6 ^ dataset[r7 & MASK]; // 56 - r1 = rotr_var(r1, r5); // 57 - r5 = r5 ^ dataset[r4 & MASK]; // 58 - r6 = r6 ^ dataset[r2 & MASK]; // 59 - r3 = r5 * r0 + r3; // 60 - r5 = r5 + r7 + select(0xaf9dd72du, 0xad7493e7u, ((sel >> 31u) & 1u) != 0u); // 61 - r4 = r4 + r6 + select(0x89841d87u, 0x1e07c3d9u, ((sel >> 27u) & 1u) != 0u); // 62 - r5 = rotl_imm(r5, 19u); // 63 - // int8 tile block (Counter ASIC 4.0 research, experimental): 1430 mma.m8n8k16 u8 tiles per iteration, both outputs consumed - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t0 mm8 a=r6 b=r5 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t1 mm8 a=r1 b=r1 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t2 mm8 a=r2 b=r5 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t3 mm8 a=r1 b=r5 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t4 mm8 a=r2 b=r0 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t5 mm8 a=r2 b=r6 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t6 mm8 a=r1 b=r4 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t7 mm8 a=r4 b=r6 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t8 mm8 a=r1 b=r1 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t9 mm8 a=r3 b=r7 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t10 mm8 a=r3 b=r0 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t11 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t12 mm8 a=r3 b=r1 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t13 mm8 a=r1 b=r5 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t14 mm8 a=r3 b=r3 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t15 mm8 a=r1 b=r0 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t16 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t17 mm8 a=r0 b=r5 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t18 mm8 a=r1 b=r1 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t19 mm8 a=r3 b=r3 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t20 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t21 mm8 a=r0 b=r3 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t22 mm8 a=r1 b=r7 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t23 mm8 a=r1 b=r2 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t24 mm8 a=r3 b=r4 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t25 mm8 a=r5 b=r0 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t26 mm8 a=r1 b=r0 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t27 mm8 a=r3 b=r2 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t28 mm8 a=r2 b=r4 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t29 mm8 a=r3 b=r4 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t30 mm8 a=r6 b=r5 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t31 mm8 a=r1 b=r7 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t32 mm8 a=r7 b=r6 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t33 mm8 a=r7 b=r1 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t34 mm8 a=r1 b=r0 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t35 mm8 a=r2 b=r4 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t36 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t37 mm8 a=r1 b=r7 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t38 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t39 mm8 a=r6 b=r2 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t40 mm8 a=r6 b=r1 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t41 mm8 a=r6 b=r0 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t42 mm8 a=r7 b=r4 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t43 mm8 a=r6 b=r1 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t44 mm8 a=r0 b=r6 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t45 mm8 a=r6 b=r5 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t46 mm8 a=r6 b=r6 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t47 mm8 a=r7 b=r3 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t48 mm8 a=r7 b=r7 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t49 mm8 a=r4 b=r5 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t50 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t51 mm8 a=r3 b=r4 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t52 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t53 mm8 a=r7 b=r7 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t54 mm8 a=r7 b=r6 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t55 mm8 a=r1 b=r7 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t56 mm8 a=r4 b=r0 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t57 mm8 a=r4 b=r3 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t58 mm8 a=r2 b=r3 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t59 mm8 a=r6 b=r7 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t60 mm8 a=r6 b=r0 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t61 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t62 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t63 mm8 a=r1 b=r6 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t64 mm8 a=r0 b=r5 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t65 mm8 a=r0 b=r1 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t66 mm8 a=r3 b=r5 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t67 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t68 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t69 mm8 a=r6 b=r7 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t70 mm8 a=r1 b=r7 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t71 mm8 a=r3 b=r0 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t72 mm8 a=r4 b=r0 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t73 mm8 a=r4 b=r1 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t74 mm8 a=r5 b=r1 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t75 mm8 a=r4 b=r4 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t76 mm8 a=r5 b=r6 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t77 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t78 mm8 a=r2 b=r5 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t79 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t80 mm8 a=r6 b=r7 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t81 mm8 a=r4 b=r3 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t82 mm8 a=r6 b=r0 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t83 mm8 a=r2 b=r2 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t84 mm8 a=r0 b=r6 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t85 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t86 mm8 a=r3 b=r4 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t87 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t88 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t89 mm8 a=r4 b=r2 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t90 mm8 a=r6 b=r5 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t91 mm8 a=r6 b=r5 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t92 mm8 a=r2 b=r6 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t93 mm8 a=r2 b=r2 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t94 mm8 a=r2 b=r4 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t95 mm8 a=r5 b=r4 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t96 mm8 a=r0 b=r6 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t97 mm8 a=r0 b=r5 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t98 mm8 a=r2 b=r7 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t99 mm8 a=r1 b=r4 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t100 mm8 a=r2 b=r4 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t101 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t102 mm8 a=r5 b=r7 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t103 mm8 a=r4 b=r7 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t104 mm8 a=r2 b=r0 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t105 mm8 a=r6 b=r1 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t106 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t107 mm8 a=r2 b=r1 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t108 mm8 a=r4 b=r6 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t109 mm8 a=r2 b=r7 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t110 mm8 a=r1 b=r0 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t111 mm8 a=r6 b=r5 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t112 mm8 a=r5 b=r7 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t113 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t114 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t115 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t116 mm8 a=r7 b=r2 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t117 mm8 a=r7 b=r6 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t118 mm8 a=r0 b=r1 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t119 mm8 a=r4 b=r3 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t120 mm8 a=r5 b=r3 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t121 mm8 a=r2 b=r7 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t122 mm8 a=r0 b=r1 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t123 mm8 a=r5 b=r2 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t124 mm8 a=r5 b=r1 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t125 mm8 a=r7 b=r2 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t126 mm8 a=r6 b=r3 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t127 mm8 a=r6 b=r7 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t128 mm8 a=r6 b=r7 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t129 mm8 a=r3 b=r2 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t130 mm8 a=r0 b=r4 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t131 mm8 a=r3 b=r5 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t132 mm8 a=r0 b=r7 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t133 mm8 a=r2 b=r6 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t134 mm8 a=r5 b=r6 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t135 mm8 a=r0 b=r2 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t136 mm8 a=r3 b=r1 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t137 mm8 a=r7 b=r0 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t138 mm8 a=r2 b=r2 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t139 mm8 a=r0 b=r4 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t140 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t141 mm8 a=r5 b=r5 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t142 mm8 a=r2 b=r2 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t143 mm8 a=r4 b=r3 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t144 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t145 mm8 a=r7 b=r2 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t146 mm8 a=r2 b=r7 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t147 mm8 a=r3 b=r1 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t148 mm8 a=r2 b=r1 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t149 mm8 a=r4 b=r6 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t150 mm8 a=r3 b=r6 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t151 mm8 a=r1 b=r4 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t152 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t153 mm8 a=r0 b=r3 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t154 mm8 a=r7 b=r3 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t155 mm8 a=r1 b=r3 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t156 mm8 a=r6 b=r3 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t157 mm8 a=r6 b=r4 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t158 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t159 mm8 a=r1 b=r0 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t160 mm8 a=r0 b=r1 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t161 mm8 a=r4 b=r0 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t162 mm8 a=r6 b=r6 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t163 mm8 a=r5 b=r3 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t164 mm8 a=r0 b=r7 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t165 mm8 a=r6 b=r5 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t166 mm8 a=r6 b=r5 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t167 mm8 a=r4 b=r5 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t168 mm8 a=r3 b=r6 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t169 mm8 a=r6 b=r2 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t170 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t171 mm8 a=r6 b=r3 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t172 mm8 a=r7 b=r3 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t173 mm8 a=r0 b=r0 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t174 mm8 a=r6 b=r6 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t175 mm8 a=r1 b=r6 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t176 mm8 a=r2 b=r4 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t177 mm8 a=r1 b=r7 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t178 mm8 a=r4 b=r2 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t179 mm8 a=r5 b=r2 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t180 mm8 a=r0 b=r4 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t181 mm8 a=r3 b=r5 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t182 mm8 a=r1 b=r1 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t183 mm8 a=r3 b=r1 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t184 mm8 a=r7 b=r4 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t185 mm8 a=r5 b=r0 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t186 mm8 a=r2 b=r5 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t187 mm8 a=r3 b=r0 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t188 mm8 a=r1 b=r4 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t189 mm8 a=r7 b=r2 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t190 mm8 a=r1 b=r5 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t191 mm8 a=r2 b=r1 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t192 mm8 a=r5 b=r0 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t193 mm8 a=r2 b=r1 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t194 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t195 mm8 a=r2 b=r7 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t196 mm8 a=r5 b=r6 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t197 mm8 a=r5 b=r3 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t198 mm8 a=r1 b=r0 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t199 mm8 a=r2 b=r5 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t200 mm8 a=r5 b=r7 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t201 mm8 a=r7 b=r2 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t202 mm8 a=r2 b=r5 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t203 mm8 a=r0 b=r5 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t204 mm8 a=r4 b=r7 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t205 mm8 a=r3 b=r7 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t206 mm8 a=r0 b=r6 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t207 mm8 a=r7 b=r7 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t208 mm8 a=r0 b=r1 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t209 mm8 a=r4 b=r4 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t210 mm8 a=r7 b=r1 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t211 mm8 a=r1 b=r5 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t212 mm8 a=r2 b=r4 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t213 mm8 a=r2 b=r2 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t214 mm8 a=r4 b=r6 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t215 mm8 a=r7 b=r3 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t216 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t217 mm8 a=r5 b=r7 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t218 mm8 a=r7 b=r1 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t219 mm8 a=r2 b=r1 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t220 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t221 mm8 a=r7 b=r1 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t222 mm8 a=r0 b=r6 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t223 mm8 a=r2 b=r2 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t224 mm8 a=r1 b=r1 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t225 mm8 a=r2 b=r7 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t226 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t227 mm8 a=r3 b=r5 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t228 mm8 a=r7 b=r1 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t229 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t230 mm8 a=r0 b=r3 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t231 mm8 a=r1 b=r3 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t232 mm8 a=r5 b=r4 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t233 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t234 mm8 a=r2 b=r0 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t235 mm8 a=r3 b=r4 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t236 mm8 a=r1 b=r7 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t237 mm8 a=r3 b=r5 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t238 mm8 a=r2 b=r0 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t239 mm8 a=r5 b=r7 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t240 mm8 a=r2 b=r7 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t241 mm8 a=r0 b=r1 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t242 mm8 a=r0 b=r2 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t243 mm8 a=r4 b=r4 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t244 mm8 a=r7 b=r6 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t245 mm8 a=r4 b=r4 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t246 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t247 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t248 mm8 a=r7 b=r5 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t249 mm8 a=r4 b=r4 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t250 mm8 a=r0 b=r6 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t251 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t252 mm8 a=r0 b=r2 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t253 mm8 a=r5 b=r5 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t254 mm8 a=r6 b=r7 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t255 mm8 a=r5 b=r0 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t256 mm8 a=r2 b=r1 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t257 mm8 a=r4 b=r3 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t258 mm8 a=r4 b=r1 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t259 mm8 a=r5 b=r5 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t260 mm8 a=r3 b=r2 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t261 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t262 mm8 a=r6 b=r4 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t263 mm8 a=r6 b=r4 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t264 mm8 a=r7 b=r2 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t265 mm8 a=r6 b=r1 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t266 mm8 a=r0 b=r0 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t267 mm8 a=r6 b=r3 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t268 mm8 a=r1 b=r0 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t269 mm8 a=r7 b=r7 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t270 mm8 a=r6 b=r2 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t271 mm8 a=r0 b=r3 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t272 mm8 a=r6 b=r0 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t273 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t274 mm8 a=r1 b=r0 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t275 mm8 a=r1 b=r3 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t276 mm8 a=r1 b=r5 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t277 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t278 mm8 a=r0 b=r0 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t279 mm8 a=r2 b=r4 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t280 mm8 a=r7 b=r2 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t281 mm8 a=r1 b=r0 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t282 mm8 a=r0 b=r1 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t283 mm8 a=r0 b=r2 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t284 mm8 a=r1 b=r5 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t285 mm8 a=r4 b=r7 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t286 mm8 a=r1 b=r0 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t287 mm8 a=r7 b=r1 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t288 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t289 mm8 a=r0 b=r5 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t290 mm8 a=r3 b=r2 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t291 mm8 a=r7 b=r4 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t292 mm8 a=r6 b=r7 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t293 mm8 a=r5 b=r5 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t294 mm8 a=r5 b=r7 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t295 mm8 a=r0 b=r7 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t296 mm8 a=r6 b=r1 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t297 mm8 a=r4 b=r5 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t298 mm8 a=r6 b=r0 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t299 mm8 a=r6 b=r0 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t300 mm8 a=r6 b=r4 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t301 mm8 a=r6 b=r4 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t302 mm8 a=r7 b=r6 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t303 mm8 a=r3 b=r2 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t304 mm8 a=r6 b=r5 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t305 mm8 a=r4 b=r5 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t306 mm8 a=r3 b=r1 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t307 mm8 a=r4 b=r3 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t308 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t309 mm8 a=r2 b=r1 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t310 mm8 a=r6 b=r1 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t311 mm8 a=r4 b=r6 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t312 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t313 mm8 a=r2 b=r6 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t314 mm8 a=r0 b=r3 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t315 mm8 a=r7 b=r1 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t316 mm8 a=r2 b=r4 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t317 mm8 a=r1 b=r6 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t318 mm8 a=r2 b=r1 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t319 mm8 a=r0 b=r2 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t320 mm8 a=r5 b=r2 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t321 mm8 a=r7 b=r4 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t322 mm8 a=r2 b=r4 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t323 mm8 a=r4 b=r3 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t324 mm8 a=r5 b=r7 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t325 mm8 a=r6 b=r2 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t326 mm8 a=r6 b=r1 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t327 mm8 a=r1 b=r5 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t328 mm8 a=r0 b=r1 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t329 mm8 a=r0 b=r3 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t330 mm8 a=r6 b=r7 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t331 mm8 a=r4 b=r4 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t332 mm8 a=r4 b=r4 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t333 mm8 a=r4 b=r6 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t334 mm8 a=r2 b=r2 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t335 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t336 mm8 a=r4 b=r0 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t337 mm8 a=r1 b=r4 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t338 mm8 a=r3 b=r7 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t339 mm8 a=r5 b=r1 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t340 mm8 a=r5 b=r0 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t341 mm8 a=r6 b=r6 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t342 mm8 a=r4 b=r7 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t343 mm8 a=r0 b=r4 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t344 mm8 a=r2 b=r5 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t345 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t346 mm8 a=r6 b=r0 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t347 mm8 a=r5 b=r3 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t348 mm8 a=r0 b=r4 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t349 mm8 a=r6 b=r1 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t350 mm8 a=r1 b=r5 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t351 mm8 a=r6 b=r4 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t352 mm8 a=r4 b=r1 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t353 mm8 a=r7 b=r3 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t354 mm8 a=r4 b=r1 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t355 mm8 a=r4 b=r5 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t356 mm8 a=r7 b=r6 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t357 mm8 a=r7 b=r3 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t358 mm8 a=r4 b=r7 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t359 mm8 a=r2 b=r7 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t360 mm8 a=r4 b=r4 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t361 mm8 a=r2 b=r2 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t362 mm8 a=r2 b=r7 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t363 mm8 a=r7 b=r6 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t364 mm8 a=r6 b=r4 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t365 mm8 a=r6 b=r6 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t366 mm8 a=r7 b=r7 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t367 mm8 a=r4 b=r1 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t368 mm8 a=r7 b=r0 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t369 mm8 a=r1 b=r0 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t370 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t371 mm8 a=r5 b=r4 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t372 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t373 mm8 a=r4 b=r6 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t374 mm8 a=r1 b=r5 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t375 mm8 a=r6 b=r5 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t376 mm8 a=r3 b=r6 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t377 mm8 a=r6 b=r4 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t378 mm8 a=r6 b=r3 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t379 mm8 a=r3 b=r0 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t380 mm8 a=r2 b=r0 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t381 mm8 a=r3 b=r5 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t382 mm8 a=r5 b=r2 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t383 mm8 a=r1 b=r2 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t384 mm8 a=r3 b=r1 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t385 mm8 a=r6 b=r0 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t386 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t387 mm8 a=r3 b=r7 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t388 mm8 a=r3 b=r0 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t389 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t390 mm8 a=r5 b=r4 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t391 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t392 mm8 a=r2 b=r3 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t393 mm8 a=r3 b=r6 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t394 mm8 a=r0 b=r7 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t395 mm8 a=r0 b=r7 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t396 mm8 a=r7 b=r7 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t397 mm8 a=r5 b=r7 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t398 mm8 a=r6 b=r5 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t399 mm8 a=r6 b=r3 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t400 mm8 a=r2 b=r7 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t401 mm8 a=r7 b=r6 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t402 mm8 a=r0 b=r0 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t403 mm8 a=r7 b=r4 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t404 mm8 a=r1 b=r1 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t405 mm8 a=r6 b=r6 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t406 mm8 a=r6 b=r1 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t407 mm8 a=r5 b=r7 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t408 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t409 mm8 a=r3 b=r4 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t410 mm8 a=r0 b=r5 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t411 mm8 a=r2 b=r0 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t412 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t413 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t414 mm8 a=r7 b=r1 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t415 mm8 a=r4 b=r7 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t416 mm8 a=r5 b=r4 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t417 mm8 a=r6 b=r6 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t418 mm8 a=r1 b=r3 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t419 mm8 a=r5 b=r0 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t420 mm8 a=r4 b=r2 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t421 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t422 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t423 mm8 a=r1 b=r7 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t424 mm8 a=r5 b=r1 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t425 mm8 a=r2 b=r7 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t426 mm8 a=r1 b=r6 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t427 mm8 a=r6 b=r6 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t428 mm8 a=r6 b=r2 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t429 mm8 a=r3 b=r3 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t430 mm8 a=r7 b=r7 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t431 mm8 a=r6 b=r4 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t432 mm8 a=r3 b=r2 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t433 mm8 a=r3 b=r0 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t434 mm8 a=r0 b=r7 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t435 mm8 a=r7 b=r5 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t436 mm8 a=r6 b=r4 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t437 mm8 a=r0 b=r7 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t438 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t439 mm8 a=r4 b=r7 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t440 mm8 a=r3 b=r5 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t441 mm8 a=r5 b=r1 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t442 mm8 a=r1 b=r2 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t443 mm8 a=r1 b=r7 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t444 mm8 a=r3 b=r7 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t445 mm8 a=r7 b=r5 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t446 mm8 a=r3 b=r1 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t447 mm8 a=r6 b=r7 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t448 mm8 a=r5 b=r2 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t449 mm8 a=r7 b=r1 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t450 mm8 a=r7 b=r5 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t451 mm8 a=r0 b=r3 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t452 mm8 a=r2 b=r4 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t453 mm8 a=r5 b=r7 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t454 mm8 a=r7 b=r7 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t455 mm8 a=r7 b=r1 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t456 mm8 a=r6 b=r5 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t457 mm8 a=r7 b=r7 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t458 mm8 a=r5 b=r1 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t459 mm8 a=r6 b=r7 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t460 mm8 a=r7 b=r7 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t461 mm8 a=r0 b=r6 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t462 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t463 mm8 a=r2 b=r0 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t464 mm8 a=r0 b=r3 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t465 mm8 a=r1 b=r7 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t466 mm8 a=r1 b=r2 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t467 mm8 a=r1 b=r0 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t468 mm8 a=r4 b=r1 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t469 mm8 a=r2 b=r0 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t470 mm8 a=r3 b=r1 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t471 mm8 a=r2 b=r2 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t472 mm8 a=r1 b=r7 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t473 mm8 a=r2 b=r7 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t474 mm8 a=r5 b=r3 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t475 mm8 a=r7 b=r6 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t476 mm8 a=r3 b=r0 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t477 mm8 a=r3 b=r1 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t478 mm8 a=r0 b=r6 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t479 mm8 a=r7 b=r6 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t480 mm8 a=r6 b=r0 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t481 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t482 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t483 mm8 a=r7 b=r0 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t484 mm8 a=r3 b=r7 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t485 mm8 a=r6 b=r4 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t486 mm8 a=r6 b=r5 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t487 mm8 a=r2 b=r5 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t488 mm8 a=r1 b=r2 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t489 mm8 a=r5 b=r2 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t490 mm8 a=r7 b=r1 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t491 mm8 a=r0 b=r0 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t492 mm8 a=r3 b=r1 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t493 mm8 a=r6 b=r1 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t494 mm8 a=r5 b=r6 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t495 mm8 a=r2 b=r2 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t496 mm8 a=r3 b=r5 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t497 mm8 a=r3 b=r2 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t498 mm8 a=r3 b=r1 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t499 mm8 a=r1 b=r6 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t500 mm8 a=r7 b=r6 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t501 mm8 a=r4 b=r4 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t502 mm8 a=r5 b=r7 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t503 mm8 a=r7 b=r5 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t504 mm8 a=r2 b=r0 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t505 mm8 a=r4 b=r0 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t506 mm8 a=r0 b=r2 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t507 mm8 a=r5 b=r6 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t508 mm8 a=r7 b=r3 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t509 mm8 a=r4 b=r4 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t510 mm8 a=r0 b=r6 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t511 mm8 a=r2 b=r4 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t512 mm8 a=r2 b=r7 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t513 mm8 a=r0 b=r3 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t514 mm8 a=r6 b=r7 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t515 mm8 a=r5 b=r2 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t516 mm8 a=r0 b=r2 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t517 mm8 a=r6 b=r7 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t518 mm8 a=r1 b=r4 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t519 mm8 a=r6 b=r0 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t520 mm8 a=r1 b=r2 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t521 mm8 a=r0 b=r4 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t522 mm8 a=r7 b=r2 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t523 mm8 a=r5 b=r3 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t524 mm8 a=r0 b=r4 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t525 mm8 a=r1 b=r5 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t526 mm8 a=r7 b=r1 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t527 mm8 a=r5 b=r1 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t528 mm8 a=r4 b=r1 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t529 mm8 a=r4 b=r2 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t530 mm8 a=r0 b=r2 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t531 mm8 a=r7 b=r0 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t532 mm8 a=r7 b=r2 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t533 mm8 a=r3 b=r2 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t534 mm8 a=r4 b=r3 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t535 mm8 a=r0 b=r6 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t536 mm8 a=r3 b=r1 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t537 mm8 a=r1 b=r6 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t538 mm8 a=r6 b=r6 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t539 mm8 a=r4 b=r4 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t540 mm8 a=r1 b=r4 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t541 mm8 a=r5 b=r3 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t542 mm8 a=r7 b=r7 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t543 mm8 a=r4 b=r2 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t544 mm8 a=r5 b=r7 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t545 mm8 a=r5 b=r2 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t546 mm8 a=r7 b=r1 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t547 mm8 a=r3 b=r5 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t548 mm8 a=r6 b=r3 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t549 mm8 a=r2 b=r2 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t550 mm8 a=r5 b=r5 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t551 mm8 a=r7 b=r6 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t552 mm8 a=r6 b=r7 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t553 mm8 a=r6 b=r3 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t554 mm8 a=r4 b=r2 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t555 mm8 a=r2 b=r7 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t556 mm8 a=r3 b=r6 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t557 mm8 a=r7 b=r7 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t558 mm8 a=r5 b=r0 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t559 mm8 a=r4 b=r1 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t560 mm8 a=r5 b=r4 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t561 mm8 a=r6 b=r1 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t562 mm8 a=r1 b=r2 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t563 mm8 a=r5 b=r3 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t564 mm8 a=r4 b=r3 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t565 mm8 a=r6 b=r0 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t566 mm8 a=r1 b=r6 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t567 mm8 a=r3 b=r6 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t568 mm8 a=r6 b=r5 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t569 mm8 a=r3 b=r7 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t570 mm8 a=r5 b=r1 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t571 mm8 a=r6 b=r0 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t572 mm8 a=r5 b=r0 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t573 mm8 a=r3 b=r0 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t574 mm8 a=r2 b=r2 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t575 mm8 a=r3 b=r1 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t576 mm8 a=r2 b=r3 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t577 mm8 a=r6 b=r3 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t578 mm8 a=r5 b=r6 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t579 mm8 a=r4 b=r7 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t580 mm8 a=r7 b=r1 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t581 mm8 a=r1 b=r6 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t582 mm8 a=r2 b=r1 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t583 mm8 a=r5 b=r0 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t584 mm8 a=r1 b=r7 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t585 mm8 a=r0 b=r4 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t586 mm8 a=r4 b=r4 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t587 mm8 a=r0 b=r4 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t588 mm8 a=r5 b=r5 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t589 mm8 a=r3 b=r6 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t590 mm8 a=r3 b=r2 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t591 mm8 a=r5 b=r1 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t592 mm8 a=r5 b=r3 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t593 mm8 a=r5 b=r3 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t594 mm8 a=r6 b=r7 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t595 mm8 a=r3 b=r0 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t596 mm8 a=r7 b=r6 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t597 mm8 a=r2 b=r4 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t598 mm8 a=r7 b=r7 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t599 mm8 a=r7 b=r2 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t600 mm8 a=r7 b=r7 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t601 mm8 a=r2 b=r7 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t602 mm8 a=r0 b=r1 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t603 mm8 a=r0 b=r0 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t604 mm8 a=r5 b=r5 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t605 mm8 a=r2 b=r0 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t606 mm8 a=r7 b=r5 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t607 mm8 a=r0 b=r2 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t608 mm8 a=r7 b=r6 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t609 mm8 a=r1 b=r5 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t610 mm8 a=r3 b=r3 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t611 mm8 a=r2 b=r2 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t612 mm8 a=r1 b=r1 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t613 mm8 a=r6 b=r3 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t614 mm8 a=r3 b=r4 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t615 mm8 a=r6 b=r2 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t616 mm8 a=r2 b=r3 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t617 mm8 a=r5 b=r4 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t618 mm8 a=r3 b=r6 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t619 mm8 a=r5 b=r0 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t620 mm8 a=r3 b=r5 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t621 mm8 a=r4 b=r2 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t622 mm8 a=r7 b=r0 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t623 mm8 a=r4 b=r6 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t624 mm8 a=r6 b=r2 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t625 mm8 a=r6 b=r2 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t626 mm8 a=r5 b=r2 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t627 mm8 a=r4 b=r0 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t628 mm8 a=r1 b=r5 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t629 mm8 a=r4 b=r5 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t630 mm8 a=r4 b=r5 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t631 mm8 a=r6 b=r4 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t632 mm8 a=r2 b=r0 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t633 mm8 a=r2 b=r1 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t634 mm8 a=r1 b=r1 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t635 mm8 a=r6 b=r7 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t636 mm8 a=r2 b=r5 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t637 mm8 a=r5 b=r7 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t638 mm8 a=r1 b=r4 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t639 mm8 a=r1 b=r7 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t640 mm8 a=r0 b=r4 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t641 mm8 a=r5 b=r6 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t642 mm8 a=r7 b=r5 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t643 mm8 a=r3 b=r5 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t644 mm8 a=r1 b=r6 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t645 mm8 a=r0 b=r7 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t646 mm8 a=r5 b=r6 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t647 mm8 a=r6 b=r4 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t648 mm8 a=r3 b=r1 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t649 mm8 a=r7 b=r2 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t650 mm8 a=r7 b=r7 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t651 mm8 a=r6 b=r3 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t652 mm8 a=r1 b=r6 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t653 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t654 mm8 a=r1 b=r6 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t655 mm8 a=r7 b=r5 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t656 mm8 a=r6 b=r5 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t657 mm8 a=r7 b=r4 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t658 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t659 mm8 a=r0 b=r7 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t660 mm8 a=r6 b=r1 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t661 mm8 a=r3 b=r7 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t662 mm8 a=r2 b=r3 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t663 mm8 a=r0 b=r3 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t664 mm8 a=r2 b=r0 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t665 mm8 a=r1 b=r1 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t666 mm8 a=r7 b=r1 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t667 mm8 a=r2 b=r0 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t668 mm8 a=r3 b=r1 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t669 mm8 a=r1 b=r4 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t670 mm8 a=r7 b=r7 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t671 mm8 a=r0 b=r3 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t672 mm8 a=r0 b=r1 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t673 mm8 a=r6 b=r2 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t674 mm8 a=r3 b=r3 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t675 mm8 a=r5 b=r6 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t676 mm8 a=r4 b=r3 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t677 mm8 a=r0 b=r1 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t678 mm8 a=r4 b=r7 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t679 mm8 a=r4 b=r7 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t680 mm8 a=r5 b=r1 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t681 mm8 a=r4 b=r2 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t682 mm8 a=r3 b=r6 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t683 mm8 a=r0 b=r5 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t684 mm8 a=r6 b=r2 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t685 mm8 a=r7 b=r3 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t686 mm8 a=r4 b=r1 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t687 mm8 a=r0 b=r5 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t688 mm8 a=r3 b=r6 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t689 mm8 a=r3 b=r1 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t690 mm8 a=r2 b=r6 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t691 mm8 a=r2 b=r6 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t692 mm8 a=r1 b=r4 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t693 mm8 a=r7 b=r1 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t694 mm8 a=r3 b=r5 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t695 mm8 a=r3 b=r2 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t696 mm8 a=r2 b=r4 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t697 mm8 a=r2 b=r3 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t698 mm8 a=r3 b=r0 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t699 mm8 a=r2 b=r1 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t700 mm8 a=r0 b=r1 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t701 mm8 a=r5 b=r7 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t702 mm8 a=r5 b=r5 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t703 mm8 a=r2 b=r2 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t704 mm8 a=r6 b=r5 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t705 mm8 a=r5 b=r4 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t706 mm8 a=r6 b=r0 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t707 mm8 a=r6 b=r4 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t708 mm8 a=r2 b=r0 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t709 mm8 a=r7 b=r0 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t710 mm8 a=r4 b=r2 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t711 mm8 a=r4 b=r7 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t712 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t713 mm8 a=r5 b=r2 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t714 mm8 a=r0 b=r0 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t715 mm8 a=r1 b=r7 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t716 mm8 a=r3 b=r3 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t717 mm8 a=r3 b=r3 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t718 mm8 a=r7 b=r5 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t719 mm8 a=r4 b=r7 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t720 mm8 a=r4 b=r7 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t721 mm8 a=r6 b=r4 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t722 mm8 a=r7 b=r5 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t723 mm8 a=r5 b=r0 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t724 mm8 a=r7 b=r4 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t725 mm8 a=r6 b=r1 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t726 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t727 mm8 a=r0 b=r6 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t728 mm8 a=r5 b=r7 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t729 mm8 a=r5 b=r6 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t730 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t731 mm8 a=r5 b=r6 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t732 mm8 a=r5 b=r1 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t733 mm8 a=r3 b=r4 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t734 mm8 a=r4 b=r3 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t735 mm8 a=r7 b=r4 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t736 mm8 a=r4 b=r2 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t737 mm8 a=r4 b=r7 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t738 mm8 a=r6 b=r2 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t739 mm8 a=r7 b=r5 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t740 mm8 a=r4 b=r3 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t741 mm8 a=r7 b=r0 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t742 mm8 a=r6 b=r5 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t743 mm8 a=r5 b=r3 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t744 mm8 a=r7 b=r4 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t745 mm8 a=r7 b=r0 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t746 mm8 a=r2 b=r5 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t747 mm8 a=r0 b=r3 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t748 mm8 a=r1 b=r6 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t749 mm8 a=r2 b=r5 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t750 mm8 a=r4 b=r3 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t751 mm8 a=r6 b=r6 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t752 mm8 a=r5 b=r0 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t753 mm8 a=r6 b=r7 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t754 mm8 a=r2 b=r4 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t755 mm8 a=r5 b=r6 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t756 mm8 a=r7 b=r3 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t757 mm8 a=r6 b=r6 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t758 mm8 a=r6 b=r2 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t759 mm8 a=r2 b=r7 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t760 mm8 a=r7 b=r7 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t761 mm8 a=r3 b=r3 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t762 mm8 a=r5 b=r2 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t763 mm8 a=r1 b=r1 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t764 mm8 a=r1 b=r2 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t765 mm8 a=r2 b=r4 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t766 mm8 a=r7 b=r6 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t767 mm8 a=r7 b=r5 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t768 mm8 a=r5 b=r6 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t769 mm8 a=r4 b=r5 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t770 mm8 a=r7 b=r5 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t771 mm8 a=r2 b=r5 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t772 mm8 a=r0 b=r4 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t773 mm8 a=r2 b=r5 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t774 mm8 a=r4 b=r0 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t775 mm8 a=r4 b=r6 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t776 mm8 a=r5 b=r7 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t777 mm8 a=r1 b=r1 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t778 mm8 a=r1 b=r2 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t779 mm8 a=r7 b=r5 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t780 mm8 a=r3 b=r7 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t781 mm8 a=r6 b=r7 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t782 mm8 a=r6 b=r4 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t783 mm8 a=r6 b=r5 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t784 mm8 a=r1 b=r5 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t785 mm8 a=r7 b=r2 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t786 mm8 a=r2 b=r4 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t787 mm8 a=r7 b=r2 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t788 mm8 a=r0 b=r0 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t789 mm8 a=r1 b=r4 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t790 mm8 a=r3 b=r1 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t791 mm8 a=r3 b=r5 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t792 mm8 a=r7 b=r1 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t793 mm8 a=r7 b=r2 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t794 mm8 a=r2 b=r4 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t795 mm8 a=r1 b=r7 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t796 mm8 a=r2 b=r4 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t797 mm8 a=r6 b=r2 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t798 mm8 a=r1 b=r3 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t799 mm8 a=r1 b=r3 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t800 mm8 a=r0 b=r4 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t801 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t802 mm8 a=r4 b=r7 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t803 mm8 a=r3 b=r2 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t804 mm8 a=r0 b=r4 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t805 mm8 a=r4 b=r6 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t806 mm8 a=r0 b=r5 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t807 mm8 a=r5 b=r0 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t808 mm8 a=r2 b=r4 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t809 mm8 a=r1 b=r6 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t810 mm8 a=r1 b=r2 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t811 mm8 a=r2 b=r7 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t812 mm8 a=r0 b=r6 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t813 mm8 a=r2 b=r2 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t814 mm8 a=r3 b=r4 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t815 mm8 a=r4 b=r5 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t816 mm8 a=r3 b=r0 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t817 mm8 a=r3 b=r4 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t818 mm8 a=r2 b=r2 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t819 mm8 a=r1 b=r2 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t820 mm8 a=r3 b=r6 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t821 mm8 a=r7 b=r3 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t822 mm8 a=r1 b=r5 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t823 mm8 a=r1 b=r4 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t824 mm8 a=r7 b=r7 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t825 mm8 a=r1 b=r1 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t826 mm8 a=r0 b=r2 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t827 mm8 a=r3 b=r1 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t828 mm8 a=r7 b=r4 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t829 mm8 a=r5 b=r2 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t830 mm8 a=r6 b=r3 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t831 mm8 a=r2 b=r7 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t832 mm8 a=r1 b=r6 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t833 mm8 a=r0 b=r6 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t834 mm8 a=r2 b=r4 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t835 mm8 a=r0 b=r7 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t836 mm8 a=r3 b=r0 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t837 mm8 a=r4 b=r7 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t838 mm8 a=r5 b=r4 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t839 mm8 a=r0 b=r6 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t840 mm8 a=r1 b=r6 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t841 mm8 a=r4 b=r0 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t842 mm8 a=r3 b=r5 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t843 mm8 a=r2 b=r2 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t844 mm8 a=r3 b=r3 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t845 mm8 a=r1 b=r7 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t846 mm8 a=r1 b=r4 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t847 mm8 a=r4 b=r0 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t848 mm8 a=r6 b=r7 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t849 mm8 a=r6 b=r6 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t850 mm8 a=r6 b=r0 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t851 mm8 a=r0 b=r5 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t852 mm8 a=r2 b=r2 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t853 mm8 a=r6 b=r5 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t854 mm8 a=r1 b=r4 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t855 mm8 a=r1 b=r7 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t856 mm8 a=r2 b=r3 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t857 mm8 a=r6 b=r5 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t858 mm8 a=r0 b=r5 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t859 mm8 a=r2 b=r6 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t860 mm8 a=r7 b=r2 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t861 mm8 a=r3 b=r3 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t862 mm8 a=r0 b=r1 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t863 mm8 a=r0 b=r0 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t864 mm8 a=r5 b=r5 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t865 mm8 a=r3 b=r7 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t866 mm8 a=r6 b=r6 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t867 mm8 a=r7 b=r3 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t868 mm8 a=r4 b=r4 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t869 mm8 a=r3 b=r6 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t870 mm8 a=r0 b=r1 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t871 mm8 a=r5 b=r1 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t872 mm8 a=r0 b=r6 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t873 mm8 a=r2 b=r1 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t874 mm8 a=r5 b=r1 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t875 mm8 a=r1 b=r7 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t876 mm8 a=r7 b=r6 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t877 mm8 a=r6 b=r2 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t878 mm8 a=r5 b=r0 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t879 mm8 a=r4 b=r1 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t880 mm8 a=r3 b=r3 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t881 mm8 a=r2 b=r7 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t882 mm8 a=r7 b=r5 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t883 mm8 a=r2 b=r3 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t884 mm8 a=r2 b=r7 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t885 mm8 a=r3 b=r5 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t886 mm8 a=r5 b=r4 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t887 mm8 a=r1 b=r1 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t888 mm8 a=r6 b=r2 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t889 mm8 a=r4 b=r6 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t890 mm8 a=r0 b=r5 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t891 mm8 a=r0 b=r7 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t892 mm8 a=r5 b=r6 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t893 mm8 a=r2 b=r1 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t894 mm8 a=r0 b=r7 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t895 mm8 a=r2 b=r4 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t896 mm8 a=r0 b=r2 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t897 mm8 a=r7 b=r4 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t898 mm8 a=r1 b=r3 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t899 mm8 a=r5 b=r1 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t900 mm8 a=r7 b=r7 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t901 mm8 a=r4 b=r6 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t902 mm8 a=r2 b=r1 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t903 mm8 a=r0 b=r3 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t904 mm8 a=r6 b=r7 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t905 mm8 a=r0 b=r4 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t906 mm8 a=r3 b=r7 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t907 mm8 a=r2 b=r4 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t908 mm8 a=r7 b=r1 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t909 mm8 a=r4 b=r5 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t910 mm8 a=r3 b=r7 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t911 mm8 a=r2 b=r6 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t912 mm8 a=r6 b=r1 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t913 mm8 a=r2 b=r3 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t914 mm8 a=r7 b=r4 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t915 mm8 a=r4 b=r3 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t916 mm8 a=r4 b=r2 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t917 mm8 a=r5 b=r5 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t918 mm8 a=r7 b=r2 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t919 mm8 a=r3 b=r4 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t920 mm8 a=r6 b=r5 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t921 mm8 a=r6 b=r5 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t922 mm8 a=r1 b=r0 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t923 mm8 a=r6 b=r3 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t924 mm8 a=r2 b=r0 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t925 mm8 a=r6 b=r6 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t926 mm8 a=r5 b=r0 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t927 mm8 a=r1 b=r1 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t928 mm8 a=r5 b=r7 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t929 mm8 a=r1 b=r0 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t930 mm8 a=r7 b=r7 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t931 mm8 a=r7 b=r0 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t932 mm8 a=r1 b=r3 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t933 mm8 a=r1 b=r3 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t934 mm8 a=r6 b=r0 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t935 mm8 a=r5 b=r0 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t936 mm8 a=r3 b=r3 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t937 mm8 a=r0 b=r6 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t938 mm8 a=r1 b=r5 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t939 mm8 a=r7 b=r0 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t940 mm8 a=r1 b=r2 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t941 mm8 a=r1 b=r6 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t942 mm8 a=r0 b=r6 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t943 mm8 a=r4 b=r0 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t944 mm8 a=r6 b=r5 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t945 mm8 a=r3 b=r7 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t946 mm8 a=r2 b=r6 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t947 mm8 a=r7 b=r2 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t948 mm8 a=r6 b=r4 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t949 mm8 a=r6 b=r6 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t950 mm8 a=r7 b=r6 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t951 mm8 a=r2 b=r7 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t952 mm8 a=r3 b=r2 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t953 mm8 a=r4 b=r4 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t954 mm8 a=r2 b=r4 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t955 mm8 a=r0 b=r7 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t956 mm8 a=r7 b=r1 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t957 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t958 mm8 a=r6 b=r5 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t959 mm8 a=r4 b=r7 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t960 mm8 a=r1 b=r1 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t961 mm8 a=r1 b=r5 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t962 mm8 a=r6 b=r4 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t963 mm8 a=r7 b=r1 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t964 mm8 a=r2 b=r5 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t965 mm8 a=r3 b=r7 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t966 mm8 a=r7 b=r3 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t967 mm8 a=r0 b=r0 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t968 mm8 a=r5 b=r7 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t969 mm8 a=r4 b=r4 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t970 mm8 a=r4 b=r3 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t971 mm8 a=r3 b=r4 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t972 mm8 a=r1 b=r7 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t973 mm8 a=r4 b=r1 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t974 mm8 a=r5 b=r6 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t975 mm8 a=r1 b=r2 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t976 mm8 a=r0 b=r6 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t977 mm8 a=r2 b=r0 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t978 mm8 a=r0 b=r2 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t979 mm8 a=r4 b=r0 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t980 mm8 a=r3 b=r1 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t981 mm8 a=r2 b=r0 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t982 mm8 a=r1 b=r6 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t983 mm8 a=r2 b=r1 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t984 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t985 mm8 a=r4 b=r3 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t986 mm8 a=r2 b=r7 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t987 mm8 a=r5 b=r1 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t988 mm8 a=r6 b=r0 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t989 mm8 a=r2 b=r3 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t990 mm8 a=r1 b=r5 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t991 mm8 a=r6 b=r0 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t992 mm8 a=r2 b=r1 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t993 mm8 a=r6 b=r0 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t994 mm8 a=r6 b=r0 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t995 mm8 a=r0 b=r3 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t996 mm8 a=r7 b=r0 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t997 mm8 a=r5 b=r5 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t998 mm8 a=r3 b=r4 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t999 mm8 a=r7 b=r2 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t1000 mm8 a=r5 b=r3 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t1001 mm8 a=r5 b=r6 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t1002 mm8 a=r6 b=r2 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t1003 mm8 a=r4 b=r2 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t1004 mm8 a=r7 b=r3 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t1005 mm8 a=r1 b=r7 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t1006 mm8 a=r4 b=r2 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t1007 mm8 a=r0 b=r2 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t1008 mm8 a=r3 b=r5 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t1009 mm8 a=r0 b=r7 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t1010 mm8 a=r6 b=r7 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t1011 mm8 a=r7 b=r5 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t1012 mm8 a=r3 b=r5 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t1013 mm8 a=r6 b=r6 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t1014 mm8 a=r7 b=r3 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t1015 mm8 a=r7 b=r6 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t1016 mm8 a=r4 b=r0 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t1017 mm8 a=r5 b=r2 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t1018 mm8 a=r7 b=r6 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t1019 mm8 a=r0 b=r7 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t1020 mm8 a=r5 b=r6 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t1021 mm8 a=r7 b=r6 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t1022 mm8 a=r3 b=r5 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t1023 mm8 a=r0 b=r3 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t1024 mm8 a=r2 b=r6 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t1025 mm8 a=r4 b=r7 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t1026 mm8 a=r0 b=r1 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t1027 mm8 a=r1 b=r5 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t1028 mm8 a=r7 b=r6 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t1029 mm8 a=r2 b=r1 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t1030 mm8 a=r1 b=r7 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t1031 mm8 a=r3 b=r7 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t1032 mm8 a=r2 b=r1 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t1033 mm8 a=r2 b=r7 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t1034 mm8 a=r4 b=r3 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t1035 mm8 a=r3 b=r0 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t1036 mm8 a=r3 b=r0 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t1037 mm8 a=r7 b=r1 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t1038 mm8 a=r2 b=r1 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t1039 mm8 a=r5 b=r1 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t1040 mm8 a=r0 b=r7 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t1041 mm8 a=r2 b=r2 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t1042 mm8 a=r6 b=r6 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t1043 mm8 a=r6 b=r7 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t1044 mm8 a=r4 b=r2 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t1045 mm8 a=r0 b=r0 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t1046 mm8 a=r7 b=r1 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t1047 mm8 a=r5 b=r7 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t1048 mm8 a=r2 b=r3 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t1049 mm8 a=r5 b=r1 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t1050 mm8 a=r0 b=r5 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t1051 mm8 a=r7 b=r7 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t1052 mm8 a=r2 b=r3 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t1053 mm8 a=r6 b=r0 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t1054 mm8 a=r4 b=r2 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t1055 mm8 a=r5 b=r3 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t1056 mm8 a=r5 b=r1 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t1057 mm8 a=r1 b=r5 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t1058 mm8 a=r0 b=r5 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t1059 mm8 a=r5 b=r6 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t1060 mm8 a=r7 b=r3 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t1061 mm8 a=r5 b=r6 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t1062 mm8 a=r3 b=r0 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t1063 mm8 a=r1 b=r1 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t1064 mm8 a=r5 b=r4 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t1065 mm8 a=r3 b=r1 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t1066 mm8 a=r7 b=r7 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t1067 mm8 a=r0 b=r5 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t1068 mm8 a=r4 b=r6 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t1069 mm8 a=r2 b=r0 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t1070 mm8 a=r2 b=r3 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t1071 mm8 a=r2 b=r6 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t1072 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t1073 mm8 a=r3 b=r6 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t1074 mm8 a=r0 b=r0 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t1075 mm8 a=r4 b=r4 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t1076 mm8 a=r1 b=r7 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t1077 mm8 a=r7 b=r7 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t1078 mm8 a=r3 b=r7 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t1079 mm8 a=r6 b=r6 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t1080 mm8 a=r4 b=r5 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t1081 mm8 a=r1 b=r5 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t1082 mm8 a=r5 b=r4 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t1083 mm8 a=r2 b=r2 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t1084 mm8 a=r2 b=r0 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t1085 mm8 a=r3 b=r3 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t1086 mm8 a=r3 b=r1 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t1087 mm8 a=r1 b=r3 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t1088 mm8 a=r3 b=r5 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t1089 mm8 a=r0 b=r0 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t1090 mm8 a=r4 b=r6 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t1091 mm8 a=r4 b=r3 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t1092 mm8 a=r2 b=r6 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t1093 mm8 a=r5 b=r7 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t1094 mm8 a=r7 b=r3 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t1095 mm8 a=r0 b=r2 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t1096 mm8 a=r3 b=r3 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t1097 mm8 a=r6 b=r2 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t1098 mm8 a=r3 b=r6 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t1099 mm8 a=r2 b=r5 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t1100 mm8 a=r1 b=r4 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t1101 mm8 a=r0 b=r6 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t1102 mm8 a=r0 b=r5 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t1103 mm8 a=r7 b=r0 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t1104 mm8 a=r4 b=r2 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t1105 mm8 a=r3 b=r4 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t1106 mm8 a=r2 b=r1 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t1107 mm8 a=r3 b=r0 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t1108 mm8 a=r3 b=r5 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t1109 mm8 a=r6 b=r2 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t1110 mm8 a=r2 b=r4 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t1111 mm8 a=r0 b=r7 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t1112 mm8 a=r1 b=r0 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t1113 mm8 a=r4 b=r6 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t1114 mm8 a=r0 b=r5 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t1115 mm8 a=r4 b=r5 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t1116 mm8 a=r4 b=r6 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t1117 mm8 a=r0 b=r6 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t1118 mm8 a=r7 b=r6 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t1119 mm8 a=r2 b=r7 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t1120 mm8 a=r7 b=r5 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t1121 mm8 a=r4 b=r0 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t1122 mm8 a=r0 b=r5 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t1123 mm8 a=r0 b=r6 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t1124 mm8 a=r0 b=r1 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t1125 mm8 a=r6 b=r6 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t1126 mm8 a=r0 b=r0 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t1127 mm8 a=r1 b=r1 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t1128 mm8 a=r4 b=r1 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t1129 mm8 a=r0 b=r0 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t1130 mm8 a=r0 b=r4 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t1131 mm8 a=r3 b=r0 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t1132 mm8 a=r0 b=r6 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t1133 mm8 a=r6 b=r0 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t1134 mm8 a=r7 b=r3 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t1135 mm8 a=r6 b=r7 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t1136 mm8 a=r7 b=r4 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t1137 mm8 a=r4 b=r0 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t1138 mm8 a=r6 b=r4 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t1139 mm8 a=r6 b=r2 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t1140 mm8 a=r1 b=r2 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t1141 mm8 a=r0 b=r5 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t1142 mm8 a=r4 b=r5 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t1143 mm8 a=r5 b=r6 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t1144 mm8 a=r6 b=r4 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t1145 mm8 a=r5 b=r1 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t1146 mm8 a=r6 b=r5 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t1147 mm8 a=r7 b=r1 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t1148 mm8 a=r1 b=r6 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t1149 mm8 a=r5 b=r5 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t1150 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t1151 mm8 a=r7 b=r5 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t1152 mm8 a=r0 b=r6 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t1153 mm8 a=r2 b=r5 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t1154 mm8 a=r5 b=r3 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t1155 mm8 a=r0 b=r6 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t1156 mm8 a=r2 b=r6 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t1157 mm8 a=r6 b=r7 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t1158 mm8 a=r2 b=r3 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t1159 mm8 a=r1 b=r5 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t1160 mm8 a=r5 b=r0 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t1161 mm8 a=r7 b=r0 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t1162 mm8 a=r5 b=r3 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t1163 mm8 a=r4 b=r3 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t1164 mm8 a=r3 b=r4 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t1165 mm8 a=r1 b=r2 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t1166 mm8 a=r4 b=r7 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t1167 mm8 a=r2 b=r7 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t1168 mm8 a=r2 b=r6 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t1169 mm8 a=r0 b=r4 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t1170 mm8 a=r2 b=r1 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t1171 mm8 a=r1 b=r7 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t1172 mm8 a=r0 b=r1 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t1173 mm8 a=r6 b=r4 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t1174 mm8 a=r0 b=r1 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t1175 mm8 a=r0 b=r0 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t1176 mm8 a=r4 b=r5 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t1177 mm8 a=r7 b=r2 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t1178 mm8 a=r1 b=r5 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t1179 mm8 a=r0 b=r7 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t1180 mm8 a=r7 b=r3 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t1181 mm8 a=r7 b=r3 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t1182 mm8 a=r1 b=r2 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t1183 mm8 a=r2 b=r4 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t1184 mm8 a=r7 b=r0 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t1185 mm8 a=r5 b=r2 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t1186 mm8 a=r1 b=r0 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t1187 mm8 a=r5 b=r4 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t1188 mm8 a=r3 b=r0 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t1189 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t1190 mm8 a=r5 b=r0 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t1191 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t1192 mm8 a=r0 b=r2 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t1193 mm8 a=r3 b=r7 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t1194 mm8 a=r7 b=r2 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t1195 mm8 a=r1 b=r4 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t1196 mm8 a=r4 b=r6 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t1197 mm8 a=r5 b=r6 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t1198 mm8 a=r0 b=r6 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t1199 mm8 a=r5 b=r1 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t1200 mm8 a=r1 b=r3 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t1201 mm8 a=r6 b=r6 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t1202 mm8 a=r2 b=r4 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t1203 mm8 a=r0 b=r5 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t1204 mm8 a=r0 b=r2 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t1205 mm8 a=r4 b=r6 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t1206 mm8 a=r4 b=r6 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t1207 mm8 a=r3 b=r7 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t1208 mm8 a=r6 b=r1 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t1209 mm8 a=r5 b=r5 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t1210 mm8 a=r0 b=r0 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t1211 mm8 a=r7 b=r5 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t1212 mm8 a=r2 b=r6 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t1213 mm8 a=r6 b=r2 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t1214 mm8 a=r4 b=r2 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t1215 mm8 a=r7 b=r2 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t1216 mm8 a=r3 b=r3 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t1217 mm8 a=r5 b=r5 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t1218 mm8 a=r0 b=r7 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t1219 mm8 a=r2 b=r4 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t1220 mm8 a=r1 b=r7 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t1221 mm8 a=r5 b=r5 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t1222 mm8 a=r1 b=r3 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t1223 mm8 a=r0 b=r2 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t1224 mm8 a=r1 b=r1 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t1225 mm8 a=r4 b=r1 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t1226 mm8 a=r5 b=r6 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t1227 mm8 a=r1 b=r7 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t1228 mm8 a=r4 b=r2 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t1229 mm8 a=r5 b=r3 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t1230 mm8 a=r6 b=r2 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t1231 mm8 a=r4 b=r3 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t1232 mm8 a=r6 b=r7 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t1233 mm8 a=r0 b=r5 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t1234 mm8 a=r5 b=r4 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t1235 mm8 a=r5 b=r7 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t1236 mm8 a=r0 b=r7 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t1237 mm8 a=r7 b=r2 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t1238 mm8 a=r7 b=r0 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t1239 mm8 a=r3 b=r2 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t1240 mm8 a=r0 b=r5 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t1241 mm8 a=r5 b=r0 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t1242 mm8 a=r5 b=r7 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t1243 mm8 a=r6 b=r3 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t1244 mm8 a=r1 b=r5 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t1245 mm8 a=r0 b=r6 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t1246 mm8 a=r5 b=r5 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t1247 mm8 a=r1 b=r6 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t1248 mm8 a=r4 b=r0 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t1249 mm8 a=r7 b=r6 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t1250 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t1251 mm8 a=r3 b=r7 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t1252 mm8 a=r5 b=r5 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t1253 mm8 a=r5 b=r5 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t1254 mm8 a=r0 b=r6 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t1255 mm8 a=r1 b=r4 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t1256 mm8 a=r1 b=r3 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t1257 mm8 a=r0 b=r7 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t1258 mm8 a=r5 b=r4 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t1259 mm8 a=r0 b=r3 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t1260 mm8 a=r1 b=r5 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t1261 mm8 a=r6 b=r7 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t1262 mm8 a=r2 b=r5 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t1263 mm8 a=r3 b=r7 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t1264 mm8 a=r1 b=r6 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t1265 mm8 a=r2 b=r3 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t1266 mm8 a=r4 b=r6 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t1267 mm8 a=r6 b=r6 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t1268 mm8 a=r4 b=r5 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t1269 mm8 a=r5 b=r7 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t1270 mm8 a=r7 b=r3 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t1271 mm8 a=r7 b=r0 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t1272 mm8 a=r1 b=r3 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t1273 mm8 a=r3 b=r2 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t1274 mm8 a=r2 b=r0 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t1275 mm8 a=r7 b=r3 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t1276 mm8 a=r4 b=r3 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t1277 mm8 a=r5 b=r4 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t1278 mm8 a=r6 b=r3 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t1279 mm8 a=r0 b=r1 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t1280 mm8 a=r4 b=r1 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t1281 mm8 a=r6 b=r3 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t1282 mm8 a=r1 b=r7 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t1283 mm8 a=r6 b=r2 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t1284 mm8 a=r1 b=r6 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t1285 mm8 a=r3 b=r5 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t1286 mm8 a=r3 b=r1 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t1287 mm8 a=r4 b=r1 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t1288 mm8 a=r3 b=r6 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t1289 mm8 a=r6 b=r2 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t1290 mm8 a=r6 b=r5 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t1291 mm8 a=r5 b=r6 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t1292 mm8 a=r6 b=r6 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t1293 mm8 a=r6 b=r2 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t1294 mm8 a=r0 b=r5 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t1295 mm8 a=r5 b=r0 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t1296 mm8 a=r1 b=r0 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t1297 mm8 a=r0 b=r0 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t1298 mm8 a=r2 b=r6 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t1299 mm8 a=r7 b=r0 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t1300 mm8 a=r4 b=r1 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t1301 mm8 a=r4 b=r5 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t1302 mm8 a=r0 b=r4 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t1303 mm8 a=r4 b=r2 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t1304 mm8 a=r4 b=r7 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t1305 mm8 a=r3 b=r2 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t1306 mm8 a=r3 b=r0 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t1307 mm8 a=r0 b=r2 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t1308 mm8 a=r2 b=r5 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t1309 mm8 a=r5 b=r7 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t1310 mm8 a=r4 b=r5 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t1311 mm8 a=r4 b=r7 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t1312 mm8 a=r3 b=r7 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t1313 mm8 a=r5 b=r1 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t1314 mm8 a=r5 b=r2 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t1315 mm8 a=r1 b=r0 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t1316 mm8 a=r7 b=r3 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t1317 mm8 a=r2 b=r5 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t1318 mm8 a=r2 b=r3 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t1319 mm8 a=r7 b=r5 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t1320 mm8 a=r0 b=r3 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t1321 mm8 a=r1 b=r0 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t1322 mm8 a=r4 b=r0 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t1323 mm8 a=r5 b=r3 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t1324 mm8 a=r3 b=r0 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t1325 mm8 a=r6 b=r4 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t1326 mm8 a=r4 b=r6 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t1327 mm8 a=r1 b=r3 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t1328 mm8 a=r7 b=r3 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t1329 mm8 a=r3 b=r7 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t1330 mm8 a=r3 b=r6 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t1331 mm8 a=r0 b=r7 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t1332 mm8 a=r0 b=r7 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t1333 mm8 a=r3 b=r3 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t1334 mm8 a=r7 b=r5 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t1335 mm8 a=r5 b=r7 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t1336 mm8 a=r5 b=r5 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t1337 mm8 a=r6 b=r0 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t1338 mm8 a=r6 b=r4 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t1339 mm8 a=r4 b=r6 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t1340 mm8 a=r6 b=r1 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t1341 mm8 a=r0 b=r6 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t1342 mm8 a=r4 b=r6 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t1343 mm8 a=r5 b=r5 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t1344 mm8 a=r2 b=r6 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t1345 mm8 a=r7 b=r3 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t1346 mm8 a=r5 b=r0 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t1347 mm8 a=r2 b=r3 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t1348 mm8 a=r4 b=r3 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t1349 mm8 a=r0 b=r1 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t1350 mm8 a=r1 b=r6 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t1351 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t1352 mm8 a=r5 b=r4 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t1353 mm8 a=r2 b=r7 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t1354 mm8 a=r7 b=r4 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t1355 mm8 a=r7 b=r4 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t1356 mm8 a=r4 b=r6 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t1357 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t1358 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t1359 mm8 a=r7 b=r7 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t1360 mm8 a=r0 b=r4 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t1361 mm8 a=r0 b=r5 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t1362 mm8 a=r0 b=r2 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t1363 mm8 a=r4 b=r4 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t1364 mm8 a=r7 b=r3 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t1365 mm8 a=r2 b=r4 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t1366 mm8 a=r4 b=r3 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t1367 mm8 a=r3 b=r3 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t1368 mm8 a=r4 b=r0 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t1369 mm8 a=r5 b=r3 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t1370 mm8 a=r0 b=r1 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t1371 mm8 a=r0 b=r5 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t1372 mm8 a=r5 b=r6 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t1373 mm8 a=r7 b=r1 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t1374 mm8 a=r4 b=r6 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t1375 mm8 a=r5 b=r2 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t1376 mm8 a=r5 b=r5 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t1377 mm8 a=r7 b=r3 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t1378 mm8 a=r4 b=r1 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t1379 mm8 a=r4 b=r0 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t1380 mm8 a=r0 b=r7 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t1381 mm8 a=r2 b=r0 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t1382 mm8 a=r7 b=r5 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t1383 mm8 a=r1 b=r5 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t1384 mm8 a=r5 b=r2 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t1385 mm8 a=r0 b=r7 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t1386 mm8 a=r7 b=r0 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t1387 mm8 a=r4 b=r3 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t1388 mm8 a=r4 b=r6 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t1389 mm8 a=r4 b=r2 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t1390 mm8 a=r5 b=r1 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t1391 mm8 a=r6 b=r1 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t1392 mm8 a=r5 b=r7 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t1393 mm8 a=r3 b=r4 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t1394 mm8 a=r4 b=r7 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t1395 mm8 a=r7 b=r1 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t1396 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t1397 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t1398 mm8 a=r3 b=r0 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t1399 mm8 a=r3 b=r4 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t1400 mm8 a=r6 b=r4 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t1401 mm8 a=r0 b=r5 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t1402 mm8 a=r7 b=r5 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t1403 mm8 a=r0 b=r0 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t1404 mm8 a=r3 b=r6 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t1405 mm8 a=r0 b=r0 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t1406 mm8 a=r5 b=r3 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t1407 mm8 a=r7 b=r3 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t1408 mm8 a=r4 b=r7 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t1409 mm8 a=r4 b=r2 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t1410 mm8 a=r6 b=r0 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t1411 mm8 a=r1 b=r7 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t1412 mm8 a=r4 b=r3 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t1413 mm8 a=r4 b=r6 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t1414 mm8 a=r0 b=r7 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t1415 mm8 a=r3 b=r6 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t1416 mm8 a=r3 b=r2 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t1417 mm8 a=r2 b=r7 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t1418 mm8 a=r0 b=r6 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t1419 mm8 a=r0 b=r3 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t1420 mm8 a=r1 b=r3 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t1421 mm8 a=r4 b=r0 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t1422 mm8 a=r6 b=r4 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t1423 mm8 a=r4 b=r6 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t1424 mm8 a=r6 b=r3 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t1425 mm8 a=r7 b=r3 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t1426 mm8 a=r0 b=r3 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t1427 mm8 a=r5 b=r6 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t1428 mm8 a=r0 b=r3 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t1429 mm8 a=r7 b=r1 c=r3 c2=r7 - } - uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((ulong)hi << 32) | (ulong)lo; -} diff --git a/proto-cuda/packs-ca4/mx8_mm1430/vectors.h b/proto-cuda/packs-ca4/mx8_mm1430/vectors.h deleted file mode 100644 index 2d3ea39b7..000000000 --- a/proto-cuda/packs-ca4/mx8_mm1430/vectors.h +++ /dev/null @@ -1,57 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// Expected outputs: igneum-pow (Rust) CPU interpreter, generator v2, memory-hard dataset -#pragma once -#ifdef __cplusplus -#include -#else -#include -#endif - -#define IGNEUM_VEC_WARPS 3 -static const uint32_t IGNEUM_VEC_BASE[IGNEUM_VEC_WARPS] = { 0u, 4096u, 1000000u }; -static const uint64_t IGNEUM_VEC_OUT[IGNEUM_VEC_WARPS][32] = { - { // base nonce 0 - 0xf8bf8814e2416cd8ull, 0xc1df90a403316be4ull, 0xbb787d32f45e15b4ull, 0x422e3ecf86983132ull, 0xaed418273c74197dull, 0x88cbdc6fff914ea9ull, 0x52457b80ae5bfd30ull, 0x5aa82c44125405bbull, - 0x241b05accc893061ull, 0x3922a0033166f8a9ull, 0x660fe2c88f23d8b0ull, 0xbdd3700769c59c26ull, 0x13235fa1c27c06a0ull, 0x4a53c774acaca426ull, 0xada3cc0d840d682eull, 0x2e4edea8bc315874ull, - 0x2738fcc54c92addbull, 0x3d3ef6d2f89288f1ull, 0x6bf6a8a963b0c773ull, 0x676e254bd1f6fb80ull, 0xb8d118e2cf40064bull, 0x46f197c31d7109adull, 0x69adbeb0148fc4d0ull, 0xee8f6125b98a9c88ull, - 0xd4815fd2160b2b85ull, 0xd92ca82baf3e355eull, 0xa944dbf6bb30e11eull, 0x5af3c7e2d33b103full, 0xce9fd26dbee50e2dull, 0xbc25610e53326611ull, 0xf41f1ab121cd52b5ull, 0x64c33325a2b5d43bull - }, - { // base nonce 4096 - 0xb0bb9c2f786adf4dull, 0x806ba8d31f4e28d6ull, 0x5a2a835b1dc9d4b3ull, 0x6f28eeeea85118b6ull, 0x63e190b78790a6f3ull, 0x04088b6f953938aaull, 0x82eb1f881ce9ddb5ull, 0x9f91e5c7d996c7cfull, - 0xbe0355dd8d714908ull, 0x344b05f526076ceeull, 0x81ebf0504e3a9ae2ull, 0x6c522256a53ec4b4ull, 0x1ece11df1618a118ull, 0xb10ded4ba9f77544ull, 0xf8d57b6d96fd61eaull, 0xd114be73fdb26740ull, - 0x0be7112c4384a1c2ull, 0xa2f11d4a4c0c302aull, 0xfcd2146ec5fef56dull, 0xb0236e3a4cee2725ull, 0xc4e78111f99ada58ull, 0x9ab2b272ad18b1e7ull, 0x4eb50b5eda7970d0ull, 0x4dda2812d7105002ull, - 0xa66f5ff9e123030full, 0x25c689a3d95794ccull, 0xd8264dd83a7e9d0full, 0x13aac7727f2307ccull, 0xd2539ea11d360940ull, 0x80760547b9bf48a6ull, 0xd3a86440129515d8ull, 0xc10a14c2898a6e66ull - }, - { // base nonce 1000000 - 0x9fa775319d068f6eull, 0xf9053cdb1e27f106ull, 0x55e9096bbe852422ull, 0xbffd11bade2c11f4ull, 0x2c14ec73bdcd0c03ull, 0x2c838a4d3aa8463full, 0x420f94c19067fcc8ull, 0xb33d361bd93e8dbeull, - 0x64d4280f4d04ec98ull, 0x105c4e6b978f07d9ull, 0x1545cac655298f62ull, 0x8a2d5c71ec80c66dull, 0x211865bfbaf48819ull, 0x3c8de09457bb16bdull, 0x288d86d556917150ull, 0x44c3655c6cdff067ull, - 0x7bc3d9f9785b5ddcull, 0x6b9db79ebdddb1c9ull, 0x7c78589466245632ull, 0x8a958a50a9e482ddull, 0xf283899605afb637ull, 0x9ca6069af0fedd82ull, 0x32adcd4cb7d6129cull, 0xbd99077ff9750508ull, - 0xe2cdbb09d4ccb545ull, 0x2f8ca3d3d24f0d64ull, 0xc5293342a005568full, 0xf1427609c2e26234ull, 0x0d942b528f7cde27ull, 0x2f1dde333e26d338ull, 0x53a9d5353b5c71c5ull, 0x05278e408e3529d9ull - } -}; - -// Dataset self-test: dataset[0..15] and dataset[IGNEUM_MASK] (268435455). -static const uint32_t IGNEUM_DS_HEAD[16] = { - 0xfdad4319u, 0x1a7b68e1u, 0xde6db608u, 0x13d73892u, 0xd17f447au, 0xb2221ccfu, 0x9db004bdu, 0x57d7d367u, - 0xdbc4cf34u, 0x697c009au, 0xc43af1d4u, 0x97f12b2eu, 0x74c37cd0u, 0xc651ea15u, 0x665a6d29u, 0x22330a2du -}; -static const uint32_t IGNEUM_DS_LAST_INDEX = 268435455u; -static const uint32_t IGNEUM_DS_LAST = 0xa83e7aa6u; -// 64 sampled dataset words (index, value) computed on the Mac. -#define IGNEUM_DS_SAMPLES 64 -static const uint32_t IGNEUM_DS_SAMPLE_INDEX[IGNEUM_DS_SAMPLES] = { - 59471966u, 217795994u, 208353206u, 42483309u, 172547758u, 148076330u, 183853158u, 214389424u, 267488061u, 169781097u, 184093494u, 153880993u, 84977930u, 46426879u, 3093825u, 225364072u, 44593546u, 260713159u, 168250303u, 52384140u, 223401610u, 45554030u, 95410555u, 175039924u, 79171087u, 267580473u, 24168642u, 37981670u, 171551130u, 195559979u, 204611762u, 140997658u, 138925853u, 86637313u, 20736778u, 219665210u, 160430336u, 264654675u, 8013395u, 228945585u, 213884386u, 104419827u, 44185464u, 142737231u, 99284897u, 132475900u, 61861762u, 132056166u, 262388043u, 91878046u, 117353561u, 124768597u, 71352993u, 190698941u, 46055428u, 55281366u, 165145231u, 106810753u, 171985651u, 232085256u, 159510492u, 40072060u, 209107596u, 39023794u -}; -static const uint32_t IGNEUM_DS_SAMPLE_VALUE[IGNEUM_DS_SAMPLES] = { - 0x3230bc7bu, 0x7fbfe2c9u, 0xb2690991u, 0x1745c7c5u, 0x0ab0ccafu, 0x1bf87d6bu, 0x160139fdu, 0x719817acu, 0x0155df4bu, 0xbe1e86c3u, 0x680bcd6cu, 0x79c3dc6cu, 0x181e7e5fu, 0x0713a109u, 0xc705dd9fu, 0x3933b7a8u, 0xdd1c0431u, 0x50522b30u, 0xa0020b38u, 0xbff39e96u, 0x21b67e18u, 0x740f8db3u, 0x2baba568u, 0x2c9bef83u, 0x0ad9b671u, 0xc4327869u, 0x7b4fd7d0u, 0x2c29965fu, 0xec56f15fu, 0x61111746u, 0x303a1d6eu, 0xbddcfd1au, 0xf829a355u, 0x6d5df2a9u, 0x01ab8e44u, 0x06d13507u, 0xda8dcfc6u, 0x01a703e1u, 0xafe7d2c1u, 0xc091c3a2u, 0xac1814feu, 0x6e6ff62au, 0x8fdf01bau, 0xdd3f7159u, 0xdfa0d75cu, 0x26684c35u, 0x7f441e63u, 0x88df2570u, 0x8aa4d5ebu, 0xcc816c05u, 0x434df890u, 0xcd392ad6u, 0x1ab4cb63u, 0x595926fau, 0x7cd76b41u, 0x20cb95c4u, 0x13cf823fu, 0xf9daf901u, 0xff9af40au, 0x2c7dfa51u, 0x871206dbu, 0x938c116cu, 0xb64bf199u, 0x5751f874u -}; -// Cache self-test (memory-hard mode): cache[0..15], the last 16 words, and FNV-1a 64 over all 2^26 words. -static const uint32_t IGNEUM_CACHE_HEAD[16] = { - 0x355a86d2u, 0x7957db1cu, 0xd21772afu, 0x6fc1e09bu, 0xd55ce61du, 0x6e6a278bu, 0xd3f543ceu, 0x223d8e82u, - 0x143ab337u, 0x2e9f05bdu, 0x2eb389bfu, 0x0c6e449eu, 0x5cfa4222u, 0xba6560feu, 0x8e3e1aa4u, 0xdbcc1d53u -}; -static const uint32_t IGNEUM_CACHE_LAST[16] = { - 0x41190d91u, 0xbd277957u, 0x22ddbb49u, 0x6986f207u, 0xdf69a4d6u, 0x26401a3au, 0x818230fbu, 0xc417122du, - 0x3597b211u, 0xb553ce55u, 0xcf39cc0du, 0x3b7fc43au, 0x3fd43b00u, 0x67e1c80eu, 0xffa7ea7du, 0xca2960abu -}; -static const uint64_t IGNEUM_CACHE_FNV64 = 0x48c4f5bf24166b2eull; diff --git a/proto-cuda/packs-ca4/mx8_mm1430/vectors.json b/proto-cuda/packs-ca4/mx8_mm1430/vectors.json deleted file mode 100644 index b122b0512..000000000 --- a/proto-cuda/packs-ca4/mx8_mm1430/vectors.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "seed": "igneum-genesis", - "day": "2026-10-03", - "dataset_mode": "memory-hard", - "dataset_log2_words": 28, - "mask": "0x0fffffff", - "lanes": 32, - "source": "igneum-pow (Rust) CPU interpreter, generator v2, memory-hard dataset", - "warps": [ - {"base_nonce": 0, "expected": [ - "0xf8bf8814e2416cd8", "0xc1df90a403316be4", "0xbb787d32f45e15b4", "0x422e3ecf86983132", "0xaed418273c74197d", "0x88cbdc6fff914ea9", "0x52457b80ae5bfd30", "0x5aa82c44125405bb", - "0x241b05accc893061", "0x3922a0033166f8a9", "0x660fe2c88f23d8b0", "0xbdd3700769c59c26", "0x13235fa1c27c06a0", "0x4a53c774acaca426", "0xada3cc0d840d682e", "0x2e4edea8bc315874", - "0x2738fcc54c92addb", "0x3d3ef6d2f89288f1", "0x6bf6a8a963b0c773", "0x676e254bd1f6fb80", "0xb8d118e2cf40064b", "0x46f197c31d7109ad", "0x69adbeb0148fc4d0", "0xee8f6125b98a9c88", - "0xd4815fd2160b2b85", "0xd92ca82baf3e355e", "0xa944dbf6bb30e11e", "0x5af3c7e2d33b103f", "0xce9fd26dbee50e2d", "0xbc25610e53326611", "0xf41f1ab121cd52b5", "0x64c33325a2b5d43b" - ]}, - {"base_nonce": 4096, "expected": [ - "0xb0bb9c2f786adf4d", "0x806ba8d31f4e28d6", "0x5a2a835b1dc9d4b3", "0x6f28eeeea85118b6", "0x63e190b78790a6f3", "0x04088b6f953938aa", "0x82eb1f881ce9ddb5", "0x9f91e5c7d996c7cf", - "0xbe0355dd8d714908", "0x344b05f526076cee", "0x81ebf0504e3a9ae2", "0x6c522256a53ec4b4", "0x1ece11df1618a118", "0xb10ded4ba9f77544", "0xf8d57b6d96fd61ea", "0xd114be73fdb26740", - "0x0be7112c4384a1c2", "0xa2f11d4a4c0c302a", "0xfcd2146ec5fef56d", "0xb0236e3a4cee2725", "0xc4e78111f99ada58", "0x9ab2b272ad18b1e7", "0x4eb50b5eda7970d0", "0x4dda2812d7105002", - "0xa66f5ff9e123030f", "0x25c689a3d95794cc", "0xd8264dd83a7e9d0f", "0x13aac7727f2307cc", "0xd2539ea11d360940", "0x80760547b9bf48a6", "0xd3a86440129515d8", "0xc10a14c2898a6e66" - ]}, - {"base_nonce": 1000000, "expected": [ - "0x9fa775319d068f6e", "0xf9053cdb1e27f106", "0x55e9096bbe852422", "0xbffd11bade2c11f4", "0x2c14ec73bdcd0c03", "0x2c838a4d3aa8463f", "0x420f94c19067fcc8", "0xb33d361bd93e8dbe", - "0x64d4280f4d04ec98", "0x105c4e6b978f07d9", "0x1545cac655298f62", "0x8a2d5c71ec80c66d", "0x211865bfbaf48819", "0x3c8de09457bb16bd", "0x288d86d556917150", "0x44c3655c6cdff067", - "0x7bc3d9f9785b5ddc", "0x6b9db79ebdddb1c9", "0x7c78589466245632", "0x8a958a50a9e482dd", "0xf283899605afb637", "0x9ca6069af0fedd82", "0x32adcd4cb7d6129c", "0xbd99077ff9750508", - "0xe2cdbb09d4ccb545", "0x2f8ca3d3d24f0d64", "0xc5293342a005568f", "0xf1427609c2e26234", "0x0d942b528f7cde27", "0x2f1dde333e26d338", "0x53a9d5353b5c71c5", "0x05278e408e3529d9" - ]} - ], - "dataset_head": ["0xfdad4319", "0x1a7b68e1", "0xde6db608", "0x13d73892", "0xd17f447a", "0xb2221ccf", "0x9db004bd", "0x57d7d367", "0xdbc4cf34", "0x697c009a", "0xc43af1d4", "0x97f12b2e", "0x74c37cd0", "0xc651ea15", "0x665a6d29", "0x22330a2d"], - "dataset_last_index": 268435455, - "dataset_last": "0xa83e7aa6", - "dataset_samples": [{"index": 59471966, "value": "0x3230bc7b"}, {"index": 217795994, "value": "0x7fbfe2c9"}, {"index": 208353206, "value": "0xb2690991"}, {"index": 42483309, "value": "0x1745c7c5"}, {"index": 172547758, "value": "0x0ab0ccaf"}, {"index": 148076330, "value": "0x1bf87d6b"}, {"index": 183853158, "value": "0x160139fd"}, {"index": 214389424, "value": "0x719817ac"}, {"index": 267488061, "value": "0x0155df4b"}, {"index": 169781097, "value": "0xbe1e86c3"}, {"index": 184093494, "value": "0x680bcd6c"}, {"index": 153880993, "value": "0x79c3dc6c"}, {"index": 84977930, "value": "0x181e7e5f"}, {"index": 46426879, "value": "0x0713a109"}, {"index": 3093825, "value": "0xc705dd9f"}, {"index": 225364072, "value": "0x3933b7a8"}, {"index": 44593546, "value": "0xdd1c0431"}, {"index": 260713159, "value": "0x50522b30"}, {"index": 168250303, "value": "0xa0020b38"}, {"index": 52384140, "value": "0xbff39e96"}, {"index": 223401610, "value": "0x21b67e18"}, {"index": 45554030, "value": "0x740f8db3"}, {"index": 95410555, "value": "0x2baba568"}, {"index": 175039924, "value": "0x2c9bef83"}, {"index": 79171087, "value": "0x0ad9b671"}, {"index": 267580473, "value": "0xc4327869"}, {"index": 24168642, "value": "0x7b4fd7d0"}, {"index": 37981670, "value": "0x2c29965f"}, {"index": 171551130, "value": "0xec56f15f"}, {"index": 195559979, "value": "0x61111746"}, {"index": 204611762, "value": "0x303a1d6e"}, {"index": 140997658, "value": "0xbddcfd1a"}, {"index": 138925853, "value": "0xf829a355"}, {"index": 86637313, "value": "0x6d5df2a9"}, {"index": 20736778, "value": "0x01ab8e44"}, {"index": 219665210, "value": "0x06d13507"}, {"index": 160430336, "value": "0xda8dcfc6"}, {"index": 264654675, "value": "0x01a703e1"}, {"index": 8013395, "value": "0xafe7d2c1"}, {"index": 228945585, "value": "0xc091c3a2"}, {"index": 213884386, "value": "0xac1814fe"}, {"index": 104419827, "value": "0x6e6ff62a"}, {"index": 44185464, "value": "0x8fdf01ba"}, {"index": 142737231, "value": "0xdd3f7159"}, {"index": 99284897, "value": "0xdfa0d75c"}, {"index": 132475900, "value": "0x26684c35"}, {"index": 61861762, "value": "0x7f441e63"}, {"index": 132056166, "value": "0x88df2570"}, {"index": 262388043, "value": "0x8aa4d5eb"}, {"index": 91878046, "value": "0xcc816c05"}, {"index": 117353561, "value": "0x434df890"}, {"index": 124768597, "value": "0xcd392ad6"}, {"index": 71352993, "value": "0x1ab4cb63"}, {"index": 190698941, "value": "0x595926fa"}, {"index": 46055428, "value": "0x7cd76b41"}, {"index": 55281366, "value": "0x20cb95c4"}, {"index": 165145231, "value": "0x13cf823f"}, {"index": 106810753, "value": "0xf9daf901"}, {"index": 171985651, "value": "0xff9af40a"}, {"index": 232085256, "value": "0x2c7dfa51"}, {"index": 159510492, "value": "0x871206db"}, {"index": 40072060, "value": "0x938c116c"}, {"index": 209107596, "value": "0xb64bf199"}, {"index": 39023794, "value": "0x5751f874"}], - "cache_head": ["0x355a86d2", "0x7957db1c", "0xd21772af", "0x6fc1e09b", "0xd55ce61d", "0x6e6a278b", "0xd3f543ce", "0x223d8e82", "0x143ab337", "0x2e9f05bd", "0x2eb389bf", "0x0c6e449e", "0x5cfa4222", "0xba6560fe", "0x8e3e1aa4", "0xdbcc1d53"], - "cache_last_line": ["0x41190d91", "0xbd277957", "0x22ddbb49", "0x6986f207", "0xdf69a4d6", "0x26401a3a", "0x818230fb", "0xc417122d", "0x3597b211", "0xb553ce55", "0xcf39cc0d", "0x3b7fc43a", "0x3fd43b00", "0x67e1c80e", "0xffa7ea7d", "0xca2960ab"], - "cache_fnv1a64": "0x48c4f5bf24166b2e" -} diff --git a/proto-cuda/packs-ca4/mx8_mm512/kernel.cl b/proto-cuda/packs-ca4/mx8_mm512/kernel.cl deleted file mode 100644 index 2a9482a12..000000000 --- a/proto-cuda/packs-ca4/mx8_mm512/kernel.cl +++ /dev/null @@ -1,799 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// OpenCL C twin of the Metal kernel for the same seed (see proto-opencl/README.md, WAVEFRONT.md and program.metal). -// Built from source at runtime by proto-opencl/host.c, which passes these defines: -// IGNEUM_GROUP work-group size of igneum_hash, a multiple of 32 (default 32: one work-group = one 32-lane unit) -// IGNEUM_EXCHANGE 0 = local-memory exchange with a barrier (any device, any wave width; the default) -// 1 = sub_group_shuffle_xor (cl_khr_subgroup_shuffle), only with IGNEUM_GROUP 32 and a sub-group size of exactly 32 -// 2 = intel_sub_group_shuffle_xor (cl_intel_subgroups), same condition -// The verification unit is always 32 lanes. A 64-wide hardware wave (AMD GCN/CDNA, RDNA in wave64) runs two units; -// the exchange masks are 1, 2, 4, 8, 16, so every partner lane lies inside the lane's own aligned run of 32. -#ifndef IGNEUM_GROUP -#define IGNEUM_GROUP 32 -#endif -#ifndef IGNEUM_EXCHANGE -#define IGNEUM_EXCHANGE 0 -#endif -#ifdef __OPENCL_VERSION__ -#define IGNEUM_KERNEL_HASH __kernel __attribute__((reqd_work_group_size(IGNEUM_GROUP, 1, 1))) -#define IGNEUM_LOCAL_WORDS(name, n) __local uint name[n] -#if IGNEUM_EXCHANGE == 1 -#ifdef cl_khr_subgroups -#pragma OPENCL EXTENSION cl_khr_subgroups : enable -#endif -#ifdef cl_khr_subgroup_shuffle -#pragma OPENCL EXTENSION cl_khr_subgroup_shuffle : enable -#endif -#elif IGNEUM_EXCHANGE == 2 -#pragma OPENCL EXTENSION cl_intel_subgroups : enable -#endif -#else -// Not an OpenCL compiler: proto-opencl/emu compiles this file as C++ and supplies the built-ins and these two macros. -#include "emu_opencl.h" -#endif - -#if IGNEUM_EXCHANGE == 1 -#define IGNEUM_SHFL_XOR(dst, a, m) dst = sub_group_shuffle_xor((a), (uint)(m)) -#define IGNEUM_BCAST0(dst, a) dst = sub_group_broadcast((a), 0u) -#define IGNEUM_SHFL_IDX(dst, a, i) dst = sub_group_shuffle((a), (uint)(i)) -#elif IGNEUM_EXCHANGE == 2 -#define IGNEUM_SHFL_XOR(dst, a, m) dst = intel_sub_group_shuffle_xor((a), (uint)(m)) -#define IGNEUM_BCAST0(dst, a) dst = sub_group_broadcast((a), 0u) -#define IGNEUM_SHFL_IDX(dst, a, i) dst = intel_sub_group_shuffle((a), (uint)(i)) -#else -// Local-memory exchange. Two buffers of IGNEUM_GROUP words alternate (xk counts exchanges), so one barrier per -// exchange is enough: a lane can only overwrite buffer b at exchange k+2 after passing barrier k+1, and every lane -// reaches barrier k+1 only after its read of buffer b at exchange k. The partner lid ^ m stays inside the lane's -// aligned run of 32 because m < 32. Control flow is uniform, so every work-item reaches every barrier. -#define IGNEUM_SHFL_XOR(dst, a, m) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid ^ (uint)(m))]; xk += 1u; } -#define IGNEUM_BCAST0(dst, a) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid & ~31u)]; xk += 1u; } -#define IGNEUM_SHFL_IDX(dst, a, i) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid & ~31u) + (uint)(i)]; xk += 1u; } -#endif - -// int8 tile reference (Counter ASIC 4.0 research): 12 index shuffles and 32 byte products per lane; the PTX mma.m8n8k16 u8 layout. -#define IGNEUM_MM8_REF(av, bv, d0, d1) { uint row_ = lane >> 2, col0_ = 2u * (lane & 3u); uint acc0_ = 0u, acc1_ = 0u; for (uint j_ = 0u; j_ < 4u; ++j_) { uint aw_, bw0_, bw1_; IGNEUM_SHFL_IDX(aw_, (av), 4u * row_ + j_); IGNEUM_SHFL_IDX(bw0_, (bv), 4u * col0_ + j_); IGNEUM_SHFL_IDX(bw1_, (bv), 4u * (col0_ + 1u) + j_); for (uint t_ = 0u; t_ < 4u; ++t_) { uint ab_ = (aw_ >> (8u * t_)) & 0xffu; acc0_ += ab_ * ((bw0_ >> (8u * t_)) & 0xffu); acc1_ += ab_ * ((bw1_ >> (8u * t_)) & 0xffu); } } d0 = acc0_; d1 = acc1_; } - -static inline uint splitmix32(uint x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -// n is a literal in 1..31 at every call site. OpenCL rotate() rotates left by n modulo 32. -static inline uint rotl_imm(uint x, uint n) { return rotate(x, n); } -// Right rotation by n modulo 32 as a left rotation by (32 - n) modulo 32; n == 0 gives x. -static inline uint rotr_var(uint x, uint n) { return rotate(x, (0u - n) & 31u); } -static inline uint ds_elem(uint i, uint d0, uint d1) { - uint x = i ^ d0; - x *= 0x9E3779B1u; x ^= x >> 15; - x += d1; - x *= 0x85EBCA77u; x ^= x >> 13; - x *= 0xC2B2AE3Du; x ^= x >> 16; - return x; -} - -// Memory-hard dataset core (MEMHARD.md). Cache: 2^26 words in 2^16 segments of 64 chained ChaCha12 lines. -// Item: 8 rounds of 8 x seed-parameterised mixer + one 64-byte cache read, then 8 x final mixer (class v3, mixer multiplier 8, -// docs/plans/mixer-x4.md: the round key of application j of round r is 0x9E3779B9 * (r * m + j + 1)). All parameters are literals. -#define MH_CACHE_LINE_MASK 0x003fffffu -#define MH_SEGMENT_LINES 64u -#define MH_QR(a, b, c, d, r1, r2, r3, r4) { a += b; d ^= a; d = mh_rotl(d, r1); c += d; b ^= c; b = mh_rotl(b, r2); a += b; d ^= a; d = mh_rotl(d, r3); c += d; b ^= c; b = mh_rotl(b, r4); } -static inline uint mh_rotl(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 at every call site - -// y = ChaCha12 core(x) + x -static inline void mh_chacha_block(const uint* x, uint* y) { - for (uint i = 0u; i < 16u; ++i) y[i] = x[i]; - for (uint r = 0u; r < 6u; ++r) { - MH_QR(y[0], y[4], y[8], y[12], 16u, 12u, 8u, 7u) MH_QR(y[1], y[5], y[9], y[13], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[6], y[10], y[14], 16u, 12u, 8u, 7u) MH_QR(y[3], y[7], y[11], y[15], 16u, 12u, 8u, 7u) - MH_QR(y[0], y[5], y[10], y[15], 16u, 12u, 8u, 7u) MH_QR(y[1], y[6], y[11], y[12], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[7], y[8], y[13], 16u, 12u, 8u, 7u) MH_QR(y[3], y[4], y[9], y[14], 16u, 12u, 8u, 7u) - } - for (uint i = 0u; i < 16u; ++i) y[i] += x[i]; -} - -// One cache segment: 64 chained lines written at cache[seg * 1024]. in_j = prev ^ (sigma || K || seg || j || tag), prev_0 = 0. -static inline void mh_cache_segment(__global uint* cache, uint seg) { - uint prev[16]; uint x[16]; uint y[16]; - for (uint i = 0u; i < 16u; ++i) prev[i] = 0u; - for (uint j = 0u; j < MH_SEGMENT_LINES; ++j) { - x[0] = 0x61707865u ^ prev[0]; x[1] = 0x3320646eu ^ prev[1]; x[2] = 0x79622d32u ^ prev[2]; x[3] = 0x6b206574u ^ prev[3]; - x[4] = 0x3067619fu ^ prev[4]; - x[5] = 0x3c269176u ^ prev[5]; - x[6] = 0x84a03b03u ^ prev[6]; - x[7] = 0xf8c63294u ^ prev[7]; - x[8] = 0xff977c5bu ^ prev[8]; - x[9] = 0xe60def3eu ^ prev[9]; - x[10] = 0x63630141u ^ prev[10]; - x[11] = 0xb8fbcb58u ^ prev[11]; - x[12] = seg ^ prev[12]; x[13] = j ^ prev[13]; x[14] = 0x49676e65u ^ prev[14]; x[15] = 0x756d4d48u ^ prev[15]; - mh_chacha_block(x, y); - __global uint* line = cache + ((seg * MH_SEGMENT_LINES + j) * 16u); - for (uint i = 0u; i < 16u; ++i) { line[i] = y[i]; prev[i] = y[i]; } - } -} - -// M_r: per word (s ^ (RC + rk)) * MUL, then a column round and a diagonal round with the seed-drawn rotations. -static inline void mh_mixer(uint* s, uint rk) { - s[0] = (s[0] ^ (0xbab68293u + rk)) * 0x42146205u; - s[1] = (s[1] ^ (0xcc162340u + rk)) * 0x52cbe0fbu; - s[2] = (s[2] ^ (0x6ce151ccu + rk)) * 0x7ecf4a03u; - s[3] = (s[3] ^ (0xe62b8997u + rk)) * 0x6728907fu; - s[4] = (s[4] ^ (0xc9c80297u + rk)) * 0xd81d9751u; - s[5] = (s[5] ^ (0xf74a1654u + rk)) * 0x132952c3u; - s[6] = (s[6] ^ (0x3d704af5u + rk)) * 0xf60de277u; - s[7] = (s[7] ^ (0x3cf522b7u + rk)) * 0x05358035u; - s[8] = (s[8] ^ (0x2b9cac04u + rk)) * 0xbaf6499du; - s[9] = (s[9] ^ (0xa880ac10u + rk)) * 0xe4db9667u; - s[10] = (s[10] ^ (0x13e5dd1du + rk)) * 0x3e98f45du; - s[11] = (s[11] ^ (0x6fc3e233u + rk)) * 0xd0004eddu; - s[12] = (s[12] ^ (0x2d83eeacu + rk)) * 0x2691630du; - s[13] = (s[13] ^ (0x9006e8bfu + rk)) * 0x9beb3bcfu; - s[14] = (s[14] ^ (0x2c4b5362u + rk)) * 0xab310379u; - s[15] = (s[15] ^ (0x31b49ee2u + rk)) * 0x99cfb423u; - MH_QR(s[0], s[4], s[8], s[12], 20u, 20u, 19u, 4u) MH_QR(s[1], s[5], s[9], s[13], 20u, 20u, 19u, 4u) - MH_QR(s[2], s[6], s[10], s[14], 20u, 20u, 19u, 4u) MH_QR(s[3], s[7], s[11], s[15], 20u, 20u, 19u, 4u) - MH_QR(s[0], s[5], s[10], s[15], 26u, 3u, 3u, 27u) MH_QR(s[1], s[6], s[11], s[12], 26u, 3u, 3u, 27u) - MH_QR(s[2], s[7], s[8], s[13], 26u, 3u, 3u, 27u) MH_QR(s[3], s[4], s[9], s[14], 26u, 3u, 3u, 27u) -} - -// Item t: 16 words. s = (K, t * MUL[i] + RC[i]); 8 rounds of 8 x mixer + cache line s[0] & mask; 8 x final mixer. -static inline void mh_item(__global const uint* cache, uint t, uint* s) { - s[0] = 0x3067619fu; - s[1] = 0x3c269176u; - s[2] = 0x84a03b03u; - s[3] = 0xf8c63294u; - s[4] = 0xff977c5bu; - s[5] = 0xe60def3eu; - s[6] = 0x63630141u; - s[7] = 0xb8fbcb58u; - s[8] = t * 0x42146205u + 0xbab68293u; - s[9] = t * 0x52cbe0fbu + 0xcc162340u; - s[10] = t * 0x7ecf4a03u + 0x6ce151ccu; - s[11] = t * 0x6728907fu + 0xe62b8997u; - s[12] = t * 0xd81d9751u + 0xc9c80297u; - s[13] = t * 0x132952c3u + 0xf74a1654u; - s[14] = t * 0xf60de277u + 0x3d704af5u; - s[15] = t * 0x05358035u + 0x3cf522b7u; - for (uint r = 0u; r < 8u; ++r) { - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (r * 8u + j + 1u)); - __global const uint* line = cache + ((s[0] & MH_CACHE_LINE_MASK) * 16u); - for (uint i = 0u; i < 16u; ++i) s[i] ^= line[i]; - } - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (64u + j + 1u)); -} -// dataset[w] without the dataset: derive item w >> 4 and take word w & 15. -static inline uint mh_word(__global const uint* cache, uint w) { uint s[16]; mh_item(cache, w >> 4u, s); return s[w & 15u]; } - -// Memory-hard dataset (MEMHARD.md). One work-item per cache segment; one work-item per 64-byte dataset item. -// The same constants as memhard.h in this pack (one emitter, three dialects). -__kernel void igneum_cache_fill(__global uint* cache, uint nSegments) { - uint seg = (uint)get_global_id(0); - if (seg < nSegments) mh_cache_segment(cache, seg); -} -__kernel void igneum_build(__global uint* ds, __global const uint* cache, uint nItems) { - uint t = (uint)get_global_id(0); - if (t < nItems) { - uint s[16]; - mh_item(cache, t, s); - __global uint* d = ds + ((ulong)t * 16u); - for (uint i = 0u; i < 16u; ++i) d[i] = s[i]; - } -} - -// One hash per work-item. IGNEUM_GROUP is a multiple of 32; lane = lid & 31 and every exchange stays inside the -// lane's own aligned run of 32 work-items, exactly like simd_shuffle_xor inside a 32-wide Metal SIMD group and -// __shfl_xor_sync inside a CUDA warp. Control flow is uniform (no branches at all). -IGNEUM_KERNEL_HASH void igneum_hash(__global const uint* ds, __global ulong* out, uint baseNonce, uint mask) { - uint gid = (uint)get_global_id(0); - uint lid = (uint)get_local_id(0); - uint nonce = baseNonce + gid; - uint r0, r1, r2, r3, r4, r5, r6, r7; -#if IGNEUM_EXCHANGE == 0 - IGNEUM_LOCAL_WORDS(xch, 2 * IGNEUM_GROUP); - uint xk = 0u; -#else - (void)lid; -#endif - uint lane = lid & 31u; - { uint x = nonce ^ 0x67a9a7beu; x += 0x9e3779b9u; x = splitmix32(x); r0 = x ^ 0x1a155b25u; } // SEEDW[0], 0x9e3779b9u * 1u, SEEDW[1] - { uint x = nonce ^ 0x1a155b25u; x += 0x3c6ef372u; x = splitmix32(x); r1 = x ^ 0xfddfb732u; } // SEEDW[1], 0x9e3779b9u * 2u, SEEDW[2] - { uint x = nonce ^ 0xfddfb732u; x += 0xdaa66d2bu; x = splitmix32(x); r2 = x ^ 0x4b5af2e8u; } // SEEDW[2], 0x9e3779b9u * 3u, SEEDW[3] - { uint x = nonce ^ 0x4b5af2e8u; x += 0x78dde6e4u; x = splitmix32(x); r3 = x ^ 0xc55caf33u; } // SEEDW[3], 0x9e3779b9u * 4u, SEEDW[4] - { uint x = nonce ^ 0xc55caf33u; x += 0x1715609du; x = splitmix32(x); r4 = x ^ 0xa27c13b7u; } // SEEDW[4], 0x9e3779b9u * 5u, SEEDW[5] - { uint x = nonce ^ 0xa27c13b7u; x += 0xb54cda56u; x = splitmix32(x); r5 = x ^ 0x06628a48u; } // SEEDW[5], 0x9e3779b9u * 6u, SEEDW[6] - { uint x = nonce ^ 0x06628a48u; x += 0x5384540fu; x = splitmix32(x); r6 = x ^ 0x03852469u; } // SEEDW[6], 0x9e3779b9u * 7u, SEEDW[7] - { uint x = nonce ^ 0x03852469u; x += 0xf1bbcdc8u; x = splitmix32(x); r7 = x ^ 0x67a9a7beu; } // SEEDW[7], 0x9e3779b9u * 8u, SEEDW[0] - - for (uint it = 0u; it < 8u; ++it) { - uint sel = r0; - r2 = r3 * r4 + r2; // 0 mad - r2 = r1 * r1 + r2; // 1 mad - r2 = r3 * r2 + r2; // 2 mad - r3 = r3 ^ r5; // 3 xor - r7 = r7 ^ ds[r2 & mask]; // 4 load - r5 = r5 ^ ds[r7 & mask]; // 5 load - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 8u); r1 = r1 ^ t_; } // 6 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 8u); r7 = r7 ^ t_; } // 7 shfl - r1 = mul_hi(r1, r5); // 8 mulhi - r6 = rotr_var(r6, r3); // 9 rotr - r3 = r3 | r4; // 10 or - r4 = r4 ^ ds[r3 & mask]; // 11 load - r0 = mul_hi(r0, r4); // 12 mulhi - r5 = r5 + r1 + ((((sel >> 30u) & 1u) != 0u) ? 0xd3177981u : 0xc7934706u); // 13 add - r0 = r0 ^ ds[r4 & mask]; // 14 load - r2 = r2 - r4; // 15 sub - r2 = r2 ^ ds[r0 & mask]; // 16 load - r7 = r7 ^ ds[r2 & mask]; // 17 load - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 4u); r7 = r7 ^ t_; } // 18 shfl - r5 = r5 * r0; // 19 mul - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 2u); r3 = r3 ^ t_; } // 20 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 16u); r2 = r2 ^ t_; } // 21 shfl - r6 = mul_hi(r6, r2); // 22 mulhi - r6 = r6 ^ ds[r1 & mask]; // 23 load - r5 = r5 * r0; // 24 mul - r5 = rotl_imm(r5, 19u); // 25 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 2u); r7 = r7 ^ t_; } // 26 shfl - r0 = r0 ^ r5; // 27 xor - r0 = r0 ^ r4; // 28 xor - r3 = r3 - r0; // 29 sub - r5 = r5 * r1; // 30 mul - r7 = r7 ^ ds[r2 & mask]; // 31 load - r1 = r1 ^ ds[r0 & mask]; // 32 load - r5 = r5 ^ r6; // 33 xor - r5 = r5 ^ ds[r1 & mask]; // 34 load - r0 = mul_hi(r0, r5); // 35 mulhi - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 4u); r5 = r5 ^ t_; } // 36 shfl - r7 = r7 ^ ds[r0 & mask]; // 37 load - r3 = r3 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0x230c005cu : 0x75ba2fadu); // 38 add - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 4u); r1 = r1 ^ t_; } // 39 shfl - r2 = r2 ^ r5; // 40 xor - r3 = r6 * r3 + r3; // 41 mad - r6 = r6 - r7; // 42 sub - r7 = r7 ^ r0; // 43 xor - r1 = r1 ^ ds[r7 & mask]; // 44 load - r2 = r2 * r3; // 45 mul - r1 = mul_hi(r1, r5); // 46 mulhi - r4 = r4 - r3; // 47 sub - r2 = rotr_var(r2, r6); // 48 rotr - r3 = r3 ^ ds[r5 & mask]; // 49 load - r1 = r1 + r5 + ((((sel >> 7u) & 1u) != 0u) ? 0x1907970cu : 0x81b8bc2cu); // 50 add - r0 = r0 * r2; // 51 mul - r0 = r0 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x699fd448u : 0x4f92b968u); // 52 add - r1 = r1 + r0 + ((((sel >> 12u) & 1u) != 0u) ? 0x77b1520du : 0x2bb965afu); // 53 add - r7 = rotl_imm(r7, 14u); // 54 rotl - r3 = r3 + r7 + ((((sel >> 1u) & 1u) != 0u) ? 0xa54c55a0u : 0x7b0fe07au); // 55 add - r6 = r6 ^ ds[r7 & mask]; // 56 load - r1 = rotr_var(r1, r5); // 57 rotr - r5 = r5 ^ ds[r4 & mask]; // 58 load - r6 = r6 ^ ds[r2 & mask]; // 59 load - r3 = r5 * r0 + r3; // 60 mad - r5 = r5 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0xad7493e7u : 0xaf9dd72du); // 61 add - r4 = r4 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x1e07c3d9u : 0x89841d87u); // 62 add - r5 = rotl_imm(r5, 19u); // 63 rotl - // int8 tile block (Counter ASIC 4.0 research, experimental): 512 mma.m8n8k16 u8 tiles per iteration, both outputs consumed - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r2 += d0_; r3 += d1_; } // t0 mm8 a=r6 b=r5 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r4 += d0_; r5 += d1_; } // t1 mm8 a=r1 b=r1 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r0 += d0_; r7 += d1_; } // t2 mm8 a=r2 b=r5 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r3 += d1_; } // t3 mm8 a=r1 b=r5 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r2 += d0_; r6 += d1_; } // t4 mm8 a=r2 b=r0 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r7 += d0_; r3 += d1_; } // t5 mm8 a=r2 b=r6 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t6 mm8 a=r1 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r0 += d0_; r4 += d1_; } // t7 mm8 a=r4 b=r6 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r1 += d0_; r5 += d1_; } // t8 mm8 a=r1 b=r1 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r4 += d0_; r6 += d1_; } // t9 mm8 a=r3 b=r7 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r0 += d0_; r4 += d1_; } // t10 mm8 a=r3 b=r0 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r0 += d0_; r6 += d1_; } // t11 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r6 += d0_; r0 += d1_; } // t12 mm8 a=r3 b=r1 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r5 += d1_; } // t13 mm8 a=r1 b=r5 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r3 += d0_; r4 += d1_; } // t14 mm8 a=r3 b=r3 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r5 += d0_; r4 += d1_; } // t15 mm8 a=r1 b=r0 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r0 += d0_; r4 += d1_; } // t16 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r7 += d1_; } // t17 mm8 a=r0 b=r5 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r2 += d0_; r0 += d1_; } // t18 mm8 a=r1 b=r1 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r2 += d0_; r4 += d1_; } // t19 mm8 a=r3 b=r3 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r6 += d0_; r1 += d1_; } // t20 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r6 += d0_; r0 += d1_; } // t21 mm8 a=r0 b=r3 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r4 += d1_; } // t22 mm8 a=r1 b=r7 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r0 += d0_; r3 += d1_; } // t23 mm8 a=r1 b=r2 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r0 += d0_; r2 += d1_; } // t24 mm8 a=r3 b=r4 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r3 += d0_; r7 += d1_; } // t25 mm8 a=r5 b=r0 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t26 mm8 a=r1 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r0 += d0_; r5 += d1_; } // t27 mm8 a=r3 b=r2 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r5 += d1_; } // t28 mm8 a=r2 b=r4 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r1 += d0_; r5 += d1_; } // t29 mm8 a=r3 b=r4 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r6 += d0_; r3 += d1_; } // t30 mm8 a=r6 b=r5 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t31 mm8 a=r1 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r3 += d0_; r0 += d1_; } // t32 mm8 a=r7 b=r6 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r4 += d0_; r6 += d1_; } // t33 mm8 a=r7 b=r1 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r5 += d0_; r1 += d1_; } // t34 mm8 a=r1 b=r0 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t35 mm8 a=r2 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r7 += d0_; r1 += d1_; } // t36 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r0 += d0_; r2 += d1_; } // t37 mm8 a=r1 b=r7 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r2 += d0_; r4 += d1_; } // t38 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r2 += d0_; r5 += d1_; } // t39 mm8 a=r6 b=r2 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r4 += d0_; r1 += d1_; } // t40 mm8 a=r6 b=r1 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r6 += d0_; r2 += d1_; } // t41 mm8 a=r6 b=r0 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r5 += d0_; r1 += d1_; } // t42 mm8 a=r7 b=r4 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r6 += d0_; r0 += d1_; } // t43 mm8 a=r6 b=r1 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r4 += d0_; r1 += d1_; } // t44 mm8 a=r0 b=r6 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r6 += d0_; r5 += d1_; } // t45 mm8 a=r6 b=r5 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r1 += d0_; r0 += d1_; } // t46 mm8 a=r6 b=r6 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r3 += d0_; r1 += d1_; } // t47 mm8 a=r7 b=r3 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t48 mm8 a=r7 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r1 += d0_; r7 += d1_; } // t49 mm8 a=r4 b=r5 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r4 += d0_; r5 += d1_; } // t50 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r0 += d0_; r3 += d1_; } // t51 mm8 a=r3 b=r4 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r7 += d0_; r4 += d1_; } // t52 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r1 += d0_; r3 += d1_; } // t53 mm8 a=r7 b=r7 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t54 mm8 a=r7 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r3 += d0_; r7 += d1_; } // t55 mm8 a=r1 b=r7 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t56 mm8 a=r4 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r6 += d0_; r7 += d1_; } // t57 mm8 a=r4 b=r3 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r6 += d0_; r0 += d1_; } // t58 mm8 a=r2 b=r3 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r7 += d1_; } // t59 mm8 a=r6 b=r7 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t60 mm8 a=r6 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r2 += d0_; r6 += d1_; } // t61 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r6 += d1_; } // t62 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t63 mm8 a=r1 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r5 += d0_; r3 += d1_; } // t64 mm8 a=r0 b=r5 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r2 += d0_; r4 += d1_; } // t65 mm8 a=r0 b=r1 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r1 += d0_; r3 += d1_; } // t66 mm8 a=r3 b=r5 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r3 += d0_; r4 += d1_; } // t67 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r2 += d1_; } // t68 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r4 += d1_; } // t69 mm8 a=r6 b=r7 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r1 += d0_; r5 += d1_; } // t70 mm8 a=r1 b=r7 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r1 += d0_; r0 += d1_; } // t71 mm8 a=r3 b=r0 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r3 += d0_; r6 += d1_; } // t72 mm8 a=r4 b=r0 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r0 += d0_; r2 += d1_; } // t73 mm8 a=r4 b=r1 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r3 += d0_; r4 += d1_; } // t74 mm8 a=r5 b=r1 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r7 += d0_; r3 += d1_; } // t75 mm8 a=r4 b=r4 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r1 += d0_; r7 += d1_; } // t76 mm8 a=r5 b=r6 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r3 += d0_; r2 += d1_; } // t77 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r0 += d0_; r5 += d1_; } // t78 mm8 a=r2 b=r5 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t79 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r3 += d0_; r4 += d1_; } // t80 mm8 a=r6 b=r7 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r1 += d0_; r6 += d1_; } // t81 mm8 a=r4 b=r3 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r1 += d0_; r7 += d1_; } // t82 mm8 a=r6 b=r0 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r1 += d0_; r6 += d1_; } // t83 mm8 a=r2 b=r2 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t84 mm8 a=r0 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t85 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r2 += d0_; r3 += d1_; } // t86 mm8 a=r3 b=r4 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r3 += d0_; r7 += d1_; } // t87 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r5 += d0_; r3 += d1_; } // t88 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r2 += d0_; r1 += d1_; } // t89 mm8 a=r4 b=r2 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r3 += d0_; r6 += d1_; } // t90 mm8 a=r6 b=r5 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r5 += d0_; r4 += d1_; } // t91 mm8 a=r6 b=r5 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r2 += d0_; r3 += d1_; } // t92 mm8 a=r2 b=r6 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r5 += d0_; r7 += d1_; } // t93 mm8 a=r2 b=r2 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r7 += d0_; r3 += d1_; } // t94 mm8 a=r2 b=r4 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r1 += d0_; r6 += d1_; } // t95 mm8 a=r5 b=r4 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r3 += d0_; r1 += d1_; } // t96 mm8 a=r0 b=r6 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r2 += d1_; } // t97 mm8 a=r0 b=r5 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t98 mm8 a=r2 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r0 += d0_; r2 += d1_; } // t99 mm8 a=r1 b=r4 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r2 += d0_; r5 += d1_; } // t100 mm8 a=r2 b=r4 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r7 += d1_; } // t101 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t102 mm8 a=r5 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t103 mm8 a=r4 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r3 += d0_; r7 += d1_; } // t104 mm8 a=r2 b=r0 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r0 += d0_; r5 += d1_; } // t105 mm8 a=r6 b=r1 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r0 += d0_; r5 += d1_; } // t106 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r4 += d0_; r2 += d1_; } // t107 mm8 a=r2 b=r1 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t108 mm8 a=r4 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r5 += d0_; r3 += d1_; } // t109 mm8 a=r2 b=r7 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r0 += d0_; r2 += d1_; } // t110 mm8 a=r1 b=r0 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r4 += d0_; r6 += d1_; } // t111 mm8 a=r6 b=r5 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r4 += d0_; r2 += d1_; } // t112 mm8 a=r5 b=r7 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t113 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t114 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r1 += d0_; r4 += d1_; } // t115 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r5 += d0_; r0 += d1_; } // t116 mm8 a=r7 b=r2 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r7 += d0_; r6 += d1_; } // t117 mm8 a=r7 b=r6 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r2 += d0_; r1 += d1_; } // t118 mm8 a=r0 b=r1 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r5 += d0_; r3 += d1_; } // t119 mm8 a=r4 b=r3 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r7 += d0_; r2 += d1_; } // t120 mm8 a=r5 b=r3 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r1 += d1_; } // t121 mm8 a=r2 b=r7 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r3 += d0_; r0 += d1_; } // t122 mm8 a=r0 b=r1 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r3 += d0_; r5 += d1_; } // t123 mm8 a=r5 b=r2 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t124 mm8 a=r5 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r7 += d0_; r2 += d1_; } // t125 mm8 a=r7 b=r2 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r3 += d0_; r2 += d1_; } // t126 mm8 a=r6 b=r3 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r1 += d1_; } // t127 mm8 a=r6 b=r7 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r4 += d0_; r1 += d1_; } // t128 mm8 a=r6 b=r7 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r6 += d0_; r3 += d1_; } // t129 mm8 a=r3 b=r2 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r2 += d0_; r1 += d1_; } // t130 mm8 a=r0 b=r4 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r3 += d0_; r1 += d1_; } // t131 mm8 a=r3 b=r5 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r0 += d0_; r6 += d1_; } // t132 mm8 a=r0 b=r7 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r0 += d0_; r1 += d1_; } // t133 mm8 a=r2 b=r6 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r3 += d0_; r1 += d1_; } // t134 mm8 a=r5 b=r6 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r4 += d0_; r5 += d1_; } // t135 mm8 a=r0 b=r2 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r1 += d0_; r5 += d1_; } // t136 mm8 a=r3 b=r1 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r4 += d0_; r6 += d1_; } // t137 mm8 a=r7 b=r0 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r4 += d0_; r2 += d1_; } // t138 mm8 a=r2 b=r2 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r2 += d0_; r7 += d1_; } // t139 mm8 a=r0 b=r4 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r6 += d1_; } // t140 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r1 += d0_; r2 += d1_; } // t141 mm8 a=r5 b=r5 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r6 += d0_; r2 += d1_; } // t142 mm8 a=r2 b=r2 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r7 += d0_; r1 += d1_; } // t143 mm8 a=r4 b=r3 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r1 += d0_; r7 += d1_; } // t144 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r6 += d0_; r5 += d1_; } // t145 mm8 a=r7 b=r2 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r7 += d1_; } // t146 mm8 a=r2 b=r7 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r3 += d0_; r0 += d1_; } // t147 mm8 a=r3 b=r1 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r0 += d0_; r6 += d1_; } // t148 mm8 a=r2 b=r1 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r7 += d0_; r1 += d1_; } // t149 mm8 a=r4 b=r6 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r7 += d0_; r0 += d1_; } // t150 mm8 a=r3 b=r6 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r2 += d0_; r4 += d1_; } // t151 mm8 a=r1 b=r4 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r5 += d0_; r1 += d1_; } // t152 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r2 += d0_; r3 += d1_; } // t153 mm8 a=r0 b=r3 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r0 += d0_; r4 += d1_; } // t154 mm8 a=r7 b=r3 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r7 += d0_; r5 += d1_; } // t155 mm8 a=r1 b=r3 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r2 += d0_; r3 += d1_; } // t156 mm8 a=r6 b=r3 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r4 += d0_; r5 += d1_; } // t157 mm8 a=r6 b=r4 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t158 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r5 += d0_; r2 += d1_; } // t159 mm8 a=r1 b=r0 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r5 += d0_; r7 += d1_; } // t160 mm8 a=r0 b=r1 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r7 += d0_; r0 += d1_; } // t161 mm8 a=r4 b=r0 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r6 += d0_; r1 += d1_; } // t162 mm8 a=r6 b=r6 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r7 += d0_; r3 += d1_; } // t163 mm8 a=r5 b=r3 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t164 mm8 a=r0 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r1 += d0_; r5 += d1_; } // t165 mm8 a=r6 b=r5 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r4 += d0_; r0 += d1_; } // t166 mm8 a=r6 b=r5 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r3 += d0_; r7 += d1_; } // t167 mm8 a=r4 b=r5 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r0 += d0_; r7 += d1_; } // t168 mm8 a=r3 b=r6 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r4 += d0_; r3 += d1_; } // t169 mm8 a=r6 b=r2 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t170 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r2 += d0_; r3 += d1_; } // t171 mm8 a=r6 b=r3 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r5 += d0_; r2 += d1_; } // t172 mm8 a=r7 b=r3 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t173 mm8 a=r0 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r5 += d0_; r6 += d1_; } // t174 mm8 a=r6 b=r6 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r3 += d0_; r4 += d1_; } // t175 mm8 a=r1 b=r6 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r7 += d0_; r5 += d1_; } // t176 mm8 a=r2 b=r4 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r7 += d0_; r2 += d1_; } // t177 mm8 a=r1 b=r7 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r7 += d0_; r3 += d1_; } // t178 mm8 a=r4 b=r2 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r5 += d0_; r1 += d1_; } // t179 mm8 a=r5 b=r2 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r2 += d0_; r1 += d1_; } // t180 mm8 a=r0 b=r4 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r2 += d0_; r4 += d1_; } // t181 mm8 a=r3 b=r5 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r4 += d0_; r3 += d1_; } // t182 mm8 a=r1 b=r1 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r0 += d0_; r7 += d1_; } // t183 mm8 a=r3 b=r1 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r0 += d0_; r7 += d1_; } // t184 mm8 a=r7 b=r4 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r6 += d0_; r3 += d1_; } // t185 mm8 a=r5 b=r0 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r7 += d0_; r6 += d1_; } // t186 mm8 a=r2 b=r5 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r0 += d0_; r7 += d1_; } // t187 mm8 a=r3 b=r0 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r5 += d0_; r0 += d1_; } // t188 mm8 a=r1 b=r4 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r5 += d0_; r4 += d1_; } // t189 mm8 a=r7 b=r2 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r0 += d1_; } // t190 mm8 a=r1 b=r5 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t191 mm8 a=r2 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t192 mm8 a=r5 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r4 += d0_; r6 += d1_; } // t193 mm8 a=r2 b=r1 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r0 += d0_; r5 += d1_; } // t194 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r4 += d1_; } // t195 mm8 a=r2 b=r7 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r2 += d0_; r5 += d1_; } // t196 mm8 a=r5 b=r6 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r6 += d0_; r5 += d1_; } // t197 mm8 a=r5 b=r3 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r7 += d0_; r1 += d1_; } // t198 mm8 a=r1 b=r0 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t199 mm8 a=r2 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t200 mm8 a=r5 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r2 += d0_; r1 += d1_; } // t201 mm8 a=r7 b=r2 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r5 += d0_; r6 += d1_; } // t202 mm8 a=r2 b=r5 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r6 += d1_; } // t203 mm8 a=r0 b=r5 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r3 += d0_; r7 += d1_; } // t204 mm8 a=r4 b=r7 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r1 += d0_; r3 += d1_; } // t205 mm8 a=r3 b=r7 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r7 += d0_; r3 += d1_; } // t206 mm8 a=r0 b=r6 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r1 += d0_; r4 += d1_; } // t207 mm8 a=r7 b=r7 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r6 += d0_; r5 += d1_; } // t208 mm8 a=r0 b=r1 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r6 += d0_; r2 += d1_; } // t209 mm8 a=r4 b=r4 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t210 mm8 a=r7 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r5 += d0_; r6 += d1_; } // t211 mm8 a=r1 b=r5 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r5 += d0_; r3 += d1_; } // t212 mm8 a=r2 b=r4 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r3 += d0_; r5 += d1_; } // t213 mm8 a=r2 b=r2 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r5 += d0_; r7 += d1_; } // t214 mm8 a=r4 b=r6 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r2 += d0_; r6 += d1_; } // t215 mm8 a=r7 b=r3 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r3 += d0_; r4 += d1_; } // t216 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r0 += d0_; r6 += d1_; } // t217 mm8 a=r5 b=r7 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r0 += d1_; } // t218 mm8 a=r7 b=r1 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r3 += d0_; r1 += d1_; } // t219 mm8 a=r2 b=r1 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r4 += d0_; r5 += d1_; } // t220 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r2 += d0_; r1 += d1_; } // t221 mm8 a=r7 b=r1 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r2 += d0_; r1 += d1_; } // t222 mm8 a=r0 b=r6 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r7 += d0_; r0 += d1_; } // t223 mm8 a=r2 b=r2 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r6 += d0_; r2 += d1_; } // t224 mm8 a=r1 b=r1 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r4 += d1_; } // t225 mm8 a=r2 b=r7 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t226 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r5 += d0_; r6 += d1_; } // t227 mm8 a=r3 b=r5 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r3 += d0_; r1 += d1_; } // t228 mm8 a=r7 b=r1 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r1 += d0_; r4 += d1_; } // t229 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r5 += d0_; r6 += d1_; } // t230 mm8 a=r0 b=r3 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r4 += d0_; r0 += d1_; } // t231 mm8 a=r1 b=r3 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r0 += d0_; r1 += d1_; } // t232 mm8 a=r5 b=r4 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r0 += d0_; r4 += d1_; } // t233 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r2 += d0_; r1 += d1_; } // t234 mm8 a=r2 b=r0 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t235 mm8 a=r3 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r0 += d0_; r5 += d1_; } // t236 mm8 a=r1 b=r7 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r6 += d0_; r3 += d1_; } // t237 mm8 a=r3 b=r5 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r0 += d0_; r2 += d1_; } // t238 mm8 a=r2 b=r0 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r1 += d0_; r0 += d1_; } // t239 mm8 a=r5 b=r7 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r4 += d0_; r6 += d1_; } // t240 mm8 a=r2 b=r7 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r3 += d0_; r7 += d1_; } // t241 mm8 a=r0 b=r1 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r2 += d0_; r7 += d1_; } // t242 mm8 a=r0 b=r2 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r1 += d0_; r5 += d1_; } // t243 mm8 a=r4 b=r4 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r0 += d0_; r7 += d1_; } // t244 mm8 a=r7 b=r6 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r2 += d0_; r0 += d1_; } // t245 mm8 a=r4 b=r4 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t246 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r2 += d0_; r4 += d1_; } // t247 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r7 += d0_; r3 += d1_; } // t248 mm8 a=r7 b=r5 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r0 += d0_; r5 += d1_; } // t249 mm8 a=r4 b=r4 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r0 += d0_; r1 += d1_; } // t250 mm8 a=r0 b=r6 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t251 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r5 += d0_; r4 += d1_; } // t252 mm8 a=r0 b=r2 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r7 += d0_; r4 += d1_; } // t253 mm8 a=r5 b=r5 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t254 mm8 a=r6 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r0 += d0_; r3 += d1_; } // t255 mm8 a=r5 b=r0 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r3 += d0_; r4 += d1_; } // t256 mm8 a=r2 b=r1 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r6 += d0_; r0 += d1_; } // t257 mm8 a=r4 b=r3 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r7 += d0_; r3 += d1_; } // t258 mm8 a=r4 b=r1 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r3 += d0_; r0 += d1_; } // t259 mm8 a=r5 b=r5 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r3 += d0_; r0 += d1_; } // t260 mm8 a=r3 b=r2 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r3 += d0_; r2 += d1_; } // t261 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r2 += d0_; r5 += d1_; } // t262 mm8 a=r6 b=r4 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r2 += d0_; r1 += d1_; } // t263 mm8 a=r6 b=r4 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r1 += d0_; r4 += d1_; } // t264 mm8 a=r7 b=r2 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r3 += d0_; r1 += d1_; } // t265 mm8 a=r6 b=r1 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r7 += d0_; r0 += d1_; } // t266 mm8 a=r0 b=r0 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r3 += d0_; r4 += d1_; } // t267 mm8 a=r6 b=r3 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r6 += d0_; r5 += d1_; } // t268 mm8 a=r1 b=r0 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r7 += d0_; r3 += d1_; } // t269 mm8 a=r7 b=r7 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r7 += d0_; r0 += d1_; } // t270 mm8 a=r6 b=r2 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r5 += d0_; r3 += d1_; } // t271 mm8 a=r0 b=r3 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r0 += d0_; r3 += d1_; } // t272 mm8 a=r6 b=r0 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r7 += d1_; } // t273 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r7 += d0_; r6 += d1_; } // t274 mm8 a=r1 b=r0 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r1 += d0_; r4 += d1_; } // t275 mm8 a=r1 b=r3 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r6 += d0_; r2 += d1_; } // t276 mm8 a=r1 b=r5 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r4 += d0_; r7 += d1_; } // t277 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r6 += d0_; r7 += d1_; } // t278 mm8 a=r0 b=r0 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r2 += d0_; r4 += d1_; } // t279 mm8 a=r2 b=r4 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r3 += d0_; r5 += d1_; } // t280 mm8 a=r7 b=r2 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r4 += d0_; r2 += d1_; } // t281 mm8 a=r1 b=r0 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r5 += d0_; r1 += d1_; } // t282 mm8 a=r0 b=r1 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r7 += d0_; r2 += d1_; } // t283 mm8 a=r0 b=r2 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r0 += d0_; r2 += d1_; } // t284 mm8 a=r1 b=r5 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r5 += d0_; r0 += d1_; } // t285 mm8 a=r4 b=r7 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r2 += d0_; r0 += d1_; } // t286 mm8 a=r1 b=r0 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r7 += d0_; r4 += d1_; } // t287 mm8 a=r7 b=r1 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t288 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r5 += d0_; r4 += d1_; } // t289 mm8 a=r0 b=r5 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r6 += d0_; r7 += d1_; } // t290 mm8 a=r3 b=r2 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r7 += d0_; r1 += d1_; } // t291 mm8 a=r7 b=r4 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t292 mm8 a=r6 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r6 += d0_; r4 += d1_; } // t293 mm8 a=r5 b=r5 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r1 += d0_; r6 += d1_; } // t294 mm8 a=r5 b=r7 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r1 += d0_; r3 += d1_; } // t295 mm8 a=r0 b=r7 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r5 += d0_; r6 += d1_; } // t296 mm8 a=r6 b=r1 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r3 += d0_; r5 += d1_; } // t297 mm8 a=r4 b=r5 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r3 += d0_; r1 += d1_; } // t298 mm8 a=r6 b=r0 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r2 += d0_; r3 += d1_; } // t299 mm8 a=r6 b=r0 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r4 += d0_; r2 += d1_; } // t300 mm8 a=r6 b=r4 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r6 += d0_; r3 += d1_; } // t301 mm8 a=r6 b=r4 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r6 += d0_; r4 += d1_; } // t302 mm8 a=r7 b=r6 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r3 += d0_; r6 += d1_; } // t303 mm8 a=r3 b=r2 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r2 += d0_; r4 += d1_; } // t304 mm8 a=r6 b=r5 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r0 += d0_; r2 += d1_; } // t305 mm8 a=r4 b=r5 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r0 += d0_; r2 += d1_; } // t306 mm8 a=r3 b=r1 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r5 += d0_; r0 += d1_; } // t307 mm8 a=r4 b=r3 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r2 += d0_; r6 += d1_; } // t308 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r1 += d0_; r4 += d1_; } // t309 mm8 a=r2 b=r1 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r2 += d0_; r3 += d1_; } // t310 mm8 a=r6 b=r1 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r3 += d0_; r2 += d1_; } // t311 mm8 a=r4 b=r6 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t312 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r7 += d0_; r1 += d1_; } // t313 mm8 a=r2 b=r6 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r2 += d0_; r1 += d1_; } // t314 mm8 a=r0 b=r3 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r6 += d0_; r2 += d1_; } // t315 mm8 a=r7 b=r1 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r1 += d0_; r6 += d1_; } // t316 mm8 a=r2 b=r4 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r3 += d0_; r6 += d1_; } // t317 mm8 a=r1 b=r6 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r1 += d0_; r0 += d1_; } // t318 mm8 a=r2 b=r1 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r6 += d0_; r3 += d1_; } // t319 mm8 a=r0 b=r2 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r6 += d0_; r2 += d1_; } // t320 mm8 a=r5 b=r2 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t321 mm8 a=r7 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r7 += d0_; r1 += d1_; } // t322 mm8 a=r2 b=r4 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r2 += d0_; r3 += d1_; } // t323 mm8 a=r4 b=r3 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r2 += d0_; r0 += d1_; } // t324 mm8 a=r5 b=r7 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r7 += d0_; r2 += d1_; } // t325 mm8 a=r6 b=r2 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r7 += d0_; r6 += d1_; } // t326 mm8 a=r6 b=r1 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r3 += d0_; r0 += d1_; } // t327 mm8 a=r1 b=r5 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r1 += d0_; r6 += d1_; } // t328 mm8 a=r0 b=r1 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r7 += d0_; r1 += d1_; } // t329 mm8 a=r0 b=r3 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r4 += d0_; r0 += d1_; } // t330 mm8 a=r6 b=r7 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t331 mm8 a=r4 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r6 += d0_; r3 += d1_; } // t332 mm8 a=r4 b=r4 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r4 += d0_; r2 += d1_; } // t333 mm8 a=r4 b=r6 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r0 += d0_; r1 += d1_; } // t334 mm8 a=r2 b=r2 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r2 += d1_; } // t335 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r2 += d0_; r3 += d1_; } // t336 mm8 a=r4 b=r0 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r0 += d0_; r3 += d1_; } // t337 mm8 a=r1 b=r4 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r2 += d0_; r7 += d1_; } // t338 mm8 a=r3 b=r7 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r7 += d0_; r6 += d1_; } // t339 mm8 a=r5 b=r1 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r7 += d0_; r5 += d1_; } // t340 mm8 a=r5 b=r0 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r7 += d0_; r5 += d1_; } // t341 mm8 a=r6 b=r6 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r0 += d0_; r4 += d1_; } // t342 mm8 a=r4 b=r7 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r4 += d0_; r6 += d1_; } // t343 mm8 a=r0 b=r4 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r4 += d0_; r6 += d1_; } // t344 mm8 a=r2 b=r5 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t345 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t346 mm8 a=r6 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r3 += d0_; r5 += d1_; } // t347 mm8 a=r5 b=r3 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r4 += d0_; r5 += d1_; } // t348 mm8 a=r0 b=r4 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r4 += d0_; r7 += d1_; } // t349 mm8 a=r6 b=r1 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r3 += d0_; r5 += d1_; } // t350 mm8 a=r1 b=r5 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r4 += d0_; r5 += d1_; } // t351 mm8 a=r6 b=r4 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r6 += d0_; r0 += d1_; } // t352 mm8 a=r4 b=r1 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r1 += d0_; r4 += d1_; } // t353 mm8 a=r7 b=r3 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r1 += d0_; r5 += d1_; } // t354 mm8 a=r4 b=r1 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r0 += d0_; r1 += d1_; } // t355 mm8 a=r4 b=r5 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r0 += d0_; r4 += d1_; } // t356 mm8 a=r7 b=r6 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r7 += d0_; r5 += d1_; } // t357 mm8 a=r7 b=r3 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r5 += d0_; r7 += d1_; } // t358 mm8 a=r4 b=r7 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r3 += d0_; r5 += d1_; } // t359 mm8 a=r2 b=r7 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r0 += d0_; r2 += d1_; } // t360 mm8 a=r4 b=r4 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r5 += d0_; r6 += d1_; } // t361 mm8 a=r2 b=r2 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r3 += d0_; r0 += d1_; } // t362 mm8 a=r2 b=r7 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r5 += d0_; r0 += d1_; } // t363 mm8 a=r7 b=r6 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r0 += d0_; r5 += d1_; } // t364 mm8 a=r6 b=r4 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r5 += d0_; r4 += d1_; } // t365 mm8 a=r6 b=r6 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r2 += d0_; r5 += d1_; } // t366 mm8 a=r7 b=r7 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t367 mm8 a=r4 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r3 += d0_; r5 += d1_; } // t368 mm8 a=r7 b=r0 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r6 += d0_; r0 += d1_; } // t369 mm8 a=r1 b=r0 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r7 += d0_; r4 += d1_; } // t370 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r2 += d0_; r6 += d1_; } // t371 mm8 a=r5 b=r4 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r2 += d1_; } // t372 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r2 += d0_; r4 += d1_; } // t373 mm8 a=r4 b=r6 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t374 mm8 a=r1 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r6 += d0_; r2 += d1_; } // t375 mm8 a=r6 b=r5 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t376 mm8 a=r3 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r6 += d0_; r4 += d1_; } // t377 mm8 a=r6 b=r4 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r2 += d0_; r4 += d1_; } // t378 mm8 a=r6 b=r3 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t379 mm8 a=r3 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r6 += d0_; r2 += d1_; } // t380 mm8 a=r2 b=r0 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r1 += d0_; r2 += d1_; } // t381 mm8 a=r3 b=r5 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r2 += d0_; r4 += d1_; } // t382 mm8 a=r5 b=r2 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r0 += d0_; r2 += d1_; } // t383 mm8 a=r1 b=r2 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r1 += d0_; r0 += d1_; } // t384 mm8 a=r3 b=r1 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r7 += d0_; r5 += d1_; } // t385 mm8 a=r6 b=r0 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r1 += d0_; r7 += d1_; } // t386 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r2 += d0_; r4 += d1_; } // t387 mm8 a=r3 b=r7 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r1 += d0_; r2 += d1_; } // t388 mm8 a=r3 b=r0 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r5 += d0_; r3 += d1_; } // t389 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r1 += d0_; r4 += d1_; } // t390 mm8 a=r5 b=r4 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t391 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r3 += d0_; r0 += d1_; } // t392 mm8 a=r2 b=r3 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r5 += d0_; r0 += d1_; } // t393 mm8 a=r3 b=r6 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r1 += d0_; r2 += d1_; } // t394 mm8 a=r0 b=r7 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r2 += d0_; r4 += d1_; } // t395 mm8 a=r0 b=r7 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r1 += d0_; r4 += d1_; } // t396 mm8 a=r7 b=r7 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t397 mm8 a=r5 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r7 += d0_; r0 += d1_; } // t398 mm8 a=r6 b=r5 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r3 += d0_; r5 += d1_; } // t399 mm8 a=r6 b=r3 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r0 += d0_; r4 += d1_; } // t400 mm8 a=r2 b=r7 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r2 += d0_; r1 += d1_; } // t401 mm8 a=r7 b=r6 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r7 += d0_; r1 += d1_; } // t402 mm8 a=r0 b=r0 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r1 += d0_; r0 += d1_; } // t403 mm8 a=r7 b=r4 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r1 += d0_; r4 += d1_; } // t404 mm8 a=r1 b=r1 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r3 += d0_; r6 += d1_; } // t405 mm8 a=r6 b=r6 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r4 += d0_; r3 += d1_; } // t406 mm8 a=r6 b=r1 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r2 += d0_; r4 += d1_; } // t407 mm8 a=r5 b=r7 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t408 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r4 += d0_; r7 += d1_; } // t409 mm8 a=r3 b=r4 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r5 += d0_; r2 += d1_; } // t410 mm8 a=r0 b=r5 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r5 += d0_; r1 += d1_; } // t411 mm8 a=r2 b=r0 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t412 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r2 += d1_; } // t413 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r0 += d0_; r3 += d1_; } // t414 mm8 a=r7 b=r1 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r5 += d0_; r1 += d1_; } // t415 mm8 a=r4 b=r7 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r3 += d0_; r5 += d1_; } // t416 mm8 a=r5 b=r4 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t417 mm8 a=r6 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r1 += d0_; r5 += d1_; } // t418 mm8 a=r1 b=r3 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r0 += d0_; r6 += d1_; } // t419 mm8 a=r5 b=r0 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r7 += d0_; r6 += d1_; } // t420 mm8 a=r4 b=r2 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r3 += d0_; r7 += d1_; } // t421 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t422 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r0 += d1_; } // t423 mm8 a=r1 b=r7 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r6 += d0_; r5 += d1_; } // t424 mm8 a=r5 b=r1 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t425 mm8 a=r2 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r5 += d0_; r0 += d1_; } // t426 mm8 a=r1 b=r6 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t427 mm8 a=r6 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r7 += d0_; r4 += d1_; } // t428 mm8 a=r6 b=r2 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r7 += d0_; r0 += d1_; } // t429 mm8 a=r3 b=r3 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r3 += d0_; r5 += d1_; } // t430 mm8 a=r7 b=r7 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r1 += d0_; r0 += d1_; } // t431 mm8 a=r6 b=r4 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r3 += d0_; r4 += d1_; } // t432 mm8 a=r3 b=r2 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r5 += d0_; r0 += d1_; } // t433 mm8 a=r3 b=r0 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r5 += d0_; r3 += d1_; } // t434 mm8 a=r0 b=r7 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r6 += d0_; r3 += d1_; } // t435 mm8 a=r7 b=r5 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r0 += d0_; r5 += d1_; } // t436 mm8 a=r6 b=r4 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r3 += d0_; r4 += d1_; } // t437 mm8 a=r0 b=r7 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r4 += d0_; r7 += d1_; } // t438 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t439 mm8 a=r4 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r3 += d0_; r2 += d1_; } // t440 mm8 a=r3 b=r5 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r1 += d0_; r4 += d1_; } // t441 mm8 a=r5 b=r1 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r7 += d0_; r5 += d1_; } // t442 mm8 a=r1 b=r2 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r5 += d0_; r2 += d1_; } // t443 mm8 a=r1 b=r7 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r1 += d0_; r0 += d1_; } // t444 mm8 a=r3 b=r7 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r7 += d0_; r0 += d1_; } // t445 mm8 a=r7 b=r5 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r2 += d0_; r1 += d1_; } // t446 mm8 a=r3 b=r1 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r4 += d0_; r7 += d1_; } // t447 mm8 a=r6 b=r7 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r4 += d0_; r5 += d1_; } // t448 mm8 a=r5 b=r2 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r3 += d1_; } // t449 mm8 a=r7 b=r1 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r7 += d0_; r3 += d1_; } // t450 mm8 a=r7 b=r5 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r1 += d0_; r7 += d1_; } // t451 mm8 a=r0 b=r3 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r7 += d0_; r1 += d1_; } // t452 mm8 a=r2 b=r4 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r7 += d0_; r1 += d1_; } // t453 mm8 a=r5 b=r7 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r7 += d0_; r1 += d1_; } // t454 mm8 a=r7 b=r7 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t455 mm8 a=r7 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r5 += d0_; r6 += d1_; } // t456 mm8 a=r6 b=r5 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r1 += d0_; r5 += d1_; } // t457 mm8 a=r7 b=r7 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r4 += d0_; r1 += d1_; } // t458 mm8 a=r5 b=r1 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r1 += d0_; r0 += d1_; } // t459 mm8 a=r6 b=r7 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t460 mm8 a=r7 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r6 += d0_; r2 += d1_; } // t461 mm8 a=r0 b=r6 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t462 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r3 += d0_; r2 += d1_; } // t463 mm8 a=r2 b=r0 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r0 += d0_; r1 += d1_; } // t464 mm8 a=r0 b=r3 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r0 += d0_; r5 += d1_; } // t465 mm8 a=r1 b=r7 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r4 += d0_; r1 += d1_; } // t466 mm8 a=r1 b=r2 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r1 += d0_; r6 += d1_; } // t467 mm8 a=r1 b=r0 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r6 += d0_; r2 += d1_; } // t468 mm8 a=r4 b=r1 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r0 += d0_; r1 += d1_; } // t469 mm8 a=r2 b=r0 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r2 += d0_; r0 += d1_; } // t470 mm8 a=r3 b=r1 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r7 += d0_; r1 += d1_; } // t471 mm8 a=r2 b=r2 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t472 mm8 a=r1 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r1 += d1_; } // t473 mm8 a=r2 b=r7 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r6 += d0_; r3 += d1_; } // t474 mm8 a=r5 b=r3 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r6 += d0_; r0 += d1_; } // t475 mm8 a=r7 b=r6 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r5 += d0_; r6 += d1_; } // t476 mm8 a=r3 b=r0 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r4 += d0_; r6 += d1_; } // t477 mm8 a=r3 b=r1 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r2 += d0_; r5 += d1_; } // t478 mm8 a=r0 b=r6 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r7 += d0_; r1 += d1_; } // t479 mm8 a=r7 b=r6 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r1 += d0_; r0 += d1_; } // t480 mm8 a=r6 b=r0 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r7 += d1_; } // t481 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r0 += d0_; r6 += d1_; } // t482 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r7 += d0_; r6 += d1_; } // t483 mm8 a=r7 b=r0 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r6 += d0_; r4 += d1_; } // t484 mm8 a=r3 b=r7 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r1 += d0_; r2 += d1_; } // t485 mm8 a=r6 b=r4 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r2 += d0_; r5 += d1_; } // t486 mm8 a=r6 b=r5 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r4 += d0_; r5 += d1_; } // t487 mm8 a=r2 b=r5 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r1 += d0_; r2 += d1_; } // t488 mm8 a=r1 b=r2 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r6 += d0_; r4 += d1_; } // t489 mm8 a=r5 b=r2 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r2 += d0_; r6 += d1_; } // t490 mm8 a=r7 b=r1 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r0 += d0_; r6 += d1_; } // t491 mm8 a=r0 b=r0 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r6 += d0_; r4 += d1_; } // t492 mm8 a=r3 b=r1 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r7 += d0_; r6 += d1_; } // t493 mm8 a=r6 b=r1 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r3 += d0_; r1 += d1_; } // t494 mm8 a=r5 b=r6 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r7 += d0_; r2 += d1_; } // t495 mm8 a=r2 b=r2 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r5 += d0_; r0 += d1_; } // t496 mm8 a=r3 b=r5 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r2 += d0_; r0 += d1_; } // t497 mm8 a=r3 b=r2 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r6 += d0_; r2 += d1_; } // t498 mm8 a=r3 b=r1 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r1 += d0_; r5 += d1_; } // t499 mm8 a=r1 b=r6 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r1 += d0_; r5 += d1_; } // t500 mm8 a=r7 b=r6 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r1 += d0_; r0 += d1_; } // t501 mm8 a=r4 b=r4 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r7 += d0_; r6 += d1_; } // t502 mm8 a=r5 b=r7 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r3 += d0_; r4 += d1_; } // t503 mm8 a=r7 b=r5 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r4 += d0_; r2 += d1_; } // t504 mm8 a=r2 b=r0 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r0 += d0_; r1 += d1_; } // t505 mm8 a=r4 b=r0 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r7 += d0_; r0 += d1_; } // t506 mm8 a=r0 b=r2 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r4 += d0_; r7 += d1_; } // t507 mm8 a=r5 b=r6 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r4 += d0_; r7 += d1_; } // t508 mm8 a=r7 b=r3 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r6 += d0_; r0 += d1_; } // t509 mm8 a=r4 b=r4 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t510 mm8 a=r0 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t511 mm8 a=r2 b=r4 c=r1 c2=r3 - } - uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((ulong)hi << 32) | (ulong)lo; -} - -#if IGNEUM_EXCHANGE != 0 -// Reports the sub-group size this device uses for a work-group of IGNEUM_GROUP items. host.c runs it only when the -// per-kernel query (clGetKernelSubGroupInfoKHR on igneum_hash) is unavailable; that query is preferred because a -// compiler may pick a different wave width per kernel (RDNA: wave32 or wave64). See WAVEFRONT.md. -IGNEUM_KERNEL_HASH void igneum_probe_subgroup(__global uint* out) { - if (get_local_id(0) == 0u) { out[0] = get_sub_group_size(); out[1] = get_num_sub_groups(); } -} -#endif diff --git a/proto-cuda/packs-ca4/mx8_mm512/kernel.cu b/proto-cuda/packs-ca4/mx8_mm512/kernel.cu deleted file mode 100644 index 0a53d50cf..000000000 --- a/proto-cuda/packs-ca4/mx8_mm512/kernel.cu +++ /dev/null @@ -1,677 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// Bit-exact twin of the Metal kernel for the same seed (see proto-cuda/CHECKLIST.md and program.metal). -// Compiled ahead of time by nvcc together with proto-cuda/host.cu. No NVRTC. -#include -#include -#include "program.h" -#include "memhard.h" - -__device__ __forceinline__ uint32_t splitmix32(uint32_t x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -// n is a literal in 1..31 at every call site, so both shift amounts are in 1..31. -__device__ __forceinline__ uint32_t rotl_imm(uint32_t x, uint32_t n) { return (x << n) | (x >> (32u - n)); } -// n is masked to 0..31; the second shift amount is masked too, so n == 0 gives x. -__device__ __forceinline__ uint32_t rotr_var(uint32_t x, uint32_t n) { n &= 31u; return (x >> n) | (x << ((32u - n) & 31u)); } -__device__ __forceinline__ uint32_t ds_elem(uint32_t i, uint32_t d0, uint32_t d1) { - uint32_t x = i ^ d0; - x *= 0x9E3779B1u; x ^= x >> 15; - x += d1; - x *= 0x85EBCA77u; x ^= x >> 13; - x *= 0xC2B2AE3Du; x ^= x >> 16; - return x; -} - -// Memory-hard dataset (MEMHARD.md). One thread per cache segment; one thread per 64-byte dataset item. -// The core functions (mh_cache_segment, mh_item) are in memhard.h and are also compiled for the host. -__global__ void igneum_cache_fill(uint32_t* cache, uint32_t nSegments) { - uint32_t seg = blockIdx.x * blockDim.x + threadIdx.x; - if (seg < nSegments) mh_cache_segment(cache, seg); -} -__global__ void igneum_build(uint32_t* ds, const uint32_t* cache, uint32_t nItems) { - uint32_t t = blockIdx.x * blockDim.x + threadIdx.x; - if (t < nItems) { - uint32_t s[16]; - mh_item(cache, t, s); - uint32_t* d = ds + (size_t)t * 16u; - for (uint32_t i = 0u; i < 16u; ++i) d[i] = s[i]; - } -} - -// One hash per thread. blockDim.x is a multiple of 32; lane = threadIdx.x & 31 and every -// __shfl_xor_sync stays inside the lane's own warp, exactly like simd_shuffle_xor inside a -// 32-wide Metal SIMD group. Control flow is uniform, so the full 0xffffffff member mask is valid. -__global__ void igneum_hash(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask) { - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t nonce = baseNonce + gid; - uint32_t r0, r1, r2, r3, r4, r5, r6, r7; - { uint32_t x = nonce ^ 0x67a9a7beu; x += 0x9e3779b9u; x = splitmix32(x); r0 = x ^ 0x1a155b25u; } // SEEDW[0], 0x9e3779b9u * 1u, SEEDW[1] - { uint32_t x = nonce ^ 0x1a155b25u; x += 0x3c6ef372u; x = splitmix32(x); r1 = x ^ 0xfddfb732u; } // SEEDW[1], 0x9e3779b9u * 2u, SEEDW[2] - { uint32_t x = nonce ^ 0xfddfb732u; x += 0xdaa66d2bu; x = splitmix32(x); r2 = x ^ 0x4b5af2e8u; } // SEEDW[2], 0x9e3779b9u * 3u, SEEDW[3] - { uint32_t x = nonce ^ 0x4b5af2e8u; x += 0x78dde6e4u; x = splitmix32(x); r3 = x ^ 0xc55caf33u; } // SEEDW[3], 0x9e3779b9u * 4u, SEEDW[4] - { uint32_t x = nonce ^ 0xc55caf33u; x += 0x1715609du; x = splitmix32(x); r4 = x ^ 0xa27c13b7u; } // SEEDW[4], 0x9e3779b9u * 5u, SEEDW[5] - { uint32_t x = nonce ^ 0xa27c13b7u; x += 0xb54cda56u; x = splitmix32(x); r5 = x ^ 0x06628a48u; } // SEEDW[5], 0x9e3779b9u * 6u, SEEDW[6] - { uint32_t x = nonce ^ 0x06628a48u; x += 0x5384540fu; x = splitmix32(x); r6 = x ^ 0x03852469u; } // SEEDW[6], 0x9e3779b9u * 7u, SEEDW[7] - { uint32_t x = nonce ^ 0x03852469u; x += 0xf1bbcdc8u; x = splitmix32(x); r7 = x ^ 0x67a9a7beu; } // SEEDW[7], 0x9e3779b9u * 8u, SEEDW[0] - - for (uint32_t it = 0u; it < 8u; ++it) { - uint32_t sel = r0; - r2 = r3 * r4 + r2; // 0 mad - r2 = r1 * r1 + r2; // 1 mad - r2 = r3 * r2 + r2; // 2 mad - r3 = r3 ^ r5; // 3 xor - r7 = r7 ^ ds[r2 & mask]; // 4 load - r5 = r5 ^ ds[r7 & mask]; // 5 load - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r4, 8); // 6 shfl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r3, 8); // 7 shfl - r1 = __umulhi(r1, r5); // 8 mulhi - r6 = rotr_var(r6, r3); // 9 rotr - r3 = r3 | r4; // 10 or - r4 = r4 ^ ds[r3 & mask]; // 11 load - r0 = __umulhi(r0, r4); // 12 mulhi - r5 = r5 + r1 + ((((sel >> 30u) & 1u) != 0u) ? 0xd3177981u : 0xc7934706u); // 13 add - r0 = r0 ^ ds[r4 & mask]; // 14 load - r2 = r2 - r4; // 15 sub - r2 = r2 ^ ds[r0 & mask]; // 16 load - r7 = r7 ^ ds[r2 & mask]; // 17 load - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r3, 4); // 18 shfl - r5 = r5 * r0; // 19 mul - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r4, 2); // 20 shfl - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r4, 16); // 21 shfl - r6 = __umulhi(r6, r2); // 22 mulhi - r6 = r6 ^ ds[r1 & mask]; // 23 load - r5 = r5 * r0; // 24 mul - r5 = rotl_imm(r5, 19u); // 25 rotl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r6, 2); // 26 shfl - r0 = r0 ^ r5; // 27 xor - r0 = r0 ^ r4; // 28 xor - r3 = r3 - r0; // 29 sub - r5 = r5 * r1; // 30 mul - r7 = r7 ^ ds[r2 & mask]; // 31 load - r1 = r1 ^ ds[r0 & mask]; // 32 load - r5 = r5 ^ r6; // 33 xor - r5 = r5 ^ ds[r1 & mask]; // 34 load - r0 = __umulhi(r0, r5); // 35 mulhi - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r2, 4); // 36 shfl - r7 = r7 ^ ds[r0 & mask]; // 37 load - r3 = r3 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0x230c005cu : 0x75ba2fadu); // 38 add - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r5, 4); // 39 shfl - r2 = r2 ^ r5; // 40 xor - r3 = r6 * r3 + r3; // 41 mad - r6 = r6 - r7; // 42 sub - r7 = r7 ^ r0; // 43 xor - r1 = r1 ^ ds[r7 & mask]; // 44 load - r2 = r2 * r3; // 45 mul - r1 = __umulhi(r1, r5); // 46 mulhi - r4 = r4 - r3; // 47 sub - r2 = rotr_var(r2, r6); // 48 rotr - r3 = r3 ^ ds[r5 & mask]; // 49 load - r1 = r1 + r5 + ((((sel >> 7u) & 1u) != 0u) ? 0x1907970cu : 0x81b8bc2cu); // 50 add - r0 = r0 * r2; // 51 mul - r0 = r0 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x699fd448u : 0x4f92b968u); // 52 add - r1 = r1 + r0 + ((((sel >> 12u) & 1u) != 0u) ? 0x77b1520du : 0x2bb965afu); // 53 add - r7 = rotl_imm(r7, 14u); // 54 rotl - r3 = r3 + r7 + ((((sel >> 1u) & 1u) != 0u) ? 0xa54c55a0u : 0x7b0fe07au); // 55 add - r6 = r6 ^ ds[r7 & mask]; // 56 load - r1 = rotr_var(r1, r5); // 57 rotr - r5 = r5 ^ ds[r4 & mask]; // 58 load - r6 = r6 ^ ds[r2 & mask]; // 59 load - r3 = r5 * r0 + r3; // 60 mad - r5 = r5 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0xad7493e7u : 0xaf9dd72du); // 61 add - r4 = r4 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x1e07c3d9u : 0x89841d87u); // 62 add - r5 = rotl_imm(r5, 19u); // 63 rotl - // int8 tile block (Counter ASIC 4.0 research, experimental): 512 mma.m8n8k16 u8 tiles per iteration, both outputs consumed - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t0 mm8 a=r6 b=r5 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t1 mm8 a=r1 b=r1 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t2 mm8 a=r2 b=r5 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t3 mm8 a=r1 b=r5 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t4 mm8 a=r2 b=r0 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t5 mm8 a=r2 b=r6 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t6 mm8 a=r1 b=r4 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t7 mm8 a=r4 b=r6 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t8 mm8 a=r1 b=r1 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t9 mm8 a=r3 b=r7 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t10 mm8 a=r3 b=r0 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t11 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t12 mm8 a=r3 b=r1 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t13 mm8 a=r1 b=r5 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t14 mm8 a=r3 b=r3 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t15 mm8 a=r1 b=r0 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t16 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t17 mm8 a=r0 b=r5 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t18 mm8 a=r1 b=r1 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t19 mm8 a=r3 b=r3 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t20 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t21 mm8 a=r0 b=r3 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t22 mm8 a=r1 b=r7 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t23 mm8 a=r1 b=r2 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t24 mm8 a=r3 b=r4 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t25 mm8 a=r5 b=r0 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t26 mm8 a=r1 b=r0 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t27 mm8 a=r3 b=r2 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t28 mm8 a=r2 b=r4 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t29 mm8 a=r3 b=r4 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t30 mm8 a=r6 b=r5 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t31 mm8 a=r1 b=r7 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t32 mm8 a=r7 b=r6 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t33 mm8 a=r7 b=r1 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t34 mm8 a=r1 b=r0 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t35 mm8 a=r2 b=r4 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t36 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t37 mm8 a=r1 b=r7 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t38 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t39 mm8 a=r6 b=r2 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t40 mm8 a=r6 b=r1 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t41 mm8 a=r6 b=r0 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t42 mm8 a=r7 b=r4 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t43 mm8 a=r6 b=r1 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t44 mm8 a=r0 b=r6 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t45 mm8 a=r6 b=r5 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t46 mm8 a=r6 b=r6 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t47 mm8 a=r7 b=r3 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t48 mm8 a=r7 b=r7 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t49 mm8 a=r4 b=r5 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t50 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t51 mm8 a=r3 b=r4 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t52 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t53 mm8 a=r7 b=r7 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t54 mm8 a=r7 b=r6 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t55 mm8 a=r1 b=r7 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t56 mm8 a=r4 b=r0 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t57 mm8 a=r4 b=r3 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t58 mm8 a=r2 b=r3 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t59 mm8 a=r6 b=r7 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t60 mm8 a=r6 b=r0 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t61 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t62 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t63 mm8 a=r1 b=r6 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t64 mm8 a=r0 b=r5 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t65 mm8 a=r0 b=r1 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t66 mm8 a=r3 b=r5 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t67 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t68 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t69 mm8 a=r6 b=r7 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t70 mm8 a=r1 b=r7 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t71 mm8 a=r3 b=r0 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t72 mm8 a=r4 b=r0 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t73 mm8 a=r4 b=r1 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t74 mm8 a=r5 b=r1 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t75 mm8 a=r4 b=r4 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t76 mm8 a=r5 b=r6 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t77 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t78 mm8 a=r2 b=r5 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t79 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t80 mm8 a=r6 b=r7 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t81 mm8 a=r4 b=r3 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t82 mm8 a=r6 b=r0 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t83 mm8 a=r2 b=r2 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t84 mm8 a=r0 b=r6 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t85 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t86 mm8 a=r3 b=r4 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t87 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t88 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t89 mm8 a=r4 b=r2 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t90 mm8 a=r6 b=r5 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t91 mm8 a=r6 b=r5 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t92 mm8 a=r2 b=r6 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t93 mm8 a=r2 b=r2 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t94 mm8 a=r2 b=r4 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t95 mm8 a=r5 b=r4 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t96 mm8 a=r0 b=r6 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t97 mm8 a=r0 b=r5 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t98 mm8 a=r2 b=r7 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t99 mm8 a=r1 b=r4 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t100 mm8 a=r2 b=r4 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t101 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t102 mm8 a=r5 b=r7 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t103 mm8 a=r4 b=r7 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t104 mm8 a=r2 b=r0 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t105 mm8 a=r6 b=r1 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t106 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t107 mm8 a=r2 b=r1 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t108 mm8 a=r4 b=r6 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t109 mm8 a=r2 b=r7 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t110 mm8 a=r1 b=r0 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t111 mm8 a=r6 b=r5 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t112 mm8 a=r5 b=r7 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t113 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t114 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t115 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t116 mm8 a=r7 b=r2 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t117 mm8 a=r7 b=r6 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t118 mm8 a=r0 b=r1 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t119 mm8 a=r4 b=r3 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t120 mm8 a=r5 b=r3 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t121 mm8 a=r2 b=r7 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t122 mm8 a=r0 b=r1 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t123 mm8 a=r5 b=r2 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t124 mm8 a=r5 b=r1 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t125 mm8 a=r7 b=r2 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t126 mm8 a=r6 b=r3 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t127 mm8 a=r6 b=r7 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t128 mm8 a=r6 b=r7 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t129 mm8 a=r3 b=r2 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t130 mm8 a=r0 b=r4 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t131 mm8 a=r3 b=r5 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t132 mm8 a=r0 b=r7 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t133 mm8 a=r2 b=r6 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t134 mm8 a=r5 b=r6 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t135 mm8 a=r0 b=r2 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t136 mm8 a=r3 b=r1 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t137 mm8 a=r7 b=r0 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t138 mm8 a=r2 b=r2 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t139 mm8 a=r0 b=r4 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t140 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t141 mm8 a=r5 b=r5 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t142 mm8 a=r2 b=r2 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t143 mm8 a=r4 b=r3 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t144 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t145 mm8 a=r7 b=r2 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t146 mm8 a=r2 b=r7 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t147 mm8 a=r3 b=r1 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t148 mm8 a=r2 b=r1 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t149 mm8 a=r4 b=r6 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t150 mm8 a=r3 b=r6 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t151 mm8 a=r1 b=r4 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t152 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t153 mm8 a=r0 b=r3 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t154 mm8 a=r7 b=r3 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t155 mm8 a=r1 b=r3 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t156 mm8 a=r6 b=r3 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t157 mm8 a=r6 b=r4 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t158 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t159 mm8 a=r1 b=r0 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t160 mm8 a=r0 b=r1 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t161 mm8 a=r4 b=r0 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t162 mm8 a=r6 b=r6 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t163 mm8 a=r5 b=r3 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t164 mm8 a=r0 b=r7 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t165 mm8 a=r6 b=r5 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t166 mm8 a=r6 b=r5 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t167 mm8 a=r4 b=r5 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t168 mm8 a=r3 b=r6 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t169 mm8 a=r6 b=r2 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t170 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t171 mm8 a=r6 b=r3 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t172 mm8 a=r7 b=r3 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t173 mm8 a=r0 b=r0 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t174 mm8 a=r6 b=r6 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t175 mm8 a=r1 b=r6 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t176 mm8 a=r2 b=r4 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t177 mm8 a=r1 b=r7 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t178 mm8 a=r4 b=r2 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t179 mm8 a=r5 b=r2 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t180 mm8 a=r0 b=r4 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t181 mm8 a=r3 b=r5 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t182 mm8 a=r1 b=r1 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t183 mm8 a=r3 b=r1 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t184 mm8 a=r7 b=r4 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t185 mm8 a=r5 b=r0 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t186 mm8 a=r2 b=r5 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t187 mm8 a=r3 b=r0 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t188 mm8 a=r1 b=r4 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t189 mm8 a=r7 b=r2 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t190 mm8 a=r1 b=r5 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t191 mm8 a=r2 b=r1 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t192 mm8 a=r5 b=r0 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t193 mm8 a=r2 b=r1 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t194 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t195 mm8 a=r2 b=r7 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t196 mm8 a=r5 b=r6 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t197 mm8 a=r5 b=r3 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t198 mm8 a=r1 b=r0 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t199 mm8 a=r2 b=r5 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t200 mm8 a=r5 b=r7 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t201 mm8 a=r7 b=r2 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t202 mm8 a=r2 b=r5 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t203 mm8 a=r0 b=r5 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t204 mm8 a=r4 b=r7 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t205 mm8 a=r3 b=r7 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t206 mm8 a=r0 b=r6 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t207 mm8 a=r7 b=r7 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t208 mm8 a=r0 b=r1 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t209 mm8 a=r4 b=r4 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t210 mm8 a=r7 b=r1 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t211 mm8 a=r1 b=r5 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t212 mm8 a=r2 b=r4 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t213 mm8 a=r2 b=r2 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t214 mm8 a=r4 b=r6 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t215 mm8 a=r7 b=r3 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t216 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t217 mm8 a=r5 b=r7 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t218 mm8 a=r7 b=r1 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t219 mm8 a=r2 b=r1 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t220 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t221 mm8 a=r7 b=r1 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t222 mm8 a=r0 b=r6 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t223 mm8 a=r2 b=r2 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t224 mm8 a=r1 b=r1 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t225 mm8 a=r2 b=r7 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t226 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t227 mm8 a=r3 b=r5 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t228 mm8 a=r7 b=r1 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t229 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t230 mm8 a=r0 b=r3 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t231 mm8 a=r1 b=r3 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t232 mm8 a=r5 b=r4 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t233 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t234 mm8 a=r2 b=r0 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t235 mm8 a=r3 b=r4 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t236 mm8 a=r1 b=r7 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t237 mm8 a=r3 b=r5 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t238 mm8 a=r2 b=r0 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t239 mm8 a=r5 b=r7 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t240 mm8 a=r2 b=r7 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t241 mm8 a=r0 b=r1 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t242 mm8 a=r0 b=r2 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t243 mm8 a=r4 b=r4 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t244 mm8 a=r7 b=r6 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t245 mm8 a=r4 b=r4 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t246 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t247 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t248 mm8 a=r7 b=r5 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t249 mm8 a=r4 b=r4 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t250 mm8 a=r0 b=r6 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t251 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t252 mm8 a=r0 b=r2 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t253 mm8 a=r5 b=r5 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t254 mm8 a=r6 b=r7 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t255 mm8 a=r5 b=r0 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t256 mm8 a=r2 b=r1 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t257 mm8 a=r4 b=r3 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t258 mm8 a=r4 b=r1 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t259 mm8 a=r5 b=r5 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t260 mm8 a=r3 b=r2 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t261 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t262 mm8 a=r6 b=r4 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t263 mm8 a=r6 b=r4 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t264 mm8 a=r7 b=r2 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t265 mm8 a=r6 b=r1 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t266 mm8 a=r0 b=r0 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t267 mm8 a=r6 b=r3 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t268 mm8 a=r1 b=r0 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t269 mm8 a=r7 b=r7 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t270 mm8 a=r6 b=r2 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t271 mm8 a=r0 b=r3 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t272 mm8 a=r6 b=r0 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t273 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t274 mm8 a=r1 b=r0 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t275 mm8 a=r1 b=r3 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t276 mm8 a=r1 b=r5 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t277 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t278 mm8 a=r0 b=r0 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t279 mm8 a=r2 b=r4 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t280 mm8 a=r7 b=r2 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t281 mm8 a=r1 b=r0 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t282 mm8 a=r0 b=r1 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t283 mm8 a=r0 b=r2 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t284 mm8 a=r1 b=r5 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t285 mm8 a=r4 b=r7 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t286 mm8 a=r1 b=r0 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t287 mm8 a=r7 b=r1 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t288 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t289 mm8 a=r0 b=r5 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t290 mm8 a=r3 b=r2 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t291 mm8 a=r7 b=r4 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t292 mm8 a=r6 b=r7 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t293 mm8 a=r5 b=r5 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t294 mm8 a=r5 b=r7 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t295 mm8 a=r0 b=r7 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t296 mm8 a=r6 b=r1 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t297 mm8 a=r4 b=r5 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t298 mm8 a=r6 b=r0 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t299 mm8 a=r6 b=r0 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t300 mm8 a=r6 b=r4 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t301 mm8 a=r6 b=r4 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t302 mm8 a=r7 b=r6 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t303 mm8 a=r3 b=r2 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t304 mm8 a=r6 b=r5 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t305 mm8 a=r4 b=r5 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t306 mm8 a=r3 b=r1 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t307 mm8 a=r4 b=r3 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t308 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t309 mm8 a=r2 b=r1 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t310 mm8 a=r6 b=r1 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t311 mm8 a=r4 b=r6 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t312 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t313 mm8 a=r2 b=r6 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t314 mm8 a=r0 b=r3 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t315 mm8 a=r7 b=r1 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t316 mm8 a=r2 b=r4 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t317 mm8 a=r1 b=r6 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t318 mm8 a=r2 b=r1 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t319 mm8 a=r0 b=r2 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t320 mm8 a=r5 b=r2 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t321 mm8 a=r7 b=r4 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t322 mm8 a=r2 b=r4 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t323 mm8 a=r4 b=r3 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t324 mm8 a=r5 b=r7 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t325 mm8 a=r6 b=r2 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t326 mm8 a=r6 b=r1 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t327 mm8 a=r1 b=r5 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t328 mm8 a=r0 b=r1 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t329 mm8 a=r0 b=r3 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t330 mm8 a=r6 b=r7 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t331 mm8 a=r4 b=r4 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t332 mm8 a=r4 b=r4 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t333 mm8 a=r4 b=r6 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t334 mm8 a=r2 b=r2 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t335 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t336 mm8 a=r4 b=r0 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t337 mm8 a=r1 b=r4 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t338 mm8 a=r3 b=r7 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t339 mm8 a=r5 b=r1 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t340 mm8 a=r5 b=r0 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t341 mm8 a=r6 b=r6 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t342 mm8 a=r4 b=r7 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t343 mm8 a=r0 b=r4 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t344 mm8 a=r2 b=r5 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t345 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t346 mm8 a=r6 b=r0 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t347 mm8 a=r5 b=r3 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t348 mm8 a=r0 b=r4 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t349 mm8 a=r6 b=r1 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t350 mm8 a=r1 b=r5 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t351 mm8 a=r6 b=r4 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t352 mm8 a=r4 b=r1 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t353 mm8 a=r7 b=r3 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t354 mm8 a=r4 b=r1 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t355 mm8 a=r4 b=r5 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t356 mm8 a=r7 b=r6 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t357 mm8 a=r7 b=r3 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t358 mm8 a=r4 b=r7 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t359 mm8 a=r2 b=r7 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t360 mm8 a=r4 b=r4 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t361 mm8 a=r2 b=r2 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t362 mm8 a=r2 b=r7 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t363 mm8 a=r7 b=r6 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t364 mm8 a=r6 b=r4 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t365 mm8 a=r6 b=r6 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t366 mm8 a=r7 b=r7 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t367 mm8 a=r4 b=r1 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t368 mm8 a=r7 b=r0 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t369 mm8 a=r1 b=r0 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t370 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t371 mm8 a=r5 b=r4 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t372 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t373 mm8 a=r4 b=r6 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t374 mm8 a=r1 b=r5 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t375 mm8 a=r6 b=r5 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t376 mm8 a=r3 b=r6 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t377 mm8 a=r6 b=r4 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t378 mm8 a=r6 b=r3 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t379 mm8 a=r3 b=r0 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t380 mm8 a=r2 b=r0 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t381 mm8 a=r3 b=r5 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t382 mm8 a=r5 b=r2 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t383 mm8 a=r1 b=r2 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t384 mm8 a=r3 b=r1 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t385 mm8 a=r6 b=r0 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t386 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t387 mm8 a=r3 b=r7 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t388 mm8 a=r3 b=r0 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t389 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t390 mm8 a=r5 b=r4 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t391 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t392 mm8 a=r2 b=r3 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t393 mm8 a=r3 b=r6 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t394 mm8 a=r0 b=r7 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t395 mm8 a=r0 b=r7 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t396 mm8 a=r7 b=r7 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t397 mm8 a=r5 b=r7 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t398 mm8 a=r6 b=r5 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t399 mm8 a=r6 b=r3 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t400 mm8 a=r2 b=r7 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t401 mm8 a=r7 b=r6 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t402 mm8 a=r0 b=r0 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t403 mm8 a=r7 b=r4 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t404 mm8 a=r1 b=r1 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t405 mm8 a=r6 b=r6 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t406 mm8 a=r6 b=r1 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t407 mm8 a=r5 b=r7 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t408 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t409 mm8 a=r3 b=r4 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t410 mm8 a=r0 b=r5 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t411 mm8 a=r2 b=r0 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t412 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t413 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t414 mm8 a=r7 b=r1 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t415 mm8 a=r4 b=r7 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t416 mm8 a=r5 b=r4 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t417 mm8 a=r6 b=r6 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t418 mm8 a=r1 b=r3 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t419 mm8 a=r5 b=r0 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t420 mm8 a=r4 b=r2 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t421 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t422 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t423 mm8 a=r1 b=r7 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t424 mm8 a=r5 b=r1 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t425 mm8 a=r2 b=r7 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t426 mm8 a=r1 b=r6 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t427 mm8 a=r6 b=r6 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t428 mm8 a=r6 b=r2 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t429 mm8 a=r3 b=r3 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t430 mm8 a=r7 b=r7 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t431 mm8 a=r6 b=r4 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t432 mm8 a=r3 b=r2 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t433 mm8 a=r3 b=r0 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t434 mm8 a=r0 b=r7 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t435 mm8 a=r7 b=r5 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t436 mm8 a=r6 b=r4 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t437 mm8 a=r0 b=r7 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t438 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t439 mm8 a=r4 b=r7 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t440 mm8 a=r3 b=r5 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t441 mm8 a=r5 b=r1 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t442 mm8 a=r1 b=r2 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t443 mm8 a=r1 b=r7 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t444 mm8 a=r3 b=r7 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t445 mm8 a=r7 b=r5 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t446 mm8 a=r3 b=r1 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t447 mm8 a=r6 b=r7 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t448 mm8 a=r5 b=r2 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t449 mm8 a=r7 b=r1 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t450 mm8 a=r7 b=r5 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t451 mm8 a=r0 b=r3 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t452 mm8 a=r2 b=r4 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t453 mm8 a=r5 b=r7 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t454 mm8 a=r7 b=r7 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t455 mm8 a=r7 b=r1 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t456 mm8 a=r6 b=r5 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t457 mm8 a=r7 b=r7 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t458 mm8 a=r5 b=r1 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t459 mm8 a=r6 b=r7 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t460 mm8 a=r7 b=r7 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t461 mm8 a=r0 b=r6 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t462 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t463 mm8 a=r2 b=r0 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t464 mm8 a=r0 b=r3 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t465 mm8 a=r1 b=r7 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t466 mm8 a=r1 b=r2 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t467 mm8 a=r1 b=r0 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t468 mm8 a=r4 b=r1 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t469 mm8 a=r2 b=r0 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t470 mm8 a=r3 b=r1 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t471 mm8 a=r2 b=r2 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t472 mm8 a=r1 b=r7 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t473 mm8 a=r2 b=r7 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t474 mm8 a=r5 b=r3 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t475 mm8 a=r7 b=r6 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t476 mm8 a=r3 b=r0 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t477 mm8 a=r3 b=r1 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t478 mm8 a=r0 b=r6 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t479 mm8 a=r7 b=r6 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t480 mm8 a=r6 b=r0 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t481 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t482 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t483 mm8 a=r7 b=r0 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t484 mm8 a=r3 b=r7 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t485 mm8 a=r6 b=r4 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t486 mm8 a=r6 b=r5 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t487 mm8 a=r2 b=r5 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t488 mm8 a=r1 b=r2 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t489 mm8 a=r5 b=r2 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t490 mm8 a=r7 b=r1 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t491 mm8 a=r0 b=r0 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t492 mm8 a=r3 b=r1 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t493 mm8 a=r6 b=r1 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t494 mm8 a=r5 b=r6 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t495 mm8 a=r2 b=r2 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t496 mm8 a=r3 b=r5 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t497 mm8 a=r3 b=r2 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t498 mm8 a=r3 b=r1 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t499 mm8 a=r1 b=r6 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t500 mm8 a=r7 b=r6 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t501 mm8 a=r4 b=r4 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t502 mm8 a=r5 b=r7 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t503 mm8 a=r7 b=r5 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t504 mm8 a=r2 b=r0 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t505 mm8 a=r4 b=r0 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t506 mm8 a=r0 b=r2 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t507 mm8 a=r5 b=r6 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t508 mm8 a=r7 b=r3 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t509 mm8 a=r4 b=r4 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t510 mm8 a=r0 b=r6 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t511 mm8 a=r2 b=r4 c=r1 c2=r3 - } - uint32_t lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint32_t hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((uint64_t)hi << 32) | (uint64_t)lo; -} - -// Host-side launch wrappers. Declared in program.h, called from host.cu. -cudaError_t igneum_launch_cache_fill(uint32_t* cache, uint32_t nSegments) { - if (nSegments == 0u) return cudaErrorInvalidValue; - uint32_t block = 256u; - uint32_t grid = (nSegments + block - 1u) / block; - igneum_cache_fill<<>>(cache, nSegments); - return cudaGetLastError(); -} - -cudaError_t igneum_launch_build(uint32_t* ds, const uint32_t* cache, uint32_t nItems) { - if (nItems == 0u) return cudaErrorInvalidValue; - uint32_t block = 256u; - uint32_t grid = (nItems + block - 1u) / block; - igneum_build<<>>(ds, cache, nItems); - return cudaGetLastError(); -} - -cudaError_t igneum_launch_hash(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask, - uint32_t nonces, uint32_t blockWarps) { - if (blockWarps == 0u || blockWarps > 32u) return cudaErrorInvalidValue; - uint32_t block = 32u * blockWarps; - if (nonces == 0u || (nonces % block) != 0u) return cudaErrorInvalidValue; - igneum_hash<<>>(ds, out, baseNonce, mask); - return cudaGetLastError(); -} - -cudaError_t igneum_hash_info(int* numRegs, int* blocksPerSM, uint32_t blockWarps) { - cudaFuncAttributes attr; - cudaError_t e = cudaFuncGetAttributes(&attr, igneum_hash); - if (e != cudaSuccess) return e; - *numRegs = attr.numRegs; - return cudaOccupancyMaxActiveBlocksPerMultiprocessor(blocksPerSM, igneum_hash, (int)(32u * blockWarps), 0); -} diff --git a/proto-cuda/packs-ca4/mx8_mm512/kernel_bound.cl b/proto-cuda/packs-ca4/mx8_mm512/kernel_bound.cl deleted file mode 100644 index fe2a6f822..000000000 --- a/proto-cuda/packs-ca4/mx8_mm512/kernel_bound.cl +++ /dev/null @@ -1,1407 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// OpenCL C twin of the Metal kernel for the same seed (see proto-opencl/README.md, WAVEFRONT.md and program.metal). -// Built from source at runtime by proto-opencl/host.c, which passes these defines: -// IGNEUM_GROUP work-group size of igneum_hash, a multiple of 32 (default 32: one work-group = one 32-lane unit) -// IGNEUM_EXCHANGE 0 = local-memory exchange with a barrier (any device, any wave width; the default) -// 1 = sub_group_shuffle_xor (cl_khr_subgroup_shuffle), only with IGNEUM_GROUP 32 and a sub-group size of exactly 32 -// 2 = intel_sub_group_shuffle_xor (cl_intel_subgroups), same condition -// The verification unit is always 32 lanes. A 64-wide hardware wave (AMD GCN/CDNA, RDNA in wave64) runs two units; -// the exchange masks are 1, 2, 4, 8, 16, so every partner lane lies inside the lane's own aligned run of 32. -#ifndef IGNEUM_GROUP -#define IGNEUM_GROUP 32 -#endif -#ifndef IGNEUM_EXCHANGE -#define IGNEUM_EXCHANGE 0 -#endif -#ifdef __OPENCL_VERSION__ -#define IGNEUM_KERNEL_HASH __kernel __attribute__((reqd_work_group_size(IGNEUM_GROUP, 1, 1))) -#define IGNEUM_LOCAL_WORDS(name, n) __local uint name[n] -#if IGNEUM_EXCHANGE == 1 -#ifdef cl_khr_subgroups -#pragma OPENCL EXTENSION cl_khr_subgroups : enable -#endif -#ifdef cl_khr_subgroup_shuffle -#pragma OPENCL EXTENSION cl_khr_subgroup_shuffle : enable -#endif -#elif IGNEUM_EXCHANGE == 2 -#pragma OPENCL EXTENSION cl_intel_subgroups : enable -#endif -#else -// Not an OpenCL compiler: proto-opencl/emu compiles this file as C++ and supplies the built-ins and these two macros. -#include "emu_opencl.h" -#endif - -#if IGNEUM_EXCHANGE == 1 -#define IGNEUM_SHFL_XOR(dst, a, m) dst = sub_group_shuffle_xor((a), (uint)(m)) -#define IGNEUM_BCAST0(dst, a) dst = sub_group_broadcast((a), 0u) -#define IGNEUM_SHFL_IDX(dst, a, i) dst = sub_group_shuffle((a), (uint)(i)) -#elif IGNEUM_EXCHANGE == 2 -#define IGNEUM_SHFL_XOR(dst, a, m) dst = intel_sub_group_shuffle_xor((a), (uint)(m)) -#define IGNEUM_BCAST0(dst, a) dst = sub_group_broadcast((a), 0u) -#define IGNEUM_SHFL_IDX(dst, a, i) dst = intel_sub_group_shuffle((a), (uint)(i)) -#else -// Local-memory exchange. Two buffers of IGNEUM_GROUP words alternate (xk counts exchanges), so one barrier per -// exchange is enough: a lane can only overwrite buffer b at exchange k+2 after passing barrier k+1, and every lane -// reaches barrier k+1 only after its read of buffer b at exchange k. The partner lid ^ m stays inside the lane's -// aligned run of 32 because m < 32. Control flow is uniform, so every work-item reaches every barrier. -#define IGNEUM_SHFL_XOR(dst, a, m) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid ^ (uint)(m))]; xk += 1u; } -#define IGNEUM_BCAST0(dst, a) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid & ~31u)]; xk += 1u; } -#define IGNEUM_SHFL_IDX(dst, a, i) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid & ~31u) + (uint)(i)]; xk += 1u; } -#endif - -// int8 tile reference (Counter ASIC 4.0 research): 12 index shuffles and 32 byte products per lane; the PTX mma.m8n8k16 u8 layout. -#define IGNEUM_MM8_REF(av, bv, d0, d1) { uint row_ = lane >> 2, col0_ = 2u * (lane & 3u); uint acc0_ = 0u, acc1_ = 0u; for (uint j_ = 0u; j_ < 4u; ++j_) { uint aw_, bw0_, bw1_; IGNEUM_SHFL_IDX(aw_, (av), 4u * row_ + j_); IGNEUM_SHFL_IDX(bw0_, (bv), 4u * col0_ + j_); IGNEUM_SHFL_IDX(bw1_, (bv), 4u * (col0_ + 1u) + j_); for (uint t_ = 0u; t_ < 4u; ++t_) { uint ab_ = (aw_ >> (8u * t_)) & 0xffu; acc0_ += ab_ * ((bw0_ >> (8u * t_)) & 0xffu); acc1_ += ab_ * ((bw1_ >> (8u * t_)) & 0xffu); } } d0 = acc0_; d1 = acc1_; } - -static inline uint splitmix32(uint x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -// n is a literal in 1..31 at every call site. OpenCL rotate() rotates left by n modulo 32. -static inline uint rotl_imm(uint x, uint n) { return rotate(x, n); } -// Right rotation by n modulo 32 as a left rotation by (32 - n) modulo 32; n == 0 gives x. -static inline uint rotr_var(uint x, uint n) { return rotate(x, (0u - n) & 31u); } -static inline uint ds_elem(uint i, uint d0, uint d1) { - uint x = i ^ d0; - x *= 0x9E3779B1u; x ^= x >> 15; - x += d1; - x *= 0x85EBCA77u; x ^= x >> 13; - x *= 0xC2B2AE3Du; x ^= x >> 16; - return x; -} - -// Memory-hard dataset core (MEMHARD.md). Cache: 2^26 words in 2^16 segments of 64 chained ChaCha12 lines. -// Item: 8 rounds of 8 x seed-parameterised mixer + one 64-byte cache read, then 8 x final mixer (class v3, mixer multiplier 8, -// docs/plans/mixer-x4.md: the round key of application j of round r is 0x9E3779B9 * (r * m + j + 1)). All parameters are literals. -#define MH_CACHE_LINE_MASK 0x003fffffu -#define MH_SEGMENT_LINES 64u -#define MH_QR(a, b, c, d, r1, r2, r3, r4) { a += b; d ^= a; d = mh_rotl(d, r1); c += d; b ^= c; b = mh_rotl(b, r2); a += b; d ^= a; d = mh_rotl(d, r3); c += d; b ^= c; b = mh_rotl(b, r4); } -static inline uint mh_rotl(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 at every call site - -// y = ChaCha12 core(x) + x -static inline void mh_chacha_block(const uint* x, uint* y) { - for (uint i = 0u; i < 16u; ++i) y[i] = x[i]; - for (uint r = 0u; r < 6u; ++r) { - MH_QR(y[0], y[4], y[8], y[12], 16u, 12u, 8u, 7u) MH_QR(y[1], y[5], y[9], y[13], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[6], y[10], y[14], 16u, 12u, 8u, 7u) MH_QR(y[3], y[7], y[11], y[15], 16u, 12u, 8u, 7u) - MH_QR(y[0], y[5], y[10], y[15], 16u, 12u, 8u, 7u) MH_QR(y[1], y[6], y[11], y[12], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[7], y[8], y[13], 16u, 12u, 8u, 7u) MH_QR(y[3], y[4], y[9], y[14], 16u, 12u, 8u, 7u) - } - for (uint i = 0u; i < 16u; ++i) y[i] += x[i]; -} - -// One cache segment: 64 chained lines written at cache[seg * 1024]. in_j = prev ^ (sigma || K || seg || j || tag), prev_0 = 0. -static inline void mh_cache_segment(__global uint* cache, uint seg) { - uint prev[16]; uint x[16]; uint y[16]; - for (uint i = 0u; i < 16u; ++i) prev[i] = 0u; - for (uint j = 0u; j < MH_SEGMENT_LINES; ++j) { - x[0] = 0x61707865u ^ prev[0]; x[1] = 0x3320646eu ^ prev[1]; x[2] = 0x79622d32u ^ prev[2]; x[3] = 0x6b206574u ^ prev[3]; - x[4] = 0x3067619fu ^ prev[4]; - x[5] = 0x3c269176u ^ prev[5]; - x[6] = 0x84a03b03u ^ prev[6]; - x[7] = 0xf8c63294u ^ prev[7]; - x[8] = 0xff977c5bu ^ prev[8]; - x[9] = 0xe60def3eu ^ prev[9]; - x[10] = 0x63630141u ^ prev[10]; - x[11] = 0xb8fbcb58u ^ prev[11]; - x[12] = seg ^ prev[12]; x[13] = j ^ prev[13]; x[14] = 0x49676e65u ^ prev[14]; x[15] = 0x756d4d48u ^ prev[15]; - mh_chacha_block(x, y); - __global uint* line = cache + ((seg * MH_SEGMENT_LINES + j) * 16u); - for (uint i = 0u; i < 16u; ++i) { line[i] = y[i]; prev[i] = y[i]; } - } -} - -// M_r: per word (s ^ (RC + rk)) * MUL, then a column round and a diagonal round with the seed-drawn rotations. -static inline void mh_mixer(uint* s, uint rk) { - s[0] = (s[0] ^ (0xbab68293u + rk)) * 0x42146205u; - s[1] = (s[1] ^ (0xcc162340u + rk)) * 0x52cbe0fbu; - s[2] = (s[2] ^ (0x6ce151ccu + rk)) * 0x7ecf4a03u; - s[3] = (s[3] ^ (0xe62b8997u + rk)) * 0x6728907fu; - s[4] = (s[4] ^ (0xc9c80297u + rk)) * 0xd81d9751u; - s[5] = (s[5] ^ (0xf74a1654u + rk)) * 0x132952c3u; - s[6] = (s[6] ^ (0x3d704af5u + rk)) * 0xf60de277u; - s[7] = (s[7] ^ (0x3cf522b7u + rk)) * 0x05358035u; - s[8] = (s[8] ^ (0x2b9cac04u + rk)) * 0xbaf6499du; - s[9] = (s[9] ^ (0xa880ac10u + rk)) * 0xe4db9667u; - s[10] = (s[10] ^ (0x13e5dd1du + rk)) * 0x3e98f45du; - s[11] = (s[11] ^ (0x6fc3e233u + rk)) * 0xd0004eddu; - s[12] = (s[12] ^ (0x2d83eeacu + rk)) * 0x2691630du; - s[13] = (s[13] ^ (0x9006e8bfu + rk)) * 0x9beb3bcfu; - s[14] = (s[14] ^ (0x2c4b5362u + rk)) * 0xab310379u; - s[15] = (s[15] ^ (0x31b49ee2u + rk)) * 0x99cfb423u; - MH_QR(s[0], s[4], s[8], s[12], 20u, 20u, 19u, 4u) MH_QR(s[1], s[5], s[9], s[13], 20u, 20u, 19u, 4u) - MH_QR(s[2], s[6], s[10], s[14], 20u, 20u, 19u, 4u) MH_QR(s[3], s[7], s[11], s[15], 20u, 20u, 19u, 4u) - MH_QR(s[0], s[5], s[10], s[15], 26u, 3u, 3u, 27u) MH_QR(s[1], s[6], s[11], s[12], 26u, 3u, 3u, 27u) - MH_QR(s[2], s[7], s[8], s[13], 26u, 3u, 3u, 27u) MH_QR(s[3], s[4], s[9], s[14], 26u, 3u, 3u, 27u) -} - -// Item t: 16 words. s = (K, t * MUL[i] + RC[i]); 8 rounds of 8 x mixer + cache line s[0] & mask; 8 x final mixer. -static inline void mh_item(__global const uint* cache, uint t, uint* s) { - s[0] = 0x3067619fu; - s[1] = 0x3c269176u; - s[2] = 0x84a03b03u; - s[3] = 0xf8c63294u; - s[4] = 0xff977c5bu; - s[5] = 0xe60def3eu; - s[6] = 0x63630141u; - s[7] = 0xb8fbcb58u; - s[8] = t * 0x42146205u + 0xbab68293u; - s[9] = t * 0x52cbe0fbu + 0xcc162340u; - s[10] = t * 0x7ecf4a03u + 0x6ce151ccu; - s[11] = t * 0x6728907fu + 0xe62b8997u; - s[12] = t * 0xd81d9751u + 0xc9c80297u; - s[13] = t * 0x132952c3u + 0xf74a1654u; - s[14] = t * 0xf60de277u + 0x3d704af5u; - s[15] = t * 0x05358035u + 0x3cf522b7u; - for (uint r = 0u; r < 8u; ++r) { - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (r * 8u + j + 1u)); - __global const uint* line = cache + ((s[0] & MH_CACHE_LINE_MASK) * 16u); - for (uint i = 0u; i < 16u; ++i) s[i] ^= line[i]; - } - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (64u + j + 1u)); -} -// dataset[w] without the dataset: derive item w >> 4 and take word w & 15. -static inline uint mh_word(__global const uint* cache, uint w) { uint s[16]; mh_item(cache, w >> 4u, s); return s[w & 15u]; } - -// Memory-hard dataset (MEMHARD.md). One work-item per cache segment; one work-item per 64-byte dataset item. -// The same constants as memhard.h in this pack (one emitter, three dialects). -__kernel void igneum_cache_fill(__global uint* cache, uint nSegments) { - uint seg = (uint)get_global_id(0); - if (seg < nSegments) mh_cache_segment(cache, seg); -} -__kernel void igneum_build(__global uint* ds, __global const uint* cache, uint nItems) { - uint t = (uint)get_global_id(0); - if (t < nItems) { - uint s[16]; - mh_item(cache, t, s); - __global uint* d = ds + ((ulong)t * 16u); - for (uint i = 0u; i < 16u; ++i) d[i] = s[i]; - } -} - -// One hash per work-item. IGNEUM_GROUP is a multiple of 32; lane = lid & 31 and every exchange stays inside the -// lane's own aligned run of 32 work-items, exactly like simd_shuffle_xor inside a 32-wide Metal SIMD group and -// __shfl_xor_sync inside a CUDA warp. Control flow is uniform (no branches at all). -IGNEUM_KERNEL_HASH void igneum_hash(__global const uint* ds, __global ulong* out, uint baseNonce, uint mask) { - uint gid = (uint)get_global_id(0); - uint lid = (uint)get_local_id(0); - uint nonce = baseNonce + gid; - uint r0, r1, r2, r3, r4, r5, r6, r7; -#if IGNEUM_EXCHANGE == 0 - IGNEUM_LOCAL_WORDS(xch, 2 * IGNEUM_GROUP); - uint xk = 0u; -#else - (void)lid; -#endif - uint lane = lid & 31u; - { uint x = nonce ^ 0x67a9a7beu; x += 0x9e3779b9u; x = splitmix32(x); r0 = x ^ 0x1a155b25u; } // SEEDW[0], 0x9e3779b9u * 1u, SEEDW[1] - { uint x = nonce ^ 0x1a155b25u; x += 0x3c6ef372u; x = splitmix32(x); r1 = x ^ 0xfddfb732u; } // SEEDW[1], 0x9e3779b9u * 2u, SEEDW[2] - { uint x = nonce ^ 0xfddfb732u; x += 0xdaa66d2bu; x = splitmix32(x); r2 = x ^ 0x4b5af2e8u; } // SEEDW[2], 0x9e3779b9u * 3u, SEEDW[3] - { uint x = nonce ^ 0x4b5af2e8u; x += 0x78dde6e4u; x = splitmix32(x); r3 = x ^ 0xc55caf33u; } // SEEDW[3], 0x9e3779b9u * 4u, SEEDW[4] - { uint x = nonce ^ 0xc55caf33u; x += 0x1715609du; x = splitmix32(x); r4 = x ^ 0xa27c13b7u; } // SEEDW[4], 0x9e3779b9u * 5u, SEEDW[5] - { uint x = nonce ^ 0xa27c13b7u; x += 0xb54cda56u; x = splitmix32(x); r5 = x ^ 0x06628a48u; } // SEEDW[5], 0x9e3779b9u * 6u, SEEDW[6] - { uint x = nonce ^ 0x06628a48u; x += 0x5384540fu; x = splitmix32(x); r6 = x ^ 0x03852469u; } // SEEDW[6], 0x9e3779b9u * 7u, SEEDW[7] - { uint x = nonce ^ 0x03852469u; x += 0xf1bbcdc8u; x = splitmix32(x); r7 = x ^ 0x67a9a7beu; } // SEEDW[7], 0x9e3779b9u * 8u, SEEDW[0] - - for (uint it = 0u; it < 8u; ++it) { - uint sel = r0; - r2 = r3 * r4 + r2; // 0 mad - r2 = r1 * r1 + r2; // 1 mad - r2 = r3 * r2 + r2; // 2 mad - r3 = r3 ^ r5; // 3 xor - r7 = r7 ^ ds[r2 & mask]; // 4 load - r5 = r5 ^ ds[r7 & mask]; // 5 load - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 8u); r1 = r1 ^ t_; } // 6 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 8u); r7 = r7 ^ t_; } // 7 shfl - r1 = mul_hi(r1, r5); // 8 mulhi - r6 = rotr_var(r6, r3); // 9 rotr - r3 = r3 | r4; // 10 or - r4 = r4 ^ ds[r3 & mask]; // 11 load - r0 = mul_hi(r0, r4); // 12 mulhi - r5 = r5 + r1 + ((((sel >> 30u) & 1u) != 0u) ? 0xd3177981u : 0xc7934706u); // 13 add - r0 = r0 ^ ds[r4 & mask]; // 14 load - r2 = r2 - r4; // 15 sub - r2 = r2 ^ ds[r0 & mask]; // 16 load - r7 = r7 ^ ds[r2 & mask]; // 17 load - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 4u); r7 = r7 ^ t_; } // 18 shfl - r5 = r5 * r0; // 19 mul - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 2u); r3 = r3 ^ t_; } // 20 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 16u); r2 = r2 ^ t_; } // 21 shfl - r6 = mul_hi(r6, r2); // 22 mulhi - r6 = r6 ^ ds[r1 & mask]; // 23 load - r5 = r5 * r0; // 24 mul - r5 = rotl_imm(r5, 19u); // 25 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 2u); r7 = r7 ^ t_; } // 26 shfl - r0 = r0 ^ r5; // 27 xor - r0 = r0 ^ r4; // 28 xor - r3 = r3 - r0; // 29 sub - r5 = r5 * r1; // 30 mul - r7 = r7 ^ ds[r2 & mask]; // 31 load - r1 = r1 ^ ds[r0 & mask]; // 32 load - r5 = r5 ^ r6; // 33 xor - r5 = r5 ^ ds[r1 & mask]; // 34 load - r0 = mul_hi(r0, r5); // 35 mulhi - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 4u); r5 = r5 ^ t_; } // 36 shfl - r7 = r7 ^ ds[r0 & mask]; // 37 load - r3 = r3 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0x230c005cu : 0x75ba2fadu); // 38 add - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 4u); r1 = r1 ^ t_; } // 39 shfl - r2 = r2 ^ r5; // 40 xor - r3 = r6 * r3 + r3; // 41 mad - r6 = r6 - r7; // 42 sub - r7 = r7 ^ r0; // 43 xor - r1 = r1 ^ ds[r7 & mask]; // 44 load - r2 = r2 * r3; // 45 mul - r1 = mul_hi(r1, r5); // 46 mulhi - r4 = r4 - r3; // 47 sub - r2 = rotr_var(r2, r6); // 48 rotr - r3 = r3 ^ ds[r5 & mask]; // 49 load - r1 = r1 + r5 + ((((sel >> 7u) & 1u) != 0u) ? 0x1907970cu : 0x81b8bc2cu); // 50 add - r0 = r0 * r2; // 51 mul - r0 = r0 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x699fd448u : 0x4f92b968u); // 52 add - r1 = r1 + r0 + ((((sel >> 12u) & 1u) != 0u) ? 0x77b1520du : 0x2bb965afu); // 53 add - r7 = rotl_imm(r7, 14u); // 54 rotl - r3 = r3 + r7 + ((((sel >> 1u) & 1u) != 0u) ? 0xa54c55a0u : 0x7b0fe07au); // 55 add - r6 = r6 ^ ds[r7 & mask]; // 56 load - r1 = rotr_var(r1, r5); // 57 rotr - r5 = r5 ^ ds[r4 & mask]; // 58 load - r6 = r6 ^ ds[r2 & mask]; // 59 load - r3 = r5 * r0 + r3; // 60 mad - r5 = r5 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0xad7493e7u : 0xaf9dd72du); // 61 add - r4 = r4 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x1e07c3d9u : 0x89841d87u); // 62 add - r5 = rotl_imm(r5, 19u); // 63 rotl - // int8 tile block (Counter ASIC 4.0 research, experimental): 512 mma.m8n8k16 u8 tiles per iteration, both outputs consumed - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r2 += d0_; r3 += d1_; } // t0 mm8 a=r6 b=r5 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r4 += d0_; r5 += d1_; } // t1 mm8 a=r1 b=r1 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r0 += d0_; r7 += d1_; } // t2 mm8 a=r2 b=r5 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r3 += d1_; } // t3 mm8 a=r1 b=r5 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r2 += d0_; r6 += d1_; } // t4 mm8 a=r2 b=r0 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r7 += d0_; r3 += d1_; } // t5 mm8 a=r2 b=r6 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t6 mm8 a=r1 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r0 += d0_; r4 += d1_; } // t7 mm8 a=r4 b=r6 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r1 += d0_; r5 += d1_; } // t8 mm8 a=r1 b=r1 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r4 += d0_; r6 += d1_; } // t9 mm8 a=r3 b=r7 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r0 += d0_; r4 += d1_; } // t10 mm8 a=r3 b=r0 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r0 += d0_; r6 += d1_; } // t11 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r6 += d0_; r0 += d1_; } // t12 mm8 a=r3 b=r1 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r5 += d1_; } // t13 mm8 a=r1 b=r5 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r3 += d0_; r4 += d1_; } // t14 mm8 a=r3 b=r3 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r5 += d0_; r4 += d1_; } // t15 mm8 a=r1 b=r0 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r0 += d0_; r4 += d1_; } // t16 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r7 += d1_; } // t17 mm8 a=r0 b=r5 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r2 += d0_; r0 += d1_; } // t18 mm8 a=r1 b=r1 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r2 += d0_; r4 += d1_; } // t19 mm8 a=r3 b=r3 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r6 += d0_; r1 += d1_; } // t20 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r6 += d0_; r0 += d1_; } // t21 mm8 a=r0 b=r3 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r4 += d1_; } // t22 mm8 a=r1 b=r7 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r0 += d0_; r3 += d1_; } // t23 mm8 a=r1 b=r2 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r0 += d0_; r2 += d1_; } // t24 mm8 a=r3 b=r4 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r3 += d0_; r7 += d1_; } // t25 mm8 a=r5 b=r0 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t26 mm8 a=r1 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r0 += d0_; r5 += d1_; } // t27 mm8 a=r3 b=r2 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r5 += d1_; } // t28 mm8 a=r2 b=r4 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r1 += d0_; r5 += d1_; } // t29 mm8 a=r3 b=r4 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r6 += d0_; r3 += d1_; } // t30 mm8 a=r6 b=r5 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t31 mm8 a=r1 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r3 += d0_; r0 += d1_; } // t32 mm8 a=r7 b=r6 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r4 += d0_; r6 += d1_; } // t33 mm8 a=r7 b=r1 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r5 += d0_; r1 += d1_; } // t34 mm8 a=r1 b=r0 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t35 mm8 a=r2 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r7 += d0_; r1 += d1_; } // t36 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r0 += d0_; r2 += d1_; } // t37 mm8 a=r1 b=r7 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r2 += d0_; r4 += d1_; } // t38 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r2 += d0_; r5 += d1_; } // t39 mm8 a=r6 b=r2 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r4 += d0_; r1 += d1_; } // t40 mm8 a=r6 b=r1 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r6 += d0_; r2 += d1_; } // t41 mm8 a=r6 b=r0 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r5 += d0_; r1 += d1_; } // t42 mm8 a=r7 b=r4 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r6 += d0_; r0 += d1_; } // t43 mm8 a=r6 b=r1 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r4 += d0_; r1 += d1_; } // t44 mm8 a=r0 b=r6 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r6 += d0_; r5 += d1_; } // t45 mm8 a=r6 b=r5 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r1 += d0_; r0 += d1_; } // t46 mm8 a=r6 b=r6 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r3 += d0_; r1 += d1_; } // t47 mm8 a=r7 b=r3 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t48 mm8 a=r7 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r1 += d0_; r7 += d1_; } // t49 mm8 a=r4 b=r5 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r4 += d0_; r5 += d1_; } // t50 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r0 += d0_; r3 += d1_; } // t51 mm8 a=r3 b=r4 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r7 += d0_; r4 += d1_; } // t52 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r1 += d0_; r3 += d1_; } // t53 mm8 a=r7 b=r7 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t54 mm8 a=r7 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r3 += d0_; r7 += d1_; } // t55 mm8 a=r1 b=r7 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t56 mm8 a=r4 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r6 += d0_; r7 += d1_; } // t57 mm8 a=r4 b=r3 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r6 += d0_; r0 += d1_; } // t58 mm8 a=r2 b=r3 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r7 += d1_; } // t59 mm8 a=r6 b=r7 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t60 mm8 a=r6 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r2 += d0_; r6 += d1_; } // t61 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r6 += d1_; } // t62 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t63 mm8 a=r1 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r5 += d0_; r3 += d1_; } // t64 mm8 a=r0 b=r5 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r2 += d0_; r4 += d1_; } // t65 mm8 a=r0 b=r1 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r1 += d0_; r3 += d1_; } // t66 mm8 a=r3 b=r5 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r3 += d0_; r4 += d1_; } // t67 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r2 += d1_; } // t68 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r4 += d1_; } // t69 mm8 a=r6 b=r7 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r1 += d0_; r5 += d1_; } // t70 mm8 a=r1 b=r7 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r1 += d0_; r0 += d1_; } // t71 mm8 a=r3 b=r0 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r3 += d0_; r6 += d1_; } // t72 mm8 a=r4 b=r0 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r0 += d0_; r2 += d1_; } // t73 mm8 a=r4 b=r1 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r3 += d0_; r4 += d1_; } // t74 mm8 a=r5 b=r1 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r7 += d0_; r3 += d1_; } // t75 mm8 a=r4 b=r4 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r1 += d0_; r7 += d1_; } // t76 mm8 a=r5 b=r6 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r3 += d0_; r2 += d1_; } // t77 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r0 += d0_; r5 += d1_; } // t78 mm8 a=r2 b=r5 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t79 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r3 += d0_; r4 += d1_; } // t80 mm8 a=r6 b=r7 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r1 += d0_; r6 += d1_; } // t81 mm8 a=r4 b=r3 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r1 += d0_; r7 += d1_; } // t82 mm8 a=r6 b=r0 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r1 += d0_; r6 += d1_; } // t83 mm8 a=r2 b=r2 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t84 mm8 a=r0 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t85 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r2 += d0_; r3 += d1_; } // t86 mm8 a=r3 b=r4 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r3 += d0_; r7 += d1_; } // t87 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r5 += d0_; r3 += d1_; } // t88 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r2 += d0_; r1 += d1_; } // t89 mm8 a=r4 b=r2 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r3 += d0_; r6 += d1_; } // t90 mm8 a=r6 b=r5 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r5 += d0_; r4 += d1_; } // t91 mm8 a=r6 b=r5 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r2 += d0_; r3 += d1_; } // t92 mm8 a=r2 b=r6 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r5 += d0_; r7 += d1_; } // t93 mm8 a=r2 b=r2 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r7 += d0_; r3 += d1_; } // t94 mm8 a=r2 b=r4 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r1 += d0_; r6 += d1_; } // t95 mm8 a=r5 b=r4 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r3 += d0_; r1 += d1_; } // t96 mm8 a=r0 b=r6 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r2 += d1_; } // t97 mm8 a=r0 b=r5 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t98 mm8 a=r2 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r0 += d0_; r2 += d1_; } // t99 mm8 a=r1 b=r4 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r2 += d0_; r5 += d1_; } // t100 mm8 a=r2 b=r4 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r7 += d1_; } // t101 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t102 mm8 a=r5 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t103 mm8 a=r4 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r3 += d0_; r7 += d1_; } // t104 mm8 a=r2 b=r0 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r0 += d0_; r5 += d1_; } // t105 mm8 a=r6 b=r1 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r0 += d0_; r5 += d1_; } // t106 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r4 += d0_; r2 += d1_; } // t107 mm8 a=r2 b=r1 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t108 mm8 a=r4 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r5 += d0_; r3 += d1_; } // t109 mm8 a=r2 b=r7 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r0 += d0_; r2 += d1_; } // t110 mm8 a=r1 b=r0 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r4 += d0_; r6 += d1_; } // t111 mm8 a=r6 b=r5 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r4 += d0_; r2 += d1_; } // t112 mm8 a=r5 b=r7 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t113 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t114 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r1 += d0_; r4 += d1_; } // t115 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r5 += d0_; r0 += d1_; } // t116 mm8 a=r7 b=r2 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r7 += d0_; r6 += d1_; } // t117 mm8 a=r7 b=r6 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r2 += d0_; r1 += d1_; } // t118 mm8 a=r0 b=r1 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r5 += d0_; r3 += d1_; } // t119 mm8 a=r4 b=r3 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r7 += d0_; r2 += d1_; } // t120 mm8 a=r5 b=r3 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r1 += d1_; } // t121 mm8 a=r2 b=r7 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r3 += d0_; r0 += d1_; } // t122 mm8 a=r0 b=r1 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r3 += d0_; r5 += d1_; } // t123 mm8 a=r5 b=r2 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t124 mm8 a=r5 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r7 += d0_; r2 += d1_; } // t125 mm8 a=r7 b=r2 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r3 += d0_; r2 += d1_; } // t126 mm8 a=r6 b=r3 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r1 += d1_; } // t127 mm8 a=r6 b=r7 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r4 += d0_; r1 += d1_; } // t128 mm8 a=r6 b=r7 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r6 += d0_; r3 += d1_; } // t129 mm8 a=r3 b=r2 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r2 += d0_; r1 += d1_; } // t130 mm8 a=r0 b=r4 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r3 += d0_; r1 += d1_; } // t131 mm8 a=r3 b=r5 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r0 += d0_; r6 += d1_; } // t132 mm8 a=r0 b=r7 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r0 += d0_; r1 += d1_; } // t133 mm8 a=r2 b=r6 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r3 += d0_; r1 += d1_; } // t134 mm8 a=r5 b=r6 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r4 += d0_; r5 += d1_; } // t135 mm8 a=r0 b=r2 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r1 += d0_; r5 += d1_; } // t136 mm8 a=r3 b=r1 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r4 += d0_; r6 += d1_; } // t137 mm8 a=r7 b=r0 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r4 += d0_; r2 += d1_; } // t138 mm8 a=r2 b=r2 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r2 += d0_; r7 += d1_; } // t139 mm8 a=r0 b=r4 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r6 += d1_; } // t140 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r1 += d0_; r2 += d1_; } // t141 mm8 a=r5 b=r5 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r6 += d0_; r2 += d1_; } // t142 mm8 a=r2 b=r2 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r7 += d0_; r1 += d1_; } // t143 mm8 a=r4 b=r3 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r1 += d0_; r7 += d1_; } // t144 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r6 += d0_; r5 += d1_; } // t145 mm8 a=r7 b=r2 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r7 += d1_; } // t146 mm8 a=r2 b=r7 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r3 += d0_; r0 += d1_; } // t147 mm8 a=r3 b=r1 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r0 += d0_; r6 += d1_; } // t148 mm8 a=r2 b=r1 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r7 += d0_; r1 += d1_; } // t149 mm8 a=r4 b=r6 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r7 += d0_; r0 += d1_; } // t150 mm8 a=r3 b=r6 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r2 += d0_; r4 += d1_; } // t151 mm8 a=r1 b=r4 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r5 += d0_; r1 += d1_; } // t152 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r2 += d0_; r3 += d1_; } // t153 mm8 a=r0 b=r3 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r0 += d0_; r4 += d1_; } // t154 mm8 a=r7 b=r3 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r7 += d0_; r5 += d1_; } // t155 mm8 a=r1 b=r3 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r2 += d0_; r3 += d1_; } // t156 mm8 a=r6 b=r3 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r4 += d0_; r5 += d1_; } // t157 mm8 a=r6 b=r4 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t158 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r5 += d0_; r2 += d1_; } // t159 mm8 a=r1 b=r0 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r5 += d0_; r7 += d1_; } // t160 mm8 a=r0 b=r1 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r7 += d0_; r0 += d1_; } // t161 mm8 a=r4 b=r0 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r6 += d0_; r1 += d1_; } // t162 mm8 a=r6 b=r6 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r7 += d0_; r3 += d1_; } // t163 mm8 a=r5 b=r3 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t164 mm8 a=r0 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r1 += d0_; r5 += d1_; } // t165 mm8 a=r6 b=r5 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r4 += d0_; r0 += d1_; } // t166 mm8 a=r6 b=r5 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r3 += d0_; r7 += d1_; } // t167 mm8 a=r4 b=r5 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r0 += d0_; r7 += d1_; } // t168 mm8 a=r3 b=r6 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r4 += d0_; r3 += d1_; } // t169 mm8 a=r6 b=r2 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t170 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r2 += d0_; r3 += d1_; } // t171 mm8 a=r6 b=r3 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r5 += d0_; r2 += d1_; } // t172 mm8 a=r7 b=r3 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t173 mm8 a=r0 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r5 += d0_; r6 += d1_; } // t174 mm8 a=r6 b=r6 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r3 += d0_; r4 += d1_; } // t175 mm8 a=r1 b=r6 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r7 += d0_; r5 += d1_; } // t176 mm8 a=r2 b=r4 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r7 += d0_; r2 += d1_; } // t177 mm8 a=r1 b=r7 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r7 += d0_; r3 += d1_; } // t178 mm8 a=r4 b=r2 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r5 += d0_; r1 += d1_; } // t179 mm8 a=r5 b=r2 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r2 += d0_; r1 += d1_; } // t180 mm8 a=r0 b=r4 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r2 += d0_; r4 += d1_; } // t181 mm8 a=r3 b=r5 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r4 += d0_; r3 += d1_; } // t182 mm8 a=r1 b=r1 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r0 += d0_; r7 += d1_; } // t183 mm8 a=r3 b=r1 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r0 += d0_; r7 += d1_; } // t184 mm8 a=r7 b=r4 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r6 += d0_; r3 += d1_; } // t185 mm8 a=r5 b=r0 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r7 += d0_; r6 += d1_; } // t186 mm8 a=r2 b=r5 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r0 += d0_; r7 += d1_; } // t187 mm8 a=r3 b=r0 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r5 += d0_; r0 += d1_; } // t188 mm8 a=r1 b=r4 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r5 += d0_; r4 += d1_; } // t189 mm8 a=r7 b=r2 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r0 += d1_; } // t190 mm8 a=r1 b=r5 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t191 mm8 a=r2 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t192 mm8 a=r5 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r4 += d0_; r6 += d1_; } // t193 mm8 a=r2 b=r1 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r0 += d0_; r5 += d1_; } // t194 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r4 += d1_; } // t195 mm8 a=r2 b=r7 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r2 += d0_; r5 += d1_; } // t196 mm8 a=r5 b=r6 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r6 += d0_; r5 += d1_; } // t197 mm8 a=r5 b=r3 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r7 += d0_; r1 += d1_; } // t198 mm8 a=r1 b=r0 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t199 mm8 a=r2 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t200 mm8 a=r5 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r2 += d0_; r1 += d1_; } // t201 mm8 a=r7 b=r2 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r5 += d0_; r6 += d1_; } // t202 mm8 a=r2 b=r5 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r6 += d1_; } // t203 mm8 a=r0 b=r5 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r3 += d0_; r7 += d1_; } // t204 mm8 a=r4 b=r7 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r1 += d0_; r3 += d1_; } // t205 mm8 a=r3 b=r7 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r7 += d0_; r3 += d1_; } // t206 mm8 a=r0 b=r6 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r1 += d0_; r4 += d1_; } // t207 mm8 a=r7 b=r7 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r6 += d0_; r5 += d1_; } // t208 mm8 a=r0 b=r1 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r6 += d0_; r2 += d1_; } // t209 mm8 a=r4 b=r4 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t210 mm8 a=r7 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r5 += d0_; r6 += d1_; } // t211 mm8 a=r1 b=r5 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r5 += d0_; r3 += d1_; } // t212 mm8 a=r2 b=r4 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r3 += d0_; r5 += d1_; } // t213 mm8 a=r2 b=r2 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r5 += d0_; r7 += d1_; } // t214 mm8 a=r4 b=r6 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r2 += d0_; r6 += d1_; } // t215 mm8 a=r7 b=r3 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r3 += d0_; r4 += d1_; } // t216 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r0 += d0_; r6 += d1_; } // t217 mm8 a=r5 b=r7 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r0 += d1_; } // t218 mm8 a=r7 b=r1 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r3 += d0_; r1 += d1_; } // t219 mm8 a=r2 b=r1 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r4 += d0_; r5 += d1_; } // t220 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r2 += d0_; r1 += d1_; } // t221 mm8 a=r7 b=r1 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r2 += d0_; r1 += d1_; } // t222 mm8 a=r0 b=r6 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r7 += d0_; r0 += d1_; } // t223 mm8 a=r2 b=r2 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r6 += d0_; r2 += d1_; } // t224 mm8 a=r1 b=r1 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r4 += d1_; } // t225 mm8 a=r2 b=r7 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t226 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r5 += d0_; r6 += d1_; } // t227 mm8 a=r3 b=r5 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r3 += d0_; r1 += d1_; } // t228 mm8 a=r7 b=r1 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r1 += d0_; r4 += d1_; } // t229 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r5 += d0_; r6 += d1_; } // t230 mm8 a=r0 b=r3 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r4 += d0_; r0 += d1_; } // t231 mm8 a=r1 b=r3 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r0 += d0_; r1 += d1_; } // t232 mm8 a=r5 b=r4 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r0 += d0_; r4 += d1_; } // t233 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r2 += d0_; r1 += d1_; } // t234 mm8 a=r2 b=r0 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t235 mm8 a=r3 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r0 += d0_; r5 += d1_; } // t236 mm8 a=r1 b=r7 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r6 += d0_; r3 += d1_; } // t237 mm8 a=r3 b=r5 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r0 += d0_; r2 += d1_; } // t238 mm8 a=r2 b=r0 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r1 += d0_; r0 += d1_; } // t239 mm8 a=r5 b=r7 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r4 += d0_; r6 += d1_; } // t240 mm8 a=r2 b=r7 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r3 += d0_; r7 += d1_; } // t241 mm8 a=r0 b=r1 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r2 += d0_; r7 += d1_; } // t242 mm8 a=r0 b=r2 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r1 += d0_; r5 += d1_; } // t243 mm8 a=r4 b=r4 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r0 += d0_; r7 += d1_; } // t244 mm8 a=r7 b=r6 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r2 += d0_; r0 += d1_; } // t245 mm8 a=r4 b=r4 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t246 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r2 += d0_; r4 += d1_; } // t247 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r7 += d0_; r3 += d1_; } // t248 mm8 a=r7 b=r5 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r0 += d0_; r5 += d1_; } // t249 mm8 a=r4 b=r4 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r0 += d0_; r1 += d1_; } // t250 mm8 a=r0 b=r6 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t251 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r5 += d0_; r4 += d1_; } // t252 mm8 a=r0 b=r2 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r7 += d0_; r4 += d1_; } // t253 mm8 a=r5 b=r5 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t254 mm8 a=r6 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r0 += d0_; r3 += d1_; } // t255 mm8 a=r5 b=r0 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r3 += d0_; r4 += d1_; } // t256 mm8 a=r2 b=r1 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r6 += d0_; r0 += d1_; } // t257 mm8 a=r4 b=r3 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r7 += d0_; r3 += d1_; } // t258 mm8 a=r4 b=r1 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r3 += d0_; r0 += d1_; } // t259 mm8 a=r5 b=r5 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r3 += d0_; r0 += d1_; } // t260 mm8 a=r3 b=r2 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r3 += d0_; r2 += d1_; } // t261 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r2 += d0_; r5 += d1_; } // t262 mm8 a=r6 b=r4 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r2 += d0_; r1 += d1_; } // t263 mm8 a=r6 b=r4 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r1 += d0_; r4 += d1_; } // t264 mm8 a=r7 b=r2 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r3 += d0_; r1 += d1_; } // t265 mm8 a=r6 b=r1 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r7 += d0_; r0 += d1_; } // t266 mm8 a=r0 b=r0 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r3 += d0_; r4 += d1_; } // t267 mm8 a=r6 b=r3 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r6 += d0_; r5 += d1_; } // t268 mm8 a=r1 b=r0 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r7 += d0_; r3 += d1_; } // t269 mm8 a=r7 b=r7 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r7 += d0_; r0 += d1_; } // t270 mm8 a=r6 b=r2 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r5 += d0_; r3 += d1_; } // t271 mm8 a=r0 b=r3 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r0 += d0_; r3 += d1_; } // t272 mm8 a=r6 b=r0 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r7 += d1_; } // t273 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r7 += d0_; r6 += d1_; } // t274 mm8 a=r1 b=r0 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r1 += d0_; r4 += d1_; } // t275 mm8 a=r1 b=r3 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r6 += d0_; r2 += d1_; } // t276 mm8 a=r1 b=r5 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r4 += d0_; r7 += d1_; } // t277 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r6 += d0_; r7 += d1_; } // t278 mm8 a=r0 b=r0 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r2 += d0_; r4 += d1_; } // t279 mm8 a=r2 b=r4 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r3 += d0_; r5 += d1_; } // t280 mm8 a=r7 b=r2 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r4 += d0_; r2 += d1_; } // t281 mm8 a=r1 b=r0 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r5 += d0_; r1 += d1_; } // t282 mm8 a=r0 b=r1 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r7 += d0_; r2 += d1_; } // t283 mm8 a=r0 b=r2 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r0 += d0_; r2 += d1_; } // t284 mm8 a=r1 b=r5 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r5 += d0_; r0 += d1_; } // t285 mm8 a=r4 b=r7 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r2 += d0_; r0 += d1_; } // t286 mm8 a=r1 b=r0 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r7 += d0_; r4 += d1_; } // t287 mm8 a=r7 b=r1 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t288 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r5 += d0_; r4 += d1_; } // t289 mm8 a=r0 b=r5 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r6 += d0_; r7 += d1_; } // t290 mm8 a=r3 b=r2 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r7 += d0_; r1 += d1_; } // t291 mm8 a=r7 b=r4 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t292 mm8 a=r6 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r6 += d0_; r4 += d1_; } // t293 mm8 a=r5 b=r5 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r1 += d0_; r6 += d1_; } // t294 mm8 a=r5 b=r7 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r1 += d0_; r3 += d1_; } // t295 mm8 a=r0 b=r7 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r5 += d0_; r6 += d1_; } // t296 mm8 a=r6 b=r1 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r3 += d0_; r5 += d1_; } // t297 mm8 a=r4 b=r5 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r3 += d0_; r1 += d1_; } // t298 mm8 a=r6 b=r0 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r2 += d0_; r3 += d1_; } // t299 mm8 a=r6 b=r0 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r4 += d0_; r2 += d1_; } // t300 mm8 a=r6 b=r4 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r6 += d0_; r3 += d1_; } // t301 mm8 a=r6 b=r4 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r6 += d0_; r4 += d1_; } // t302 mm8 a=r7 b=r6 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r3 += d0_; r6 += d1_; } // t303 mm8 a=r3 b=r2 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r2 += d0_; r4 += d1_; } // t304 mm8 a=r6 b=r5 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r0 += d0_; r2 += d1_; } // t305 mm8 a=r4 b=r5 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r0 += d0_; r2 += d1_; } // t306 mm8 a=r3 b=r1 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r5 += d0_; r0 += d1_; } // t307 mm8 a=r4 b=r3 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r2 += d0_; r6 += d1_; } // t308 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r1 += d0_; r4 += d1_; } // t309 mm8 a=r2 b=r1 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r2 += d0_; r3 += d1_; } // t310 mm8 a=r6 b=r1 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r3 += d0_; r2 += d1_; } // t311 mm8 a=r4 b=r6 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t312 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r7 += d0_; r1 += d1_; } // t313 mm8 a=r2 b=r6 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r2 += d0_; r1 += d1_; } // t314 mm8 a=r0 b=r3 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r6 += d0_; r2 += d1_; } // t315 mm8 a=r7 b=r1 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r1 += d0_; r6 += d1_; } // t316 mm8 a=r2 b=r4 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r3 += d0_; r6 += d1_; } // t317 mm8 a=r1 b=r6 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r1 += d0_; r0 += d1_; } // t318 mm8 a=r2 b=r1 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r6 += d0_; r3 += d1_; } // t319 mm8 a=r0 b=r2 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r6 += d0_; r2 += d1_; } // t320 mm8 a=r5 b=r2 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t321 mm8 a=r7 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r7 += d0_; r1 += d1_; } // t322 mm8 a=r2 b=r4 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r2 += d0_; r3 += d1_; } // t323 mm8 a=r4 b=r3 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r2 += d0_; r0 += d1_; } // t324 mm8 a=r5 b=r7 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r7 += d0_; r2 += d1_; } // t325 mm8 a=r6 b=r2 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r7 += d0_; r6 += d1_; } // t326 mm8 a=r6 b=r1 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r3 += d0_; r0 += d1_; } // t327 mm8 a=r1 b=r5 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r1 += d0_; r6 += d1_; } // t328 mm8 a=r0 b=r1 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r7 += d0_; r1 += d1_; } // t329 mm8 a=r0 b=r3 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r4 += d0_; r0 += d1_; } // t330 mm8 a=r6 b=r7 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t331 mm8 a=r4 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r6 += d0_; r3 += d1_; } // t332 mm8 a=r4 b=r4 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r4 += d0_; r2 += d1_; } // t333 mm8 a=r4 b=r6 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r0 += d0_; r1 += d1_; } // t334 mm8 a=r2 b=r2 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r2 += d1_; } // t335 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r2 += d0_; r3 += d1_; } // t336 mm8 a=r4 b=r0 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r0 += d0_; r3 += d1_; } // t337 mm8 a=r1 b=r4 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r2 += d0_; r7 += d1_; } // t338 mm8 a=r3 b=r7 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r7 += d0_; r6 += d1_; } // t339 mm8 a=r5 b=r1 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r7 += d0_; r5 += d1_; } // t340 mm8 a=r5 b=r0 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r7 += d0_; r5 += d1_; } // t341 mm8 a=r6 b=r6 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r0 += d0_; r4 += d1_; } // t342 mm8 a=r4 b=r7 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r4 += d0_; r6 += d1_; } // t343 mm8 a=r0 b=r4 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r4 += d0_; r6 += d1_; } // t344 mm8 a=r2 b=r5 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t345 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t346 mm8 a=r6 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r3 += d0_; r5 += d1_; } // t347 mm8 a=r5 b=r3 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r4 += d0_; r5 += d1_; } // t348 mm8 a=r0 b=r4 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r4 += d0_; r7 += d1_; } // t349 mm8 a=r6 b=r1 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r3 += d0_; r5 += d1_; } // t350 mm8 a=r1 b=r5 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r4 += d0_; r5 += d1_; } // t351 mm8 a=r6 b=r4 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r6 += d0_; r0 += d1_; } // t352 mm8 a=r4 b=r1 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r1 += d0_; r4 += d1_; } // t353 mm8 a=r7 b=r3 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r1 += d0_; r5 += d1_; } // t354 mm8 a=r4 b=r1 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r0 += d0_; r1 += d1_; } // t355 mm8 a=r4 b=r5 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r0 += d0_; r4 += d1_; } // t356 mm8 a=r7 b=r6 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r7 += d0_; r5 += d1_; } // t357 mm8 a=r7 b=r3 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r5 += d0_; r7 += d1_; } // t358 mm8 a=r4 b=r7 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r3 += d0_; r5 += d1_; } // t359 mm8 a=r2 b=r7 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r0 += d0_; r2 += d1_; } // t360 mm8 a=r4 b=r4 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r5 += d0_; r6 += d1_; } // t361 mm8 a=r2 b=r2 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r3 += d0_; r0 += d1_; } // t362 mm8 a=r2 b=r7 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r5 += d0_; r0 += d1_; } // t363 mm8 a=r7 b=r6 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r0 += d0_; r5 += d1_; } // t364 mm8 a=r6 b=r4 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r5 += d0_; r4 += d1_; } // t365 mm8 a=r6 b=r6 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r2 += d0_; r5 += d1_; } // t366 mm8 a=r7 b=r7 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t367 mm8 a=r4 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r3 += d0_; r5 += d1_; } // t368 mm8 a=r7 b=r0 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r6 += d0_; r0 += d1_; } // t369 mm8 a=r1 b=r0 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r7 += d0_; r4 += d1_; } // t370 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r2 += d0_; r6 += d1_; } // t371 mm8 a=r5 b=r4 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r2 += d1_; } // t372 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r2 += d0_; r4 += d1_; } // t373 mm8 a=r4 b=r6 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t374 mm8 a=r1 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r6 += d0_; r2 += d1_; } // t375 mm8 a=r6 b=r5 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t376 mm8 a=r3 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r6 += d0_; r4 += d1_; } // t377 mm8 a=r6 b=r4 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r2 += d0_; r4 += d1_; } // t378 mm8 a=r6 b=r3 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t379 mm8 a=r3 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r6 += d0_; r2 += d1_; } // t380 mm8 a=r2 b=r0 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r1 += d0_; r2 += d1_; } // t381 mm8 a=r3 b=r5 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r2 += d0_; r4 += d1_; } // t382 mm8 a=r5 b=r2 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r0 += d0_; r2 += d1_; } // t383 mm8 a=r1 b=r2 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r1 += d0_; r0 += d1_; } // t384 mm8 a=r3 b=r1 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r7 += d0_; r5 += d1_; } // t385 mm8 a=r6 b=r0 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r1 += d0_; r7 += d1_; } // t386 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r2 += d0_; r4 += d1_; } // t387 mm8 a=r3 b=r7 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r1 += d0_; r2 += d1_; } // t388 mm8 a=r3 b=r0 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r5 += d0_; r3 += d1_; } // t389 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r1 += d0_; r4 += d1_; } // t390 mm8 a=r5 b=r4 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t391 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r3 += d0_; r0 += d1_; } // t392 mm8 a=r2 b=r3 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r5 += d0_; r0 += d1_; } // t393 mm8 a=r3 b=r6 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r1 += d0_; r2 += d1_; } // t394 mm8 a=r0 b=r7 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r2 += d0_; r4 += d1_; } // t395 mm8 a=r0 b=r7 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r1 += d0_; r4 += d1_; } // t396 mm8 a=r7 b=r7 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t397 mm8 a=r5 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r7 += d0_; r0 += d1_; } // t398 mm8 a=r6 b=r5 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r3 += d0_; r5 += d1_; } // t399 mm8 a=r6 b=r3 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r0 += d0_; r4 += d1_; } // t400 mm8 a=r2 b=r7 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r2 += d0_; r1 += d1_; } // t401 mm8 a=r7 b=r6 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r7 += d0_; r1 += d1_; } // t402 mm8 a=r0 b=r0 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r1 += d0_; r0 += d1_; } // t403 mm8 a=r7 b=r4 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r1 += d0_; r4 += d1_; } // t404 mm8 a=r1 b=r1 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r3 += d0_; r6 += d1_; } // t405 mm8 a=r6 b=r6 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r4 += d0_; r3 += d1_; } // t406 mm8 a=r6 b=r1 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r2 += d0_; r4 += d1_; } // t407 mm8 a=r5 b=r7 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t408 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r4 += d0_; r7 += d1_; } // t409 mm8 a=r3 b=r4 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r5 += d0_; r2 += d1_; } // t410 mm8 a=r0 b=r5 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r5 += d0_; r1 += d1_; } // t411 mm8 a=r2 b=r0 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t412 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r2 += d1_; } // t413 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r0 += d0_; r3 += d1_; } // t414 mm8 a=r7 b=r1 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r5 += d0_; r1 += d1_; } // t415 mm8 a=r4 b=r7 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r3 += d0_; r5 += d1_; } // t416 mm8 a=r5 b=r4 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t417 mm8 a=r6 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r1 += d0_; r5 += d1_; } // t418 mm8 a=r1 b=r3 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r0 += d0_; r6 += d1_; } // t419 mm8 a=r5 b=r0 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r7 += d0_; r6 += d1_; } // t420 mm8 a=r4 b=r2 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r3 += d0_; r7 += d1_; } // t421 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t422 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r0 += d1_; } // t423 mm8 a=r1 b=r7 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r6 += d0_; r5 += d1_; } // t424 mm8 a=r5 b=r1 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t425 mm8 a=r2 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r5 += d0_; r0 += d1_; } // t426 mm8 a=r1 b=r6 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t427 mm8 a=r6 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r7 += d0_; r4 += d1_; } // t428 mm8 a=r6 b=r2 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r7 += d0_; r0 += d1_; } // t429 mm8 a=r3 b=r3 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r3 += d0_; r5 += d1_; } // t430 mm8 a=r7 b=r7 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r1 += d0_; r0 += d1_; } // t431 mm8 a=r6 b=r4 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r3 += d0_; r4 += d1_; } // t432 mm8 a=r3 b=r2 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r5 += d0_; r0 += d1_; } // t433 mm8 a=r3 b=r0 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r5 += d0_; r3 += d1_; } // t434 mm8 a=r0 b=r7 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r6 += d0_; r3 += d1_; } // t435 mm8 a=r7 b=r5 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r0 += d0_; r5 += d1_; } // t436 mm8 a=r6 b=r4 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r3 += d0_; r4 += d1_; } // t437 mm8 a=r0 b=r7 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r4 += d0_; r7 += d1_; } // t438 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t439 mm8 a=r4 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r3 += d0_; r2 += d1_; } // t440 mm8 a=r3 b=r5 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r1 += d0_; r4 += d1_; } // t441 mm8 a=r5 b=r1 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r7 += d0_; r5 += d1_; } // t442 mm8 a=r1 b=r2 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r5 += d0_; r2 += d1_; } // t443 mm8 a=r1 b=r7 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r1 += d0_; r0 += d1_; } // t444 mm8 a=r3 b=r7 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r7 += d0_; r0 += d1_; } // t445 mm8 a=r7 b=r5 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r2 += d0_; r1 += d1_; } // t446 mm8 a=r3 b=r1 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r4 += d0_; r7 += d1_; } // t447 mm8 a=r6 b=r7 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r4 += d0_; r5 += d1_; } // t448 mm8 a=r5 b=r2 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r3 += d1_; } // t449 mm8 a=r7 b=r1 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r7 += d0_; r3 += d1_; } // t450 mm8 a=r7 b=r5 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r1 += d0_; r7 += d1_; } // t451 mm8 a=r0 b=r3 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r7 += d0_; r1 += d1_; } // t452 mm8 a=r2 b=r4 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r7 += d0_; r1 += d1_; } // t453 mm8 a=r5 b=r7 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r7 += d0_; r1 += d1_; } // t454 mm8 a=r7 b=r7 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t455 mm8 a=r7 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r5 += d0_; r6 += d1_; } // t456 mm8 a=r6 b=r5 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r1 += d0_; r5 += d1_; } // t457 mm8 a=r7 b=r7 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r4 += d0_; r1 += d1_; } // t458 mm8 a=r5 b=r1 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r1 += d0_; r0 += d1_; } // t459 mm8 a=r6 b=r7 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t460 mm8 a=r7 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r6 += d0_; r2 += d1_; } // t461 mm8 a=r0 b=r6 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t462 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r3 += d0_; r2 += d1_; } // t463 mm8 a=r2 b=r0 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r0 += d0_; r1 += d1_; } // t464 mm8 a=r0 b=r3 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r0 += d0_; r5 += d1_; } // t465 mm8 a=r1 b=r7 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r4 += d0_; r1 += d1_; } // t466 mm8 a=r1 b=r2 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r1 += d0_; r6 += d1_; } // t467 mm8 a=r1 b=r0 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r6 += d0_; r2 += d1_; } // t468 mm8 a=r4 b=r1 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r0 += d0_; r1 += d1_; } // t469 mm8 a=r2 b=r0 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r2 += d0_; r0 += d1_; } // t470 mm8 a=r3 b=r1 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r7 += d0_; r1 += d1_; } // t471 mm8 a=r2 b=r2 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t472 mm8 a=r1 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r1 += d1_; } // t473 mm8 a=r2 b=r7 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r6 += d0_; r3 += d1_; } // t474 mm8 a=r5 b=r3 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r6 += d0_; r0 += d1_; } // t475 mm8 a=r7 b=r6 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r5 += d0_; r6 += d1_; } // t476 mm8 a=r3 b=r0 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r4 += d0_; r6 += d1_; } // t477 mm8 a=r3 b=r1 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r2 += d0_; r5 += d1_; } // t478 mm8 a=r0 b=r6 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r7 += d0_; r1 += d1_; } // t479 mm8 a=r7 b=r6 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r1 += d0_; r0 += d1_; } // t480 mm8 a=r6 b=r0 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r7 += d1_; } // t481 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r0 += d0_; r6 += d1_; } // t482 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r7 += d0_; r6 += d1_; } // t483 mm8 a=r7 b=r0 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r6 += d0_; r4 += d1_; } // t484 mm8 a=r3 b=r7 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r1 += d0_; r2 += d1_; } // t485 mm8 a=r6 b=r4 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r2 += d0_; r5 += d1_; } // t486 mm8 a=r6 b=r5 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r4 += d0_; r5 += d1_; } // t487 mm8 a=r2 b=r5 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r1 += d0_; r2 += d1_; } // t488 mm8 a=r1 b=r2 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r6 += d0_; r4 += d1_; } // t489 mm8 a=r5 b=r2 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r2 += d0_; r6 += d1_; } // t490 mm8 a=r7 b=r1 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r0 += d0_; r6 += d1_; } // t491 mm8 a=r0 b=r0 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r6 += d0_; r4 += d1_; } // t492 mm8 a=r3 b=r1 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r7 += d0_; r6 += d1_; } // t493 mm8 a=r6 b=r1 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r3 += d0_; r1 += d1_; } // t494 mm8 a=r5 b=r6 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r7 += d0_; r2 += d1_; } // t495 mm8 a=r2 b=r2 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r5 += d0_; r0 += d1_; } // t496 mm8 a=r3 b=r5 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r2 += d0_; r0 += d1_; } // t497 mm8 a=r3 b=r2 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r6 += d0_; r2 += d1_; } // t498 mm8 a=r3 b=r1 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r1 += d0_; r5 += d1_; } // t499 mm8 a=r1 b=r6 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r1 += d0_; r5 += d1_; } // t500 mm8 a=r7 b=r6 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r1 += d0_; r0 += d1_; } // t501 mm8 a=r4 b=r4 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r7 += d0_; r6 += d1_; } // t502 mm8 a=r5 b=r7 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r3 += d0_; r4 += d1_; } // t503 mm8 a=r7 b=r5 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r4 += d0_; r2 += d1_; } // t504 mm8 a=r2 b=r0 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r0 += d0_; r1 += d1_; } // t505 mm8 a=r4 b=r0 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r7 += d0_; r0 += d1_; } // t506 mm8 a=r0 b=r2 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r4 += d0_; r7 += d1_; } // t507 mm8 a=r5 b=r6 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r4 += d0_; r7 += d1_; } // t508 mm8 a=r7 b=r3 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r6 += d0_; r0 += d1_; } // t509 mm8 a=r4 b=r4 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t510 mm8 a=r0 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t511 mm8 a=r2 b=r4 c=r1 c2=r3 - } - uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((ulong)hi << 32) | (ulong)lo; -} - -#if IGNEUM_EXCHANGE != 0 -// Reports the sub-group size this device uses for a work-group of IGNEUM_GROUP items. host.c runs it only when the -// per-kernel query (clGetKernelSubGroupInfoKHR on igneum_hash) is unavailable; that query is preferred because a -// compiler may pick a different wave width per kernel (RDNA: wave32 or wave64). See WAVEFRONT.md. -IGNEUM_KERNEL_HASH void igneum_probe_subgroup(__global uint* out) { - if (get_local_id(0) == 0u) { out[0] = get_sub_group_size(); out[1] = get_num_sub_groups(); } -} -#endif - -// Header-bound variant (bind.rs): the init words come from initw, not SEEDW. Same body as igneum_hash. -IGNEUM_KERNEL_HASH void igneum_hash_bound(__global const uint* ds, __global ulong* out, uint baseNonce, uint mask, __global const uint* initw) { - uint gid = (uint)get_global_id(0); - uint lid = (uint)get_local_id(0); - uint nonce = baseNonce + gid; - uint r0, r1, r2, r3, r4, r5, r6, r7; - uint iw0 = initw[0], iw1 = initw[1], iw2 = initw[2], iw3 = initw[3], iw4 = initw[4], iw5 = initw[5], iw6 = initw[6], iw7 = initw[7]; -#if IGNEUM_EXCHANGE == 0 - IGNEUM_LOCAL_WORDS(xch, 2 * IGNEUM_GROUP); - uint xk = 0u; -#else - (void)lid; -#endif - uint lane = lid & 31u; - { uint x = nonce ^ iw0; x += 0x9e3779b9u * 1u; x = splitmix32(x); r0 = x ^ iw1; } - { uint x = nonce ^ iw1; x += 0x9e3779b9u * 2u; x = splitmix32(x); r1 = x ^ iw2; } - { uint x = nonce ^ iw2; x += 0x9e3779b9u * 3u; x = splitmix32(x); r2 = x ^ iw3; } - { uint x = nonce ^ iw3; x += 0x9e3779b9u * 4u; x = splitmix32(x); r3 = x ^ iw4; } - { uint x = nonce ^ iw4; x += 0x9e3779b9u * 5u; x = splitmix32(x); r4 = x ^ iw5; } - { uint x = nonce ^ iw5; x += 0x9e3779b9u * 6u; x = splitmix32(x); r5 = x ^ iw6; } - { uint x = nonce ^ iw6; x += 0x9e3779b9u * 7u; x = splitmix32(x); r6 = x ^ iw7; } - { uint x = nonce ^ iw7; x += 0x9e3779b9u * 8u; x = splitmix32(x); r7 = x ^ iw0; } - - for (uint it = 0u; it < 8u; ++it) { - uint sel = r0; - r2 = r3 * r4 + r2; // 0 mad - r2 = r1 * r1 + r2; // 1 mad - r2 = r3 * r2 + r2; // 2 mad - r3 = r3 ^ r5; // 3 xor - r7 = r7 ^ ds[r2 & mask]; // 4 load - r5 = r5 ^ ds[r7 & mask]; // 5 load - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 8u); r1 = r1 ^ t_; } // 6 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 8u); r7 = r7 ^ t_; } // 7 shfl - r1 = mul_hi(r1, r5); // 8 mulhi - r6 = rotr_var(r6, r3); // 9 rotr - r3 = r3 | r4; // 10 or - r4 = r4 ^ ds[r3 & mask]; // 11 load - r0 = mul_hi(r0, r4); // 12 mulhi - r5 = r5 + r1 + ((((sel >> 30u) & 1u) != 0u) ? 0xd3177981u : 0xc7934706u); // 13 add - r0 = r0 ^ ds[r4 & mask]; // 14 load - r2 = r2 - r4; // 15 sub - r2 = r2 ^ ds[r0 & mask]; // 16 load - r7 = r7 ^ ds[r2 & mask]; // 17 load - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 4u); r7 = r7 ^ t_; } // 18 shfl - r5 = r5 * r0; // 19 mul - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 2u); r3 = r3 ^ t_; } // 20 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 16u); r2 = r2 ^ t_; } // 21 shfl - r6 = mul_hi(r6, r2); // 22 mulhi - r6 = r6 ^ ds[r1 & mask]; // 23 load - r5 = r5 * r0; // 24 mul - r5 = rotl_imm(r5, 19u); // 25 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 2u); r7 = r7 ^ t_; } // 26 shfl - r0 = r0 ^ r5; // 27 xor - r0 = r0 ^ r4; // 28 xor - r3 = r3 - r0; // 29 sub - r5 = r5 * r1; // 30 mul - r7 = r7 ^ ds[r2 & mask]; // 31 load - r1 = r1 ^ ds[r0 & mask]; // 32 load - r5 = r5 ^ r6; // 33 xor - r5 = r5 ^ ds[r1 & mask]; // 34 load - r0 = mul_hi(r0, r5); // 35 mulhi - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 4u); r5 = r5 ^ t_; } // 36 shfl - r7 = r7 ^ ds[r0 & mask]; // 37 load - r3 = r3 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0x230c005cu : 0x75ba2fadu); // 38 add - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 4u); r1 = r1 ^ t_; } // 39 shfl - r2 = r2 ^ r5; // 40 xor - r3 = r6 * r3 + r3; // 41 mad - r6 = r6 - r7; // 42 sub - r7 = r7 ^ r0; // 43 xor - r1 = r1 ^ ds[r7 & mask]; // 44 load - r2 = r2 * r3; // 45 mul - r1 = mul_hi(r1, r5); // 46 mulhi - r4 = r4 - r3; // 47 sub - r2 = rotr_var(r2, r6); // 48 rotr - r3 = r3 ^ ds[r5 & mask]; // 49 load - r1 = r1 + r5 + ((((sel >> 7u) & 1u) != 0u) ? 0x1907970cu : 0x81b8bc2cu); // 50 add - r0 = r0 * r2; // 51 mul - r0 = r0 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x699fd448u : 0x4f92b968u); // 52 add - r1 = r1 + r0 + ((((sel >> 12u) & 1u) != 0u) ? 0x77b1520du : 0x2bb965afu); // 53 add - r7 = rotl_imm(r7, 14u); // 54 rotl - r3 = r3 + r7 + ((((sel >> 1u) & 1u) != 0u) ? 0xa54c55a0u : 0x7b0fe07au); // 55 add - r6 = r6 ^ ds[r7 & mask]; // 56 load - r1 = rotr_var(r1, r5); // 57 rotr - r5 = r5 ^ ds[r4 & mask]; // 58 load - r6 = r6 ^ ds[r2 & mask]; // 59 load - r3 = r5 * r0 + r3; // 60 mad - r5 = r5 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0xad7493e7u : 0xaf9dd72du); // 61 add - r4 = r4 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x1e07c3d9u : 0x89841d87u); // 62 add - r5 = rotl_imm(r5, 19u); // 63 rotl - // int8 tile block (Counter ASIC 4.0 research, experimental): 512 mma.m8n8k16 u8 tiles per iteration, both outputs consumed - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r2 += d0_; r3 += d1_; } // t0 mm8 a=r6 b=r5 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r4 += d0_; r5 += d1_; } // t1 mm8 a=r1 b=r1 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r0 += d0_; r7 += d1_; } // t2 mm8 a=r2 b=r5 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r3 += d1_; } // t3 mm8 a=r1 b=r5 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r2 += d0_; r6 += d1_; } // t4 mm8 a=r2 b=r0 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r7 += d0_; r3 += d1_; } // t5 mm8 a=r2 b=r6 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t6 mm8 a=r1 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r0 += d0_; r4 += d1_; } // t7 mm8 a=r4 b=r6 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r1 += d0_; r5 += d1_; } // t8 mm8 a=r1 b=r1 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r4 += d0_; r6 += d1_; } // t9 mm8 a=r3 b=r7 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r0 += d0_; r4 += d1_; } // t10 mm8 a=r3 b=r0 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r0 += d0_; r6 += d1_; } // t11 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r6 += d0_; r0 += d1_; } // t12 mm8 a=r3 b=r1 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r5 += d1_; } // t13 mm8 a=r1 b=r5 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r3 += d0_; r4 += d1_; } // t14 mm8 a=r3 b=r3 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r5 += d0_; r4 += d1_; } // t15 mm8 a=r1 b=r0 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r0 += d0_; r4 += d1_; } // t16 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r7 += d1_; } // t17 mm8 a=r0 b=r5 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r2 += d0_; r0 += d1_; } // t18 mm8 a=r1 b=r1 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r2 += d0_; r4 += d1_; } // t19 mm8 a=r3 b=r3 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r6 += d0_; r1 += d1_; } // t20 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r6 += d0_; r0 += d1_; } // t21 mm8 a=r0 b=r3 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r4 += d1_; } // t22 mm8 a=r1 b=r7 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r0 += d0_; r3 += d1_; } // t23 mm8 a=r1 b=r2 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r0 += d0_; r2 += d1_; } // t24 mm8 a=r3 b=r4 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r3 += d0_; r7 += d1_; } // t25 mm8 a=r5 b=r0 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t26 mm8 a=r1 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r0 += d0_; r5 += d1_; } // t27 mm8 a=r3 b=r2 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r5 += d1_; } // t28 mm8 a=r2 b=r4 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r1 += d0_; r5 += d1_; } // t29 mm8 a=r3 b=r4 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r6 += d0_; r3 += d1_; } // t30 mm8 a=r6 b=r5 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t31 mm8 a=r1 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r3 += d0_; r0 += d1_; } // t32 mm8 a=r7 b=r6 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r4 += d0_; r6 += d1_; } // t33 mm8 a=r7 b=r1 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r5 += d0_; r1 += d1_; } // t34 mm8 a=r1 b=r0 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t35 mm8 a=r2 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r7 += d0_; r1 += d1_; } // t36 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r0 += d0_; r2 += d1_; } // t37 mm8 a=r1 b=r7 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r2 += d0_; r4 += d1_; } // t38 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r2 += d0_; r5 += d1_; } // t39 mm8 a=r6 b=r2 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r4 += d0_; r1 += d1_; } // t40 mm8 a=r6 b=r1 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r6 += d0_; r2 += d1_; } // t41 mm8 a=r6 b=r0 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r5 += d0_; r1 += d1_; } // t42 mm8 a=r7 b=r4 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r6 += d0_; r0 += d1_; } // t43 mm8 a=r6 b=r1 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r4 += d0_; r1 += d1_; } // t44 mm8 a=r0 b=r6 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r6 += d0_; r5 += d1_; } // t45 mm8 a=r6 b=r5 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r1 += d0_; r0 += d1_; } // t46 mm8 a=r6 b=r6 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r3 += d0_; r1 += d1_; } // t47 mm8 a=r7 b=r3 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t48 mm8 a=r7 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r1 += d0_; r7 += d1_; } // t49 mm8 a=r4 b=r5 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r4 += d0_; r5 += d1_; } // t50 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r0 += d0_; r3 += d1_; } // t51 mm8 a=r3 b=r4 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r7 += d0_; r4 += d1_; } // t52 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r1 += d0_; r3 += d1_; } // t53 mm8 a=r7 b=r7 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t54 mm8 a=r7 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r3 += d0_; r7 += d1_; } // t55 mm8 a=r1 b=r7 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t56 mm8 a=r4 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r6 += d0_; r7 += d1_; } // t57 mm8 a=r4 b=r3 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r6 += d0_; r0 += d1_; } // t58 mm8 a=r2 b=r3 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r7 += d1_; } // t59 mm8 a=r6 b=r7 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t60 mm8 a=r6 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r2 += d0_; r6 += d1_; } // t61 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r6 += d1_; } // t62 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t63 mm8 a=r1 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r5 += d0_; r3 += d1_; } // t64 mm8 a=r0 b=r5 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r2 += d0_; r4 += d1_; } // t65 mm8 a=r0 b=r1 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r1 += d0_; r3 += d1_; } // t66 mm8 a=r3 b=r5 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r3 += d0_; r4 += d1_; } // t67 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r2 += d1_; } // t68 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r4 += d1_; } // t69 mm8 a=r6 b=r7 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r1 += d0_; r5 += d1_; } // t70 mm8 a=r1 b=r7 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r1 += d0_; r0 += d1_; } // t71 mm8 a=r3 b=r0 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r3 += d0_; r6 += d1_; } // t72 mm8 a=r4 b=r0 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r0 += d0_; r2 += d1_; } // t73 mm8 a=r4 b=r1 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r3 += d0_; r4 += d1_; } // t74 mm8 a=r5 b=r1 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r7 += d0_; r3 += d1_; } // t75 mm8 a=r4 b=r4 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r1 += d0_; r7 += d1_; } // t76 mm8 a=r5 b=r6 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r3 += d0_; r2 += d1_; } // t77 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r0 += d0_; r5 += d1_; } // t78 mm8 a=r2 b=r5 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t79 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r3 += d0_; r4 += d1_; } // t80 mm8 a=r6 b=r7 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r1 += d0_; r6 += d1_; } // t81 mm8 a=r4 b=r3 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r1 += d0_; r7 += d1_; } // t82 mm8 a=r6 b=r0 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r1 += d0_; r6 += d1_; } // t83 mm8 a=r2 b=r2 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t84 mm8 a=r0 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t85 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r2 += d0_; r3 += d1_; } // t86 mm8 a=r3 b=r4 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r3 += d0_; r7 += d1_; } // t87 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r5 += d0_; r3 += d1_; } // t88 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r2 += d0_; r1 += d1_; } // t89 mm8 a=r4 b=r2 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r3 += d0_; r6 += d1_; } // t90 mm8 a=r6 b=r5 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r5 += d0_; r4 += d1_; } // t91 mm8 a=r6 b=r5 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r2 += d0_; r3 += d1_; } // t92 mm8 a=r2 b=r6 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r5 += d0_; r7 += d1_; } // t93 mm8 a=r2 b=r2 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r7 += d0_; r3 += d1_; } // t94 mm8 a=r2 b=r4 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r1 += d0_; r6 += d1_; } // t95 mm8 a=r5 b=r4 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r3 += d0_; r1 += d1_; } // t96 mm8 a=r0 b=r6 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r2 += d1_; } // t97 mm8 a=r0 b=r5 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t98 mm8 a=r2 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r0 += d0_; r2 += d1_; } // t99 mm8 a=r1 b=r4 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r2 += d0_; r5 += d1_; } // t100 mm8 a=r2 b=r4 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r7 += d1_; } // t101 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t102 mm8 a=r5 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t103 mm8 a=r4 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r3 += d0_; r7 += d1_; } // t104 mm8 a=r2 b=r0 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r0 += d0_; r5 += d1_; } // t105 mm8 a=r6 b=r1 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r0 += d0_; r5 += d1_; } // t106 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r4 += d0_; r2 += d1_; } // t107 mm8 a=r2 b=r1 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t108 mm8 a=r4 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r5 += d0_; r3 += d1_; } // t109 mm8 a=r2 b=r7 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r0 += d0_; r2 += d1_; } // t110 mm8 a=r1 b=r0 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r4 += d0_; r6 += d1_; } // t111 mm8 a=r6 b=r5 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r4 += d0_; r2 += d1_; } // t112 mm8 a=r5 b=r7 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t113 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t114 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r1 += d0_; r4 += d1_; } // t115 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r5 += d0_; r0 += d1_; } // t116 mm8 a=r7 b=r2 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r7 += d0_; r6 += d1_; } // t117 mm8 a=r7 b=r6 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r2 += d0_; r1 += d1_; } // t118 mm8 a=r0 b=r1 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r5 += d0_; r3 += d1_; } // t119 mm8 a=r4 b=r3 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r7 += d0_; r2 += d1_; } // t120 mm8 a=r5 b=r3 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r1 += d1_; } // t121 mm8 a=r2 b=r7 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r3 += d0_; r0 += d1_; } // t122 mm8 a=r0 b=r1 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r3 += d0_; r5 += d1_; } // t123 mm8 a=r5 b=r2 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t124 mm8 a=r5 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r7 += d0_; r2 += d1_; } // t125 mm8 a=r7 b=r2 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r3 += d0_; r2 += d1_; } // t126 mm8 a=r6 b=r3 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r5 += d0_; r1 += d1_; } // t127 mm8 a=r6 b=r7 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r4 += d0_; r1 += d1_; } // t128 mm8 a=r6 b=r7 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r6 += d0_; r3 += d1_; } // t129 mm8 a=r3 b=r2 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r2 += d0_; r1 += d1_; } // t130 mm8 a=r0 b=r4 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r3 += d0_; r1 += d1_; } // t131 mm8 a=r3 b=r5 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r0 += d0_; r6 += d1_; } // t132 mm8 a=r0 b=r7 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r0 += d0_; r1 += d1_; } // t133 mm8 a=r2 b=r6 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r3 += d0_; r1 += d1_; } // t134 mm8 a=r5 b=r6 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r4 += d0_; r5 += d1_; } // t135 mm8 a=r0 b=r2 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r1 += d0_; r5 += d1_; } // t136 mm8 a=r3 b=r1 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r4 += d0_; r6 += d1_; } // t137 mm8 a=r7 b=r0 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r4 += d0_; r2 += d1_; } // t138 mm8 a=r2 b=r2 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r2 += d0_; r7 += d1_; } // t139 mm8 a=r0 b=r4 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r1 += d0_; r6 += d1_; } // t140 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r1 += d0_; r2 += d1_; } // t141 mm8 a=r5 b=r5 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r6 += d0_; r2 += d1_; } // t142 mm8 a=r2 b=r2 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r7 += d0_; r1 += d1_; } // t143 mm8 a=r4 b=r3 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r1 += d0_; r7 += d1_; } // t144 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r6 += d0_; r5 += d1_; } // t145 mm8 a=r7 b=r2 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r7 += d1_; } // t146 mm8 a=r2 b=r7 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r3 += d0_; r0 += d1_; } // t147 mm8 a=r3 b=r1 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r0 += d0_; r6 += d1_; } // t148 mm8 a=r2 b=r1 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r7 += d0_; r1 += d1_; } // t149 mm8 a=r4 b=r6 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r7 += d0_; r0 += d1_; } // t150 mm8 a=r3 b=r6 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r2 += d0_; r4 += d1_; } // t151 mm8 a=r1 b=r4 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r5 += d0_; r1 += d1_; } // t152 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r2 += d0_; r3 += d1_; } // t153 mm8 a=r0 b=r3 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r0 += d0_; r4 += d1_; } // t154 mm8 a=r7 b=r3 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r7 += d0_; r5 += d1_; } // t155 mm8 a=r1 b=r3 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r2 += d0_; r3 += d1_; } // t156 mm8 a=r6 b=r3 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r4 += d0_; r5 += d1_; } // t157 mm8 a=r6 b=r4 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t158 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r5 += d0_; r2 += d1_; } // t159 mm8 a=r1 b=r0 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r5 += d0_; r7 += d1_; } // t160 mm8 a=r0 b=r1 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r7 += d0_; r0 += d1_; } // t161 mm8 a=r4 b=r0 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r6 += d0_; r1 += d1_; } // t162 mm8 a=r6 b=r6 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r7 += d0_; r3 += d1_; } // t163 mm8 a=r5 b=r3 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t164 mm8 a=r0 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r1 += d0_; r5 += d1_; } // t165 mm8 a=r6 b=r5 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r4 += d0_; r0 += d1_; } // t166 mm8 a=r6 b=r5 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r3 += d0_; r7 += d1_; } // t167 mm8 a=r4 b=r5 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r0 += d0_; r7 += d1_; } // t168 mm8 a=r3 b=r6 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r4 += d0_; r3 += d1_; } // t169 mm8 a=r6 b=r2 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t170 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r2 += d0_; r3 += d1_; } // t171 mm8 a=r6 b=r3 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r5 += d0_; r2 += d1_; } // t172 mm8 a=r7 b=r3 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t173 mm8 a=r0 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r5 += d0_; r6 += d1_; } // t174 mm8 a=r6 b=r6 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r3 += d0_; r4 += d1_; } // t175 mm8 a=r1 b=r6 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r7 += d0_; r5 += d1_; } // t176 mm8 a=r2 b=r4 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r7 += d0_; r2 += d1_; } // t177 mm8 a=r1 b=r7 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r7 += d0_; r3 += d1_; } // t178 mm8 a=r4 b=r2 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r5 += d0_; r1 += d1_; } // t179 mm8 a=r5 b=r2 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r2 += d0_; r1 += d1_; } // t180 mm8 a=r0 b=r4 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r2 += d0_; r4 += d1_; } // t181 mm8 a=r3 b=r5 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r4 += d0_; r3 += d1_; } // t182 mm8 a=r1 b=r1 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r0 += d0_; r7 += d1_; } // t183 mm8 a=r3 b=r1 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r0 += d0_; r7 += d1_; } // t184 mm8 a=r7 b=r4 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r6 += d0_; r3 += d1_; } // t185 mm8 a=r5 b=r0 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r7 += d0_; r6 += d1_; } // t186 mm8 a=r2 b=r5 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r0 += d0_; r7 += d1_; } // t187 mm8 a=r3 b=r0 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r5 += d0_; r0 += d1_; } // t188 mm8 a=r1 b=r4 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r5 += d0_; r4 += d1_; } // t189 mm8 a=r7 b=r2 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r2 += d0_; r0 += d1_; } // t190 mm8 a=r1 b=r5 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t191 mm8 a=r2 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t192 mm8 a=r5 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r4 += d0_; r6 += d1_; } // t193 mm8 a=r2 b=r1 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r0 += d0_; r5 += d1_; } // t194 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r2 += d0_; r4 += d1_; } // t195 mm8 a=r2 b=r7 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r2 += d0_; r5 += d1_; } // t196 mm8 a=r5 b=r6 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r6 += d0_; r5 += d1_; } // t197 mm8 a=r5 b=r3 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r7 += d0_; r1 += d1_; } // t198 mm8 a=r1 b=r0 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t199 mm8 a=r2 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t200 mm8 a=r5 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r2 += d0_; r1 += d1_; } // t201 mm8 a=r7 b=r2 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r5 += d0_; r6 += d1_; } // t202 mm8 a=r2 b=r5 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r1 += d0_; r6 += d1_; } // t203 mm8 a=r0 b=r5 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r3 += d0_; r7 += d1_; } // t204 mm8 a=r4 b=r7 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r1 += d0_; r3 += d1_; } // t205 mm8 a=r3 b=r7 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r7 += d0_; r3 += d1_; } // t206 mm8 a=r0 b=r6 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r1 += d0_; r4 += d1_; } // t207 mm8 a=r7 b=r7 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r6 += d0_; r5 += d1_; } // t208 mm8 a=r0 b=r1 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r6 += d0_; r2 += d1_; } // t209 mm8 a=r4 b=r4 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r2 += d0_; r5 += d1_; } // t210 mm8 a=r7 b=r1 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r5 += d0_; r6 += d1_; } // t211 mm8 a=r1 b=r5 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r5 += d0_; r3 += d1_; } // t212 mm8 a=r2 b=r4 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r3 += d0_; r5 += d1_; } // t213 mm8 a=r2 b=r2 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r5 += d0_; r7 += d1_; } // t214 mm8 a=r4 b=r6 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r2 += d0_; r6 += d1_; } // t215 mm8 a=r7 b=r3 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r3 += d0_; r4 += d1_; } // t216 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r0 += d0_; r6 += d1_; } // t217 mm8 a=r5 b=r7 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r0 += d1_; } // t218 mm8 a=r7 b=r1 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r3 += d0_; r1 += d1_; } // t219 mm8 a=r2 b=r1 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r4 += d0_; r5 += d1_; } // t220 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r2 += d0_; r1 += d1_; } // t221 mm8 a=r7 b=r1 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r2 += d0_; r1 += d1_; } // t222 mm8 a=r0 b=r6 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r7 += d0_; r0 += d1_; } // t223 mm8 a=r2 b=r2 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r6 += d0_; r2 += d1_; } // t224 mm8 a=r1 b=r1 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r4 += d1_; } // t225 mm8 a=r2 b=r7 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t226 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r5 += d0_; r6 += d1_; } // t227 mm8 a=r3 b=r5 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r3 += d0_; r1 += d1_; } // t228 mm8 a=r7 b=r1 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r1 += d0_; r4 += d1_; } // t229 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r5 += d0_; r6 += d1_; } // t230 mm8 a=r0 b=r3 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r4 += d0_; r0 += d1_; } // t231 mm8 a=r1 b=r3 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r0 += d0_; r1 += d1_; } // t232 mm8 a=r5 b=r4 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r0 += d0_; r4 += d1_; } // t233 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r2 += d0_; r1 += d1_; } // t234 mm8 a=r2 b=r0 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t235 mm8 a=r3 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r0 += d0_; r5 += d1_; } // t236 mm8 a=r1 b=r7 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r6 += d0_; r3 += d1_; } // t237 mm8 a=r3 b=r5 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r0 += d0_; r2 += d1_; } // t238 mm8 a=r2 b=r0 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r1 += d0_; r0 += d1_; } // t239 mm8 a=r5 b=r7 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r4 += d0_; r6 += d1_; } // t240 mm8 a=r2 b=r7 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r3 += d0_; r7 += d1_; } // t241 mm8 a=r0 b=r1 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r2 += d0_; r7 += d1_; } // t242 mm8 a=r0 b=r2 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r1 += d0_; r5 += d1_; } // t243 mm8 a=r4 b=r4 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r0 += d0_; r7 += d1_; } // t244 mm8 a=r7 b=r6 c=r0 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r2 += d0_; r0 += d1_; } // t245 mm8 a=r4 b=r4 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t246 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r2 += d0_; r4 += d1_; } // t247 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r7 += d0_; r3 += d1_; } // t248 mm8 a=r7 b=r5 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r0 += d0_; r5 += d1_; } // t249 mm8 a=r4 b=r4 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r0 += d0_; r1 += d1_; } // t250 mm8 a=r0 b=r6 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t251 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r5 += d0_; r4 += d1_; } // t252 mm8 a=r0 b=r2 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r7 += d0_; r4 += d1_; } // t253 mm8 a=r5 b=r5 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t254 mm8 a=r6 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r0 += d0_; r3 += d1_; } // t255 mm8 a=r5 b=r0 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r3 += d0_; r4 += d1_; } // t256 mm8 a=r2 b=r1 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r6 += d0_; r0 += d1_; } // t257 mm8 a=r4 b=r3 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r7 += d0_; r3 += d1_; } // t258 mm8 a=r4 b=r1 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r3 += d0_; r0 += d1_; } // t259 mm8 a=r5 b=r5 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r3 += d0_; r0 += d1_; } // t260 mm8 a=r3 b=r2 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r3 += d0_; r2 += d1_; } // t261 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r2 += d0_; r5 += d1_; } // t262 mm8 a=r6 b=r4 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r2 += d0_; r1 += d1_; } // t263 mm8 a=r6 b=r4 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r1 += d0_; r4 += d1_; } // t264 mm8 a=r7 b=r2 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r3 += d0_; r1 += d1_; } // t265 mm8 a=r6 b=r1 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r7 += d0_; r0 += d1_; } // t266 mm8 a=r0 b=r0 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r3 += d0_; r4 += d1_; } // t267 mm8 a=r6 b=r3 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r6 += d0_; r5 += d1_; } // t268 mm8 a=r1 b=r0 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r7 += d0_; r3 += d1_; } // t269 mm8 a=r7 b=r7 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r7 += d0_; r0 += d1_; } // t270 mm8 a=r6 b=r2 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r5 += d0_; r3 += d1_; } // t271 mm8 a=r0 b=r3 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r0 += d0_; r3 += d1_; } // t272 mm8 a=r6 b=r0 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r7 += d1_; } // t273 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r7 += d0_; r6 += d1_; } // t274 mm8 a=r1 b=r0 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r1 += d0_; r4 += d1_; } // t275 mm8 a=r1 b=r3 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r6 += d0_; r2 += d1_; } // t276 mm8 a=r1 b=r5 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r4 += d0_; r7 += d1_; } // t277 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r6 += d0_; r7 += d1_; } // t278 mm8 a=r0 b=r0 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r2 += d0_; r4 += d1_; } // t279 mm8 a=r2 b=r4 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r2, d0_, d1_); r3 += d0_; r5 += d1_; } // t280 mm8 a=r7 b=r2 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r4 += d0_; r2 += d1_; } // t281 mm8 a=r1 b=r0 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r5 += d0_; r1 += d1_; } // t282 mm8 a=r0 b=r1 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r7 += d0_; r2 += d1_; } // t283 mm8 a=r0 b=r2 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r0 += d0_; r2 += d1_; } // t284 mm8 a=r1 b=r5 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r5 += d0_; r0 += d1_; } // t285 mm8 a=r4 b=r7 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r2 += d0_; r0 += d1_; } // t286 mm8 a=r1 b=r0 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r7 += d0_; r4 += d1_; } // t287 mm8 a=r7 b=r1 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t288 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r5 += d0_; r4 += d1_; } // t289 mm8 a=r0 b=r5 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r6 += d0_; r7 += d1_; } // t290 mm8 a=r3 b=r2 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r7 += d0_; r1 += d1_; } // t291 mm8 a=r7 b=r4 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r6 += d0_; r3 += d1_; } // t292 mm8 a=r6 b=r7 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r5, d0_, d1_); r6 += d0_; r4 += d1_; } // t293 mm8 a=r5 b=r5 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r1 += d0_; r6 += d1_; } // t294 mm8 a=r5 b=r7 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r1 += d0_; r3 += d1_; } // t295 mm8 a=r0 b=r7 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r5 += d0_; r6 += d1_; } // t296 mm8 a=r6 b=r1 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r3 += d0_; r5 += d1_; } // t297 mm8 a=r4 b=r5 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r3 += d0_; r1 += d1_; } // t298 mm8 a=r6 b=r0 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r2 += d0_; r3 += d1_; } // t299 mm8 a=r6 b=r0 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r4 += d0_; r2 += d1_; } // t300 mm8 a=r6 b=r4 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r6 += d0_; r3 += d1_; } // t301 mm8 a=r6 b=r4 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r6 += d0_; r4 += d1_; } // t302 mm8 a=r7 b=r6 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r3 += d0_; r6 += d1_; } // t303 mm8 a=r3 b=r2 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r2 += d0_; r4 += d1_; } // t304 mm8 a=r6 b=r5 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r0 += d0_; r2 += d1_; } // t305 mm8 a=r4 b=r5 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r0 += d0_; r2 += d1_; } // t306 mm8 a=r3 b=r1 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r5 += d0_; r0 += d1_; } // t307 mm8 a=r4 b=r3 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r2 += d0_; r6 += d1_; } // t308 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r1 += d0_; r4 += d1_; } // t309 mm8 a=r2 b=r1 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r2 += d0_; r3 += d1_; } // t310 mm8 a=r6 b=r1 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r3 += d0_; r2 += d1_; } // t311 mm8 a=r4 b=r6 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t312 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r6, d0_, d1_); r7 += d0_; r1 += d1_; } // t313 mm8 a=r2 b=r6 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r2 += d0_; r1 += d1_; } // t314 mm8 a=r0 b=r3 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r6 += d0_; r2 += d1_; } // t315 mm8 a=r7 b=r1 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r1 += d0_; r6 += d1_; } // t316 mm8 a=r2 b=r4 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r3 += d0_; r6 += d1_; } // t317 mm8 a=r1 b=r6 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r1, d0_, d1_); r1 += d0_; r0 += d1_; } // t318 mm8 a=r2 b=r1 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r6 += d0_; r3 += d1_; } // t319 mm8 a=r0 b=r2 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r6 += d0_; r2 += d1_; } // t320 mm8 a=r5 b=r2 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t321 mm8 a=r7 b=r4 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r7 += d0_; r1 += d1_; } // t322 mm8 a=r2 b=r4 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r3, d0_, d1_); r2 += d0_; r3 += d1_; } // t323 mm8 a=r4 b=r3 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r2 += d0_; r0 += d1_; } // t324 mm8 a=r5 b=r7 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r7 += d0_; r2 += d1_; } // t325 mm8 a=r6 b=r2 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r7 += d0_; r6 += d1_; } // t326 mm8 a=r6 b=r1 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r3 += d0_; r0 += d1_; } // t327 mm8 a=r1 b=r5 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r1, d0_, d1_); r1 += d0_; r6 += d1_; } // t328 mm8 a=r0 b=r1 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r7 += d0_; r1 += d1_; } // t329 mm8 a=r0 b=r3 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r4 += d0_; r0 += d1_; } // t330 mm8 a=r6 b=r7 c=r4 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t331 mm8 a=r4 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r6 += d0_; r3 += d1_; } // t332 mm8 a=r4 b=r4 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r4 += d0_; r2 += d1_; } // t333 mm8 a=r4 b=r6 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r0 += d0_; r1 += d1_; } // t334 mm8 a=r2 b=r2 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r2 += d1_; } // t335 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r2 += d0_; r3 += d1_; } // t336 mm8 a=r4 b=r0 c=r2 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r4, d0_, d1_); r0 += d0_; r3 += d1_; } // t337 mm8 a=r1 b=r4 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r2 += d0_; r7 += d1_; } // t338 mm8 a=r3 b=r7 c=r2 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r7 += d0_; r6 += d1_; } // t339 mm8 a=r5 b=r1 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r7 += d0_; r5 += d1_; } // t340 mm8 a=r5 b=r0 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r7 += d0_; r5 += d1_; } // t341 mm8 a=r6 b=r6 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r0 += d0_; r4 += d1_; } // t342 mm8 a=r4 b=r7 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r4 += d0_; r6 += d1_; } // t343 mm8 a=r0 b=r4 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r4 += d0_; r6 += d1_; } // t344 mm8 a=r2 b=r5 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t345 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r7 += d0_; r3 += d1_; } // t346 mm8 a=r6 b=r0 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r3 += d0_; r5 += d1_; } // t347 mm8 a=r5 b=r3 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r4, d0_, d1_); r4 += d0_; r5 += d1_; } // t348 mm8 a=r0 b=r4 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r4 += d0_; r7 += d1_; } // t349 mm8 a=r6 b=r1 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r3 += d0_; r5 += d1_; } // t350 mm8 a=r1 b=r5 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r4 += d0_; r5 += d1_; } // t351 mm8 a=r6 b=r4 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r6 += d0_; r0 += d1_; } // t352 mm8 a=r4 b=r1 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r1 += d0_; r4 += d1_; } // t353 mm8 a=r7 b=r3 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r1 += d0_; r5 += d1_; } // t354 mm8 a=r4 b=r1 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r5, d0_, d1_); r0 += d0_; r1 += d1_; } // t355 mm8 a=r4 b=r5 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r0 += d0_; r4 += d1_; } // t356 mm8 a=r7 b=r6 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r7 += d0_; r5 += d1_; } // t357 mm8 a=r7 b=r3 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r5 += d0_; r7 += d1_; } // t358 mm8 a=r4 b=r7 c=r5 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r3 += d0_; r5 += d1_; } // t359 mm8 a=r2 b=r7 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r0 += d0_; r2 += d1_; } // t360 mm8 a=r4 b=r4 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r5 += d0_; r6 += d1_; } // t361 mm8 a=r2 b=r2 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r3 += d0_; r0 += d1_; } // t362 mm8 a=r2 b=r7 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r5 += d0_; r0 += d1_; } // t363 mm8 a=r7 b=r6 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r0 += d0_; r5 += d1_; } // t364 mm8 a=r6 b=r4 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r5 += d0_; r4 += d1_; } // t365 mm8 a=r6 b=r6 c=r5 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r2 += d0_; r5 += d1_; } // t366 mm8 a=r7 b=r7 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t367 mm8 a=r4 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r3 += d0_; r5 += d1_; } // t368 mm8 a=r7 b=r0 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r6 += d0_; r0 += d1_; } // t369 mm8 a=r1 b=r0 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r7 += d0_; r4 += d1_; } // t370 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r2 += d0_; r6 += d1_; } // t371 mm8 a=r5 b=r4 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r2 += d1_; } // t372 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r6, d0_, d1_); r2 += d0_; r4 += d1_; } // t373 mm8 a=r4 b=r6 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r5, d0_, d1_); r0 += d0_; r4 += d1_; } // t374 mm8 a=r1 b=r5 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r6 += d0_; r2 += d1_; } // t375 mm8 a=r6 b=r5 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r0 += d0_; r3 += d1_; } // t376 mm8 a=r3 b=r6 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r6 += d0_; r4 += d1_; } // t377 mm8 a=r6 b=r4 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r2 += d0_; r4 += d1_; } // t378 mm8 a=r6 b=r3 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r2 += d0_; r4 += d1_; } // t379 mm8 a=r3 b=r0 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r6 += d0_; r2 += d1_; } // t380 mm8 a=r2 b=r0 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r1 += d0_; r2 += d1_; } // t381 mm8 a=r3 b=r5 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r2 += d0_; r4 += d1_; } // t382 mm8 a=r5 b=r2 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r0 += d0_; r2 += d1_; } // t383 mm8 a=r1 b=r2 c=r0 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r1 += d0_; r0 += d1_; } // t384 mm8 a=r3 b=r1 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r7 += d0_; r5 += d1_; } // t385 mm8 a=r6 b=r0 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r1 += d0_; r7 += d1_; } // t386 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r2 += d0_; r4 += d1_; } // t387 mm8 a=r3 b=r7 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r1 += d0_; r2 += d1_; } // t388 mm8 a=r3 b=r0 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r5 += d0_; r3 += d1_; } // t389 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r1 += d0_; r4 += d1_; } // t390 mm8 a=r5 b=r4 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r4 += d0_; r1 += d1_; } // t391 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r3 += d0_; r0 += d1_; } // t392 mm8 a=r2 b=r3 c=r3 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r6, d0_, d1_); r5 += d0_; r0 += d1_; } // t393 mm8 a=r3 b=r6 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r1 += d0_; r2 += d1_; } // t394 mm8 a=r0 b=r7 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r2 += d0_; r4 += d1_; } // t395 mm8 a=r0 b=r7 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r1 += d0_; r4 += d1_; } // t396 mm8 a=r7 b=r7 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t397 mm8 a=r5 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r7 += d0_; r0 += d1_; } // t398 mm8 a=r6 b=r5 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r3, d0_, d1_); r3 += d0_; r5 += d1_; } // t399 mm8 a=r6 b=r3 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r0 += d0_; r4 += d1_; } // t400 mm8 a=r2 b=r7 c=r0 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r2 += d0_; r1 += d1_; } // t401 mm8 a=r7 b=r6 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r7 += d0_; r1 += d1_; } // t402 mm8 a=r0 b=r0 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r1 += d0_; r0 += d1_; } // t403 mm8 a=r7 b=r4 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r1, d0_, d1_); r1 += d0_; r4 += d1_; } // t404 mm8 a=r1 b=r1 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r3 += d0_; r6 += d1_; } // t405 mm8 a=r6 b=r6 c=r3 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r4 += d0_; r3 += d1_; } // t406 mm8 a=r6 b=r1 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r2 += d0_; r4 += d1_; } // t407 mm8 a=r5 b=r7 c=r2 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r4 += d0_; r1 += d1_; } // t408 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r4, d0_, d1_); r4 += d0_; r7 += d1_; } // t409 mm8 a=r3 b=r4 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r5, d0_, d1_); r5 += d0_; r2 += d1_; } // t410 mm8 a=r0 b=r5 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r5 += d0_; r1 += d1_; } // t411 mm8 a=r2 b=r0 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r4, d0_, d1_); r3 += d0_; r1 += d1_; } // t412 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r2 += d1_; } // t413 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r0 += d0_; r3 += d1_; } // t414 mm8 a=r7 b=r1 c=r0 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r5 += d0_; r1 += d1_; } // t415 mm8 a=r4 b=r7 c=r5 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r4, d0_, d1_); r3 += d0_; r5 += d1_; } // t416 mm8 a=r5 b=r4 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t417 mm8 a=r6 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r3, d0_, d1_); r1 += d0_; r5 += d1_; } // t418 mm8 a=r1 b=r3 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r0, d0_, d1_); r0 += d0_; r6 += d1_; } // t419 mm8 a=r5 b=r0 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r2, d0_, d1_); r7 += d0_; r6 += d1_; } // t420 mm8 a=r4 b=r2 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r3 += d0_; r7 += d1_; } // t421 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r4 += d0_; r3 += d1_; } // t422 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r0 += d1_; } // t423 mm8 a=r1 b=r7 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r6 += d0_; r5 += d1_; } // t424 mm8 a=r5 b=r1 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t425 mm8 a=r2 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r5 += d0_; r0 += d1_; } // t426 mm8 a=r1 b=r6 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t427 mm8 a=r6 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r2, d0_, d1_); r7 += d0_; r4 += d1_; } // t428 mm8 a=r6 b=r2 c=r7 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r3, d0_, d1_); r7 += d0_; r0 += d1_; } // t429 mm8 a=r3 b=r3 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r3 += d0_; r5 += d1_; } // t430 mm8 a=r7 b=r7 c=r3 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r1 += d0_; r0 += d1_; } // t431 mm8 a=r6 b=r4 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r3 += d0_; r4 += d1_; } // t432 mm8 a=r3 b=r2 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r5 += d0_; r0 += d1_; } // t433 mm8 a=r3 b=r0 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r5 += d0_; r3 += d1_; } // t434 mm8 a=r0 b=r7 c=r5 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r6 += d0_; r3 += d1_; } // t435 mm8 a=r7 b=r5 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r0 += d0_; r5 += d1_; } // t436 mm8 a=r6 b=r4 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r7, d0_, d1_); r3 += d0_; r4 += d1_; } // t437 mm8 a=r0 b=r7 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r4 += d0_; r7 += d1_; } // t438 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r7, d0_, d1_); r3 += d0_; r2 += d1_; } // t439 mm8 a=r4 b=r7 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r3 += d0_; r2 += d1_; } // t440 mm8 a=r3 b=r5 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r1 += d0_; r4 += d1_; } // t441 mm8 a=r5 b=r1 c=r1 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r7 += d0_; r5 += d1_; } // t442 mm8 a=r1 b=r2 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r5 += d0_; r2 += d1_; } // t443 mm8 a=r1 b=r7 c=r5 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r1 += d0_; r0 += d1_; } // t444 mm8 a=r3 b=r7 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r7 += d0_; r0 += d1_; } // t445 mm8 a=r7 b=r5 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r2 += d0_; r1 += d1_; } // t446 mm8 a=r3 b=r1 c=r2 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r4 += d0_; r7 += d1_; } // t447 mm8 a=r6 b=r7 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r4 += d0_; r5 += d1_; } // t448 mm8 a=r5 b=r2 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r1 += d0_; r3 += d1_; } // t449 mm8 a=r7 b=r1 c=r1 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r7 += d0_; r3 += d1_; } // t450 mm8 a=r7 b=r5 c=r7 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r1 += d0_; r7 += d1_; } // t451 mm8 a=r0 b=r3 c=r1 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r7 += d0_; r1 += d1_; } // t452 mm8 a=r2 b=r4 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r7 += d0_; r1 += d1_; } // t453 mm8 a=r5 b=r7 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r7 += d0_; r1 += d1_; } // t454 mm8 a=r7 b=r7 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r7 += d0_; r5 += d1_; } // t455 mm8 a=r7 b=r1 c=r7 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r5 += d0_; r6 += d1_; } // t456 mm8 a=r6 b=r5 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r1 += d0_; r5 += d1_; } // t457 mm8 a=r7 b=r7 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r1, d0_, d1_); r4 += d0_; r1 += d1_; } // t458 mm8 a=r5 b=r1 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r7, d0_, d1_); r1 += d0_; r0 += d1_; } // t459 mm8 a=r6 b=r7 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t460 mm8 a=r7 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r6 += d0_; r2 += d1_; } // t461 mm8 a=r0 b=r6 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r1 += d0_; r2 += d1_; } // t462 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r3 += d0_; r2 += d1_; } // t463 mm8 a=r2 b=r0 c=r3 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r3, d0_, d1_); r0 += d0_; r1 += d1_; } // t464 mm8 a=r0 b=r3 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r0 += d0_; r5 += d1_; } // t465 mm8 a=r1 b=r7 c=r0 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r4 += d0_; r1 += d1_; } // t466 mm8 a=r1 b=r2 c=r4 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r0, d0_, d1_); r1 += d0_; r6 += d1_; } // t467 mm8 a=r1 b=r0 c=r1 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r1, d0_, d1_); r6 += d0_; r2 += d1_; } // t468 mm8 a=r4 b=r1 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r0 += d0_; r1 += d1_; } // t469 mm8 a=r2 b=r0 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r2 += d0_; r0 += d1_; } // t470 mm8 a=r3 b=r1 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r7 += d0_; r1 += d1_; } // t471 mm8 a=r2 b=r2 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r7, d0_, d1_); r6 += d0_; r5 += d1_; } // t472 mm8 a=r1 b=r7 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r7, d0_, d1_); r6 += d0_; r1 += d1_; } // t473 mm8 a=r2 b=r7 c=r6 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r3, d0_, d1_); r6 += d0_; r3 += d1_; } // t474 mm8 a=r5 b=r3 c=r6 c2=r3 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r6 += d0_; r0 += d1_; } // t475 mm8 a=r7 b=r6 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r0, d0_, d1_); r5 += d0_; r6 += d1_; } // t476 mm8 a=r3 b=r0 c=r5 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r4 += d0_; r6 += d1_; } // t477 mm8 a=r3 b=r1 c=r4 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r2 += d0_; r5 += d1_; } // t478 mm8 a=r0 b=r6 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r7 += d0_; r1 += d1_; } // t479 mm8 a=r7 b=r6 c=r7 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r0, d0_, d1_); r1 += d0_; r0 += d1_; } // t480 mm8 a=r6 b=r0 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r6 += d0_; r7 += d1_; } // t481 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r3, d0_, d1_); r0 += d0_; r6 += d1_; } // t482 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r0, d0_, d1_); r7 += d0_; r6 += d1_; } // t483 mm8 a=r7 b=r0 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r7, d0_, d1_); r6 += d0_; r4 += d1_; } // t484 mm8 a=r3 b=r7 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r4, d0_, d1_); r1 += d0_; r2 += d1_; } // t485 mm8 a=r6 b=r4 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r5, d0_, d1_); r2 += d0_; r5 += d1_; } // t486 mm8 a=r6 b=r5 c=r2 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r5, d0_, d1_); r4 += d0_; r5 += d1_; } // t487 mm8 a=r2 b=r5 c=r4 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r2, d0_, d1_); r1 += d0_; r2 += d1_; } // t488 mm8 a=r1 b=r2 c=r1 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r2, d0_, d1_); r6 += d0_; r4 += d1_; } // t489 mm8 a=r5 b=r2 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r1, d0_, d1_); r2 += d0_; r6 += d1_; } // t490 mm8 a=r7 b=r1 c=r2 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r0, d0_, d1_); r0 += d0_; r6 += d1_; } // t491 mm8 a=r0 b=r0 c=r0 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r6 += d0_; r4 += d1_; } // t492 mm8 a=r3 b=r1 c=r6 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r6, r1, d0_, d1_); r7 += d0_; r6 += d1_; } // t493 mm8 a=r6 b=r1 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r3 += d0_; r1 += d1_; } // t494 mm8 a=r5 b=r6 c=r3 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r2, d0_, d1_); r7 += d0_; r2 += d1_; } // t495 mm8 a=r2 b=r2 c=r7 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r5, d0_, d1_); r5 += d0_; r0 += d1_; } // t496 mm8 a=r3 b=r5 c=r5 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r2, d0_, d1_); r2 += d0_; r0 += d1_; } // t497 mm8 a=r3 b=r2 c=r2 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r3, r1, d0_, d1_); r6 += d0_; r2 += d1_; } // t498 mm8 a=r3 b=r1 c=r6 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r1, r6, d0_, d1_); r1 += d0_; r5 += d1_; } // t499 mm8 a=r1 b=r6 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r6, d0_, d1_); r1 += d0_; r5 += d1_; } // t500 mm8 a=r7 b=r6 c=r1 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r1 += d0_; r0 += d1_; } // t501 mm8 a=r4 b=r4 c=r1 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r7, d0_, d1_); r7 += d0_; r6 += d1_; } // t502 mm8 a=r5 b=r7 c=r7 c2=r6 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r5, d0_, d1_); r3 += d0_; r4 += d1_; } // t503 mm8 a=r7 b=r5 c=r3 c2=r4 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r0, d0_, d1_); r4 += d0_; r2 += d1_; } // t504 mm8 a=r2 b=r0 c=r4 c2=r2 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r0, d0_, d1_); r0 += d0_; r1 += d1_; } // t505 mm8 a=r4 b=r0 c=r0 c2=r1 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r2, d0_, d1_); r7 += d0_; r0 += d1_; } // t506 mm8 a=r0 b=r2 c=r7 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r5, r6, d0_, d1_); r4 += d0_; r7 += d1_; } // t507 mm8 a=r5 b=r6 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r7, r3, d0_, d1_); r4 += d0_; r7 += d1_; } // t508 mm8 a=r7 b=r3 c=r4 c2=r7 - { uint d0_, d1_; IGNEUM_MM8_REF(r4, r4, d0_, d1_); r6 += d0_; r0 += d1_; } // t509 mm8 a=r4 b=r4 c=r6 c2=r0 - { uint d0_, d1_; IGNEUM_MM8_REF(r0, r6, d0_, d1_); r6 += d0_; r5 += d1_; } // t510 mm8 a=r0 b=r6 c=r6 c2=r5 - { uint d0_, d1_; IGNEUM_MM8_REF(r2, r4, d0_, d1_); r1 += d0_; r3 += d1_; } // t511 mm8 a=r2 b=r4 c=r1 c2=r3 - } - uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((ulong)hi << 32) | (ulong)lo; -} diff --git a/proto-cuda/packs-ca4/mx8_mm512/kernel_bound.cu b/proto-cuda/packs-ca4/mx8_mm512/kernel_bound.cu deleted file mode 100644 index b2bed38e8..000000000 --- a/proto-cuda/packs-ca4/mx8_mm512/kernel_bound.cu +++ /dev/null @@ -1,636 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// Header-bound twin of igneum_hash in kernel.cu: the init words come from a kernel argument, not SEEDW. -// Host declarations (also in program_bound.h if present): -// struct IgneumInitWords { uint32_t w[8]; }; -// cudaError_t igneum_launch_hash_bound(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask, -// IgneumInitWords iw, uint32_t nonces, uint32_t blockWarps); -// cudaError_t igneum_hash_bound_info(int* numRegs, int* blocksPerSM, uint32_t blockWarps); -#include -#include -#include "program.h" - -struct IgneumInitWords { uint32_t w[8]; }; - -__device__ __forceinline__ uint32_t splitmix32(uint32_t x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -__device__ __forceinline__ uint32_t rotl_imm(uint32_t x, uint32_t n) { return (x << n) | (x >> (32u - n)); } -__device__ __forceinline__ uint32_t rotr_var(uint32_t x, uint32_t n) { n &= 31u; return (x >> n) | (x << ((32u - n) & 31u)); } - -__global__ void igneum_hash_bound(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask, IgneumInitWords iw) { - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t nonce = baseNonce + gid; - uint32_t r0, r1, r2, r3, r4, r5, r6, r7; - { uint32_t x = nonce ^ iw.w[0]; x += 0x9e3779b9u * 1u; x = splitmix32(x); r0 = x ^ iw.w[1]; } - { uint32_t x = nonce ^ iw.w[1]; x += 0x9e3779b9u * 2u; x = splitmix32(x); r1 = x ^ iw.w[2]; } - { uint32_t x = nonce ^ iw.w[2]; x += 0x9e3779b9u * 3u; x = splitmix32(x); r2 = x ^ iw.w[3]; } - { uint32_t x = nonce ^ iw.w[3]; x += 0x9e3779b9u * 4u; x = splitmix32(x); r3 = x ^ iw.w[4]; } - { uint32_t x = nonce ^ iw.w[4]; x += 0x9e3779b9u * 5u; x = splitmix32(x); r4 = x ^ iw.w[5]; } - { uint32_t x = nonce ^ iw.w[5]; x += 0x9e3779b9u * 6u; x = splitmix32(x); r5 = x ^ iw.w[6]; } - { uint32_t x = nonce ^ iw.w[6]; x += 0x9e3779b9u * 7u; x = splitmix32(x); r6 = x ^ iw.w[7]; } - { uint32_t x = nonce ^ iw.w[7]; x += 0x9e3779b9u * 8u; x = splitmix32(x); r7 = x ^ iw.w[0]; } - - for (uint32_t it = 0u; it < 8u; ++it) { - uint32_t sel = r0; - r2 = r3 * r4 + r2; // 0 mad - r2 = r1 * r1 + r2; // 1 mad - r2 = r3 * r2 + r2; // 2 mad - r3 = r3 ^ r5; // 3 xor - r7 = r7 ^ ds[r2 & mask]; // 4 load - r5 = r5 ^ ds[r7 & mask]; // 5 load - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r4, 8); // 6 shfl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r3, 8); // 7 shfl - r1 = __umulhi(r1, r5); // 8 mulhi - r6 = rotr_var(r6, r3); // 9 rotr - r3 = r3 | r4; // 10 or - r4 = r4 ^ ds[r3 & mask]; // 11 load - r0 = __umulhi(r0, r4); // 12 mulhi - r5 = r5 + r1 + ((((sel >> 30u) & 1u) != 0u) ? 0xd3177981u : 0xc7934706u); // 13 add - r0 = r0 ^ ds[r4 & mask]; // 14 load - r2 = r2 - r4; // 15 sub - r2 = r2 ^ ds[r0 & mask]; // 16 load - r7 = r7 ^ ds[r2 & mask]; // 17 load - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r3, 4); // 18 shfl - r5 = r5 * r0; // 19 mul - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r4, 2); // 20 shfl - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r4, 16); // 21 shfl - r6 = __umulhi(r6, r2); // 22 mulhi - r6 = r6 ^ ds[r1 & mask]; // 23 load - r5 = r5 * r0; // 24 mul - r5 = rotl_imm(r5, 19u); // 25 rotl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r6, 2); // 26 shfl - r0 = r0 ^ r5; // 27 xor - r0 = r0 ^ r4; // 28 xor - r3 = r3 - r0; // 29 sub - r5 = r5 * r1; // 30 mul - r7 = r7 ^ ds[r2 & mask]; // 31 load - r1 = r1 ^ ds[r0 & mask]; // 32 load - r5 = r5 ^ r6; // 33 xor - r5 = r5 ^ ds[r1 & mask]; // 34 load - r0 = __umulhi(r0, r5); // 35 mulhi - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r2, 4); // 36 shfl - r7 = r7 ^ ds[r0 & mask]; // 37 load - r3 = r3 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0x230c005cu : 0x75ba2fadu); // 38 add - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r5, 4); // 39 shfl - r2 = r2 ^ r5; // 40 xor - r3 = r6 * r3 + r3; // 41 mad - r6 = r6 - r7; // 42 sub - r7 = r7 ^ r0; // 43 xor - r1 = r1 ^ ds[r7 & mask]; // 44 load - r2 = r2 * r3; // 45 mul - r1 = __umulhi(r1, r5); // 46 mulhi - r4 = r4 - r3; // 47 sub - r2 = rotr_var(r2, r6); // 48 rotr - r3 = r3 ^ ds[r5 & mask]; // 49 load - r1 = r1 + r5 + ((((sel >> 7u) & 1u) != 0u) ? 0x1907970cu : 0x81b8bc2cu); // 50 add - r0 = r0 * r2; // 51 mul - r0 = r0 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x699fd448u : 0x4f92b968u); // 52 add - r1 = r1 + r0 + ((((sel >> 12u) & 1u) != 0u) ? 0x77b1520du : 0x2bb965afu); // 53 add - r7 = rotl_imm(r7, 14u); // 54 rotl - r3 = r3 + r7 + ((((sel >> 1u) & 1u) != 0u) ? 0xa54c55a0u : 0x7b0fe07au); // 55 add - r6 = r6 ^ ds[r7 & mask]; // 56 load - r1 = rotr_var(r1, r5); // 57 rotr - r5 = r5 ^ ds[r4 & mask]; // 58 load - r6 = r6 ^ ds[r2 & mask]; // 59 load - r3 = r5 * r0 + r3; // 60 mad - r5 = r5 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0xad7493e7u : 0xaf9dd72du); // 61 add - r4 = r4 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x1e07c3d9u : 0x89841d87u); // 62 add - r5 = rotl_imm(r5, 19u); // 63 rotl - // int8 tile block (Counter ASIC 4.0 research, experimental): 512 mma.m8n8k16 u8 tiles per iteration, both outputs consumed - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t0 mm8 a=r6 b=r5 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t1 mm8 a=r1 b=r1 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t2 mm8 a=r2 b=r5 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t3 mm8 a=r1 b=r5 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t4 mm8 a=r2 b=r0 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t5 mm8 a=r2 b=r6 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t6 mm8 a=r1 b=r4 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t7 mm8 a=r4 b=r6 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t8 mm8 a=r1 b=r1 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t9 mm8 a=r3 b=r7 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t10 mm8 a=r3 b=r0 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t11 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t12 mm8 a=r3 b=r1 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t13 mm8 a=r1 b=r5 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t14 mm8 a=r3 b=r3 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t15 mm8 a=r1 b=r0 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t16 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t17 mm8 a=r0 b=r5 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t18 mm8 a=r1 b=r1 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t19 mm8 a=r3 b=r3 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t20 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t21 mm8 a=r0 b=r3 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t22 mm8 a=r1 b=r7 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t23 mm8 a=r1 b=r2 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t24 mm8 a=r3 b=r4 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t25 mm8 a=r5 b=r0 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t26 mm8 a=r1 b=r0 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t27 mm8 a=r3 b=r2 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t28 mm8 a=r2 b=r4 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t29 mm8 a=r3 b=r4 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t30 mm8 a=r6 b=r5 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t31 mm8 a=r1 b=r7 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t32 mm8 a=r7 b=r6 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t33 mm8 a=r7 b=r1 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t34 mm8 a=r1 b=r0 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t35 mm8 a=r2 b=r4 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t36 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t37 mm8 a=r1 b=r7 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t38 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t39 mm8 a=r6 b=r2 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t40 mm8 a=r6 b=r1 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t41 mm8 a=r6 b=r0 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t42 mm8 a=r7 b=r4 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t43 mm8 a=r6 b=r1 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t44 mm8 a=r0 b=r6 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t45 mm8 a=r6 b=r5 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t46 mm8 a=r6 b=r6 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t47 mm8 a=r7 b=r3 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t48 mm8 a=r7 b=r7 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t49 mm8 a=r4 b=r5 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t50 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t51 mm8 a=r3 b=r4 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t52 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t53 mm8 a=r7 b=r7 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t54 mm8 a=r7 b=r6 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t55 mm8 a=r1 b=r7 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t56 mm8 a=r4 b=r0 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t57 mm8 a=r4 b=r3 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t58 mm8 a=r2 b=r3 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t59 mm8 a=r6 b=r7 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t60 mm8 a=r6 b=r0 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t61 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t62 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t63 mm8 a=r1 b=r6 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t64 mm8 a=r0 b=r5 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t65 mm8 a=r0 b=r1 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t66 mm8 a=r3 b=r5 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t67 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t68 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t69 mm8 a=r6 b=r7 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t70 mm8 a=r1 b=r7 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t71 mm8 a=r3 b=r0 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t72 mm8 a=r4 b=r0 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t73 mm8 a=r4 b=r1 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t74 mm8 a=r5 b=r1 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t75 mm8 a=r4 b=r4 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t76 mm8 a=r5 b=r6 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t77 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t78 mm8 a=r2 b=r5 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t79 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t80 mm8 a=r6 b=r7 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t81 mm8 a=r4 b=r3 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t82 mm8 a=r6 b=r0 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t83 mm8 a=r2 b=r2 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t84 mm8 a=r0 b=r6 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t85 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t86 mm8 a=r3 b=r4 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t87 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t88 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t89 mm8 a=r4 b=r2 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t90 mm8 a=r6 b=r5 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t91 mm8 a=r6 b=r5 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t92 mm8 a=r2 b=r6 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t93 mm8 a=r2 b=r2 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t94 mm8 a=r2 b=r4 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t95 mm8 a=r5 b=r4 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t96 mm8 a=r0 b=r6 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t97 mm8 a=r0 b=r5 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t98 mm8 a=r2 b=r7 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t99 mm8 a=r1 b=r4 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t100 mm8 a=r2 b=r4 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t101 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t102 mm8 a=r5 b=r7 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t103 mm8 a=r4 b=r7 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t104 mm8 a=r2 b=r0 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t105 mm8 a=r6 b=r1 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t106 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t107 mm8 a=r2 b=r1 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t108 mm8 a=r4 b=r6 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t109 mm8 a=r2 b=r7 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t110 mm8 a=r1 b=r0 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t111 mm8 a=r6 b=r5 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t112 mm8 a=r5 b=r7 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t113 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t114 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t115 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t116 mm8 a=r7 b=r2 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t117 mm8 a=r7 b=r6 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t118 mm8 a=r0 b=r1 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t119 mm8 a=r4 b=r3 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t120 mm8 a=r5 b=r3 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t121 mm8 a=r2 b=r7 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t122 mm8 a=r0 b=r1 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t123 mm8 a=r5 b=r2 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t124 mm8 a=r5 b=r1 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t125 mm8 a=r7 b=r2 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t126 mm8 a=r6 b=r3 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t127 mm8 a=r6 b=r7 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t128 mm8 a=r6 b=r7 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t129 mm8 a=r3 b=r2 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t130 mm8 a=r0 b=r4 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t131 mm8 a=r3 b=r5 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t132 mm8 a=r0 b=r7 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t133 mm8 a=r2 b=r6 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t134 mm8 a=r5 b=r6 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t135 mm8 a=r0 b=r2 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t136 mm8 a=r3 b=r1 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t137 mm8 a=r7 b=r0 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t138 mm8 a=r2 b=r2 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t139 mm8 a=r0 b=r4 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t140 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t141 mm8 a=r5 b=r5 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t142 mm8 a=r2 b=r2 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t143 mm8 a=r4 b=r3 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t144 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t145 mm8 a=r7 b=r2 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t146 mm8 a=r2 b=r7 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t147 mm8 a=r3 b=r1 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t148 mm8 a=r2 b=r1 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t149 mm8 a=r4 b=r6 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t150 mm8 a=r3 b=r6 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t151 mm8 a=r1 b=r4 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t152 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t153 mm8 a=r0 b=r3 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t154 mm8 a=r7 b=r3 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t155 mm8 a=r1 b=r3 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t156 mm8 a=r6 b=r3 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t157 mm8 a=r6 b=r4 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t158 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t159 mm8 a=r1 b=r0 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t160 mm8 a=r0 b=r1 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t161 mm8 a=r4 b=r0 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t162 mm8 a=r6 b=r6 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t163 mm8 a=r5 b=r3 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t164 mm8 a=r0 b=r7 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t165 mm8 a=r6 b=r5 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t166 mm8 a=r6 b=r5 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t167 mm8 a=r4 b=r5 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t168 mm8 a=r3 b=r6 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t169 mm8 a=r6 b=r2 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t170 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t171 mm8 a=r6 b=r3 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t172 mm8 a=r7 b=r3 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t173 mm8 a=r0 b=r0 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t174 mm8 a=r6 b=r6 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t175 mm8 a=r1 b=r6 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t176 mm8 a=r2 b=r4 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t177 mm8 a=r1 b=r7 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t178 mm8 a=r4 b=r2 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t179 mm8 a=r5 b=r2 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t180 mm8 a=r0 b=r4 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t181 mm8 a=r3 b=r5 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t182 mm8 a=r1 b=r1 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t183 mm8 a=r3 b=r1 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t184 mm8 a=r7 b=r4 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t185 mm8 a=r5 b=r0 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t186 mm8 a=r2 b=r5 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t187 mm8 a=r3 b=r0 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t188 mm8 a=r1 b=r4 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t189 mm8 a=r7 b=r2 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t190 mm8 a=r1 b=r5 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t191 mm8 a=r2 b=r1 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t192 mm8 a=r5 b=r0 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t193 mm8 a=r2 b=r1 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t194 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t195 mm8 a=r2 b=r7 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t196 mm8 a=r5 b=r6 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t197 mm8 a=r5 b=r3 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t198 mm8 a=r1 b=r0 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t199 mm8 a=r2 b=r5 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t200 mm8 a=r5 b=r7 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t201 mm8 a=r7 b=r2 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t202 mm8 a=r2 b=r5 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t203 mm8 a=r0 b=r5 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t204 mm8 a=r4 b=r7 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t205 mm8 a=r3 b=r7 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t206 mm8 a=r0 b=r6 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t207 mm8 a=r7 b=r7 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t208 mm8 a=r0 b=r1 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t209 mm8 a=r4 b=r4 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t210 mm8 a=r7 b=r1 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t211 mm8 a=r1 b=r5 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t212 mm8 a=r2 b=r4 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t213 mm8 a=r2 b=r2 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t214 mm8 a=r4 b=r6 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t215 mm8 a=r7 b=r3 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t216 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t217 mm8 a=r5 b=r7 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t218 mm8 a=r7 b=r1 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t219 mm8 a=r2 b=r1 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t220 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t221 mm8 a=r7 b=r1 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t222 mm8 a=r0 b=r6 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t223 mm8 a=r2 b=r2 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t224 mm8 a=r1 b=r1 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t225 mm8 a=r2 b=r7 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t226 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t227 mm8 a=r3 b=r5 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t228 mm8 a=r7 b=r1 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t229 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t230 mm8 a=r0 b=r3 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t231 mm8 a=r1 b=r3 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t232 mm8 a=r5 b=r4 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t233 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t234 mm8 a=r2 b=r0 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t235 mm8 a=r3 b=r4 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t236 mm8 a=r1 b=r7 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t237 mm8 a=r3 b=r5 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t238 mm8 a=r2 b=r0 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t239 mm8 a=r5 b=r7 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t240 mm8 a=r2 b=r7 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t241 mm8 a=r0 b=r1 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t242 mm8 a=r0 b=r2 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t243 mm8 a=r4 b=r4 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r7 += d1_; } // t244 mm8 a=r7 b=r6 c=r0 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t245 mm8 a=r4 b=r4 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t246 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t247 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t248 mm8 a=r7 b=r5 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t249 mm8 a=r4 b=r4 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t250 mm8 a=r0 b=r6 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t251 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t252 mm8 a=r0 b=r2 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t253 mm8 a=r5 b=r5 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t254 mm8 a=r6 b=r7 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t255 mm8 a=r5 b=r0 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t256 mm8 a=r2 b=r1 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t257 mm8 a=r4 b=r3 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t258 mm8 a=r4 b=r1 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t259 mm8 a=r5 b=r5 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t260 mm8 a=r3 b=r2 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t261 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t262 mm8 a=r6 b=r4 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t263 mm8 a=r6 b=r4 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t264 mm8 a=r7 b=r2 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t265 mm8 a=r6 b=r1 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t266 mm8 a=r0 b=r0 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t267 mm8 a=r6 b=r3 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t268 mm8 a=r1 b=r0 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t269 mm8 a=r7 b=r7 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t270 mm8 a=r6 b=r2 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t271 mm8 a=r0 b=r3 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t272 mm8 a=r6 b=r0 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t273 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t274 mm8 a=r1 b=r0 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t275 mm8 a=r1 b=r3 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t276 mm8 a=r1 b=r5 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t277 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t278 mm8 a=r0 b=r0 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t279 mm8 a=r2 b=r4 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t280 mm8 a=r7 b=r2 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t281 mm8 a=r1 b=r0 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t282 mm8 a=r0 b=r1 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t283 mm8 a=r0 b=r2 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t284 mm8 a=r1 b=r5 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t285 mm8 a=r4 b=r7 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t286 mm8 a=r1 b=r0 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t287 mm8 a=r7 b=r1 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t288 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t289 mm8 a=r0 b=r5 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t290 mm8 a=r3 b=r2 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t291 mm8 a=r7 b=r4 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t292 mm8 a=r6 b=r7 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t293 mm8 a=r5 b=r5 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t294 mm8 a=r5 b=r7 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t295 mm8 a=r0 b=r7 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t296 mm8 a=r6 b=r1 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t297 mm8 a=r4 b=r5 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t298 mm8 a=r6 b=r0 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t299 mm8 a=r6 b=r0 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t300 mm8 a=r6 b=r4 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t301 mm8 a=r6 b=r4 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t302 mm8 a=r7 b=r6 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t303 mm8 a=r3 b=r2 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t304 mm8 a=r6 b=r5 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t305 mm8 a=r4 b=r5 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t306 mm8 a=r3 b=r1 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t307 mm8 a=r4 b=r3 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t308 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t309 mm8 a=r2 b=r1 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t310 mm8 a=r6 b=r1 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t311 mm8 a=r4 b=r6 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t312 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t313 mm8 a=r2 b=r6 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t314 mm8 a=r0 b=r3 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t315 mm8 a=r7 b=r1 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t316 mm8 a=r2 b=r4 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t317 mm8 a=r1 b=r6 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t318 mm8 a=r2 b=r1 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t319 mm8 a=r0 b=r2 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t320 mm8 a=r5 b=r2 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t321 mm8 a=r7 b=r4 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t322 mm8 a=r2 b=r4 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t323 mm8 a=r4 b=r3 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t324 mm8 a=r5 b=r7 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t325 mm8 a=r6 b=r2 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t326 mm8 a=r6 b=r1 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t327 mm8 a=r1 b=r5 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t328 mm8 a=r0 b=r1 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t329 mm8 a=r0 b=r3 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r0 += d1_; } // t330 mm8 a=r6 b=r7 c=r4 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t331 mm8 a=r4 b=r4 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t332 mm8 a=r4 b=r4 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t333 mm8 a=r4 b=r6 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t334 mm8 a=r2 b=r2 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t335 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r3 += d1_; } // t336 mm8 a=r4 b=r0 c=r2 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t337 mm8 a=r1 b=r4 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r7 += d1_; } // t338 mm8 a=r3 b=r7 c=r2 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t339 mm8 a=r5 b=r1 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t340 mm8 a=r5 b=r0 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t341 mm8 a=r6 b=r6 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t342 mm8 a=r4 b=r7 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t343 mm8 a=r0 b=r4 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t344 mm8 a=r2 b=r5 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t345 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t346 mm8 a=r6 b=r0 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t347 mm8 a=r5 b=r3 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t348 mm8 a=r0 b=r4 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t349 mm8 a=r6 b=r1 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t350 mm8 a=r1 b=r5 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t351 mm8 a=r6 b=r4 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t352 mm8 a=r4 b=r1 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t353 mm8 a=r7 b=r3 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t354 mm8 a=r4 b=r1 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t355 mm8 a=r4 b=r5 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t356 mm8 a=r7 b=r6 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t357 mm8 a=r7 b=r3 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r7 += d1_; } // t358 mm8 a=r4 b=r7 c=r5 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t359 mm8 a=r2 b=r7 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t360 mm8 a=r4 b=r4 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t361 mm8 a=r2 b=r2 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t362 mm8 a=r2 b=r7 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t363 mm8 a=r7 b=r6 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t364 mm8 a=r6 b=r4 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r4 += d1_; } // t365 mm8 a=r6 b=r6 c=r5 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t366 mm8 a=r7 b=r7 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t367 mm8 a=r4 b=r1 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t368 mm8 a=r7 b=r0 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t369 mm8 a=r1 b=r0 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t370 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t371 mm8 a=r5 b=r4 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t372 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t373 mm8 a=r4 b=r6 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r5), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t374 mm8 a=r1 b=r5 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t375 mm8 a=r6 b=r5 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t376 mm8 a=r3 b=r6 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t377 mm8 a=r6 b=r4 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t378 mm8 a=r6 b=r3 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t379 mm8 a=r3 b=r0 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t380 mm8 a=r2 b=r0 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t381 mm8 a=r3 b=r5 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t382 mm8 a=r5 b=r2 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r0 += d0_; r2 += d1_; } // t383 mm8 a=r1 b=r2 c=r0 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t384 mm8 a=r3 b=r1 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t385 mm8 a=r6 b=r0 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t386 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t387 mm8 a=r3 b=r7 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t388 mm8 a=r3 b=r0 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t389 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t390 mm8 a=r5 b=r4 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t391 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r0 += d1_; } // t392 mm8 a=r2 b=r3 c=r3 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t393 mm8 a=r3 b=r6 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t394 mm8 a=r0 b=r7 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t395 mm8 a=r0 b=r7 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t396 mm8 a=r7 b=r7 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t397 mm8 a=r5 b=r7 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t398 mm8 a=r6 b=r5 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r3), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t399 mm8 a=r6 b=r3 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r4 += d1_; } // t400 mm8 a=r2 b=r7 c=r0 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t401 mm8 a=r7 b=r6 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t402 mm8 a=r0 b=r0 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t403 mm8 a=r7 b=r4 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t404 mm8 a=r1 b=r1 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r6 += d1_; } // t405 mm8 a=r6 b=r6 c=r3 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t406 mm8 a=r6 b=r1 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r2 += d0_; r4 += d1_; } // t407 mm8 a=r5 b=r7 c=r2 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t408 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r4), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t409 mm8 a=r3 b=r4 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t410 mm8 a=r0 b=r5 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t411 mm8 a=r2 b=r0 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t412 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t413 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r0 += d0_; r3 += d1_; } // t414 mm8 a=r7 b=r1 c=r0 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r1 += d1_; } // t415 mm8 a=r4 b=r7 c=r5 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t416 mm8 a=r5 b=r4 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t417 mm8 a=r6 b=r6 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t418 mm8 a=r1 b=r3 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t419 mm8 a=r5 b=r0 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t420 mm8 a=r4 b=r2 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r3 += d0_; r7 += d1_; } // t421 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r3 += d1_; } // t422 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t423 mm8 a=r1 b=r7 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t424 mm8 a=r5 b=r1 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t425 mm8 a=r2 b=r7 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t426 mm8 a=r1 b=r6 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t427 mm8 a=r6 b=r6 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r4 += d1_; } // t428 mm8 a=r6 b=r2 c=r7 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r3), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t429 mm8 a=r3 b=r3 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r5 += d1_; } // t430 mm8 a=r7 b=r7 c=r3 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t431 mm8 a=r6 b=r4 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t432 mm8 a=r3 b=r2 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t433 mm8 a=r3 b=r0 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r3 += d1_; } // t434 mm8 a=r0 b=r7 c=r5 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t435 mm8 a=r7 b=r5 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t436 mm8 a=r6 b=r4 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t437 mm8 a=r0 b=r7 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t438 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r7), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t439 mm8 a=r4 b=r7 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t440 mm8 a=r3 b=r5 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r4 += d1_; } // t441 mm8 a=r5 b=r1 c=r1 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t442 mm8 a=r1 b=r2 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r5 += d0_; r2 += d1_; } // t443 mm8 a=r1 b=r7 c=r5 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t444 mm8 a=r3 b=r7 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t445 mm8 a=r7 b=r5 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r1 += d1_; } // t446 mm8 a=r3 b=r1 c=r2 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t447 mm8 a=r6 b=r7 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t448 mm8 a=r5 b=r2 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t449 mm8 a=r7 b=r1 c=r1 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r7 += d0_; r3 += d1_; } // t450 mm8 a=r7 b=r5 c=r7 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r1 += d0_; r7 += d1_; } // t451 mm8 a=r0 b=r3 c=r1 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t452 mm8 a=r2 b=r4 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t453 mm8 a=r5 b=r7 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t454 mm8 a=r7 b=r7 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r5 += d1_; } // t455 mm8 a=r7 b=r1 c=r7 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t456 mm8 a=r6 b=r5 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t457 mm8 a=r7 b=r7 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t458 mm8 a=r5 b=r1 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r7), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t459 mm8 a=r6 b=r7 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t460 mm8 a=r7 b=r7 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t461 mm8 a=r0 b=r6 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t462 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r3 += d0_; r2 += d1_; } // t463 mm8 a=r2 b=r0 c=r3 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t464 mm8 a=r0 b=r3 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r0 += d0_; r5 += d1_; } // t465 mm8 a=r1 b=r7 c=r0 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r4 += d0_; r1 += d1_; } // t466 mm8 a=r1 b=r2 c=r4 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r6 += d1_; } // t467 mm8 a=r1 b=r0 c=r1 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t468 mm8 a=r4 b=r1 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t469 mm8 a=r2 b=r0 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t470 mm8 a=r3 b=r1 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t471 mm8 a=r2 b=r2 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t472 mm8 a=r1 b=r7 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r1 += d1_; } // t473 mm8 a=r2 b=r7 c=r6 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r3), "r"(0u), "r"(0u)); r6 += d0_; r3 += d1_; } // t474 mm8 a=r5 b=r3 c=r6 c2=r3 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t475 mm8 a=r7 b=r6 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r0), "r"(0u), "r"(0u)); r5 += d0_; r6 += d1_; } // t476 mm8 a=r3 b=r0 c=r5 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r4 += d0_; r6 += d1_; } // t477 mm8 a=r3 b=r1 c=r4 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t478 mm8 a=r0 b=r6 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r7 += d0_; r1 += d1_; } // t479 mm8 a=r7 b=r6 c=r7 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r0), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t480 mm8 a=r6 b=r0 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r7 += d1_; } // t481 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r3), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t482 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r0), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t483 mm8 a=r7 b=r0 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r7), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t484 mm8 a=r3 b=r7 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t485 mm8 a=r6 b=r4 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r5), "r"(0u), "r"(0u)); r2 += d0_; r5 += d1_; } // t486 mm8 a=r6 b=r5 c=r2 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r5), "r"(0u), "r"(0u)); r4 += d0_; r5 += d1_; } // t487 mm8 a=r2 b=r5 c=r4 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r2), "r"(0u), "r"(0u)); r1 += d0_; r2 += d1_; } // t488 mm8 a=r1 b=r2 c=r1 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r2), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t489 mm8 a=r5 b=r2 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r1), "r"(0u), "r"(0u)); r2 += d0_; r6 += d1_; } // t490 mm8 a=r7 b=r1 c=r2 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r6 += d1_; } // t491 mm8 a=r0 b=r0 c=r0 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r4 += d1_; } // t492 mm8 a=r3 b=r1 c=r6 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r6), "r"(r1), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t493 mm8 a=r6 b=r1 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r3 += d0_; r1 += d1_; } // t494 mm8 a=r5 b=r6 c=r3 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r2 += d1_; } // t495 mm8 a=r2 b=r2 c=r7 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r5), "r"(0u), "r"(0u)); r5 += d0_; r0 += d1_; } // t496 mm8 a=r3 b=r5 c=r5 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r2), "r"(0u), "r"(0u)); r2 += d0_; r0 += d1_; } // t497 mm8 a=r3 b=r2 c=r2 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r3), "r"(r1), "r"(0u), "r"(0u)); r6 += d0_; r2 += d1_; } // t498 mm8 a=r3 b=r1 c=r6 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r1), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t499 mm8 a=r1 b=r6 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r6), "r"(0u), "r"(0u)); r1 += d0_; r5 += d1_; } // t500 mm8 a=r7 b=r6 c=r1 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r0 += d1_; } // t501 mm8 a=r4 b=r4 c=r1 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r7), "r"(0u), "r"(0u)); r7 += d0_; r6 += d1_; } // t502 mm8 a=r5 b=r7 c=r7 c2=r6 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r5), "r"(0u), "r"(0u)); r3 += d0_; r4 += d1_; } // t503 mm8 a=r7 b=r5 c=r3 c2=r4 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r0), "r"(0u), "r"(0u)); r4 += d0_; r2 += d1_; } // t504 mm8 a=r2 b=r0 c=r4 c2=r2 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r0), "r"(0u), "r"(0u)); r0 += d0_; r1 += d1_; } // t505 mm8 a=r4 b=r0 c=r0 c2=r1 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r2), "r"(0u), "r"(0u)); r7 += d0_; r0 += d1_; } // t506 mm8 a=r0 b=r2 c=r7 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r5), "r"(r6), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t507 mm8 a=r5 b=r6 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r7), "r"(r3), "r"(0u), "r"(0u)); r4 += d0_; r7 += d1_; } // t508 mm8 a=r7 b=r3 c=r4 c2=r7 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r4), "r"(r4), "r"(0u), "r"(0u)); r6 += d0_; r0 += d1_; } // t509 mm8 a=r4 b=r4 c=r6 c2=r0 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r0), "r"(r6), "r"(0u), "r"(0u)); r6 += d0_; r5 += d1_; } // t510 mm8 a=r0 b=r6 c=r6 c2=r5 - { uint32_t d0_, d1_; asm volatile("mma.sync.aligned.m8n8k16.row.col.s32.u8.u8.s32 {%0,%1}, {%2}, {%3}, {%4,%5};" : "=r"(d0_), "=r"(d1_) : "r"(r2), "r"(r4), "r"(0u), "r"(0u)); r1 += d0_; r3 += d1_; } // t511 mm8 a=r2 b=r4 c=r1 c2=r3 - } - uint32_t lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint32_t hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((uint64_t)hi << 32) | (uint64_t)lo; -} - -cudaError_t igneum_launch_hash_bound(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask, - IgneumInitWords iw, uint32_t nonces, uint32_t blockWarps) { - if (blockWarps == 0u || blockWarps > 32u) return cudaErrorInvalidValue; - uint32_t block = 32u * blockWarps; - if (nonces == 0u || (nonces % block) != 0u) return cudaErrorInvalidValue; - igneum_hash_bound<<>>(ds, out, baseNonce, mask, iw); - return cudaGetLastError(); -} - -cudaError_t igneum_hash_bound_info(int* numRegs, int* blocksPerSM, uint32_t blockWarps) { - cudaFuncAttributes attr; - cudaError_t e = cudaFuncGetAttributes(&attr, igneum_hash_bound); - if (e != cudaSuccess) return e; - *numRegs = attr.numRegs; - return cudaOccupancyMaxActiveBlocksPerMultiprocessor(blocksPerSM, igneum_hash_bound, (int)(32u * blockWarps), 0); -} diff --git a/proto-cuda/packs-ca4/mx8_mm512/memhard.h b/proto-cuda/packs-ca4/mx8_mm512/memhard.h deleted file mode 100644 index f7f34c732..000000000 --- a/proto-cuda/packs-ca4/mx8_mm512/memhard.h +++ /dev/null @@ -1,109 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// Memory-hard dataset core, the same text that the Mac's Metal kernels and CPU verifier were checked against. -// Included by kernel.cu (device), host.cu (host reference) and proto-opencl/host.c (C99 host reference). -// See proto-metal/MEMHARD.md for the construction. kernel.cl carries the same text in OpenCL C. -#pragma once -#ifdef __cplusplus -#include -#else -#include -#endif -#if defined(__CUDACC__) -#define IGNEUM_HD __host__ __device__ __forceinline__ -#elif defined(_MSC_VER) && !defined(__cplusplus) -#define IGNEUM_HD static __inline -#else -#define IGNEUM_HD static inline -#endif -// Memory-hard dataset core (MEMHARD.md). Cache: 2^26 words in 2^16 segments of 64 chained ChaCha12 lines. -// Item: 8 rounds of 8 x seed-parameterised mixer + one 64-byte cache read, then 8 x final mixer (class v3, mixer multiplier 8, -// docs/plans/mixer-x4.md: the round key of application j of round r is 0x9E3779B9 * (r * m + j + 1)). All parameters are literals. -#define MH_CACHE_LINE_MASK 0x003fffffu -#define MH_SEGMENT_LINES 64u -#define MH_QR(a, b, c, d, r1, r2, r3, r4) { a += b; d ^= a; d = mh_rotl(d, r1); c += d; b ^= c; b = mh_rotl(b, r2); a += b; d ^= a; d = mh_rotl(d, r3); c += d; b ^= c; b = mh_rotl(b, r4); } -IGNEUM_HD uint32_t mh_rotl(uint32_t x, uint32_t n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 at every call site - -// y = ChaCha12 core(x) + x -IGNEUM_HD void mh_chacha_block(const uint32_t* x, uint32_t* y) { - for (uint32_t i = 0u; i < 16u; ++i) y[i] = x[i]; - for (uint32_t r = 0u; r < 6u; ++r) { - MH_QR(y[0], y[4], y[8], y[12], 16u, 12u, 8u, 7u) MH_QR(y[1], y[5], y[9], y[13], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[6], y[10], y[14], 16u, 12u, 8u, 7u) MH_QR(y[3], y[7], y[11], y[15], 16u, 12u, 8u, 7u) - MH_QR(y[0], y[5], y[10], y[15], 16u, 12u, 8u, 7u) MH_QR(y[1], y[6], y[11], y[12], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[7], y[8], y[13], 16u, 12u, 8u, 7u) MH_QR(y[3], y[4], y[9], y[14], 16u, 12u, 8u, 7u) - } - for (uint32_t i = 0u; i < 16u; ++i) y[i] += x[i]; -} - -// One cache segment: 64 chained lines written at cache[seg * 1024]. in_j = prev ^ (sigma || K || seg || j || tag), prev_0 = 0. -IGNEUM_HD void mh_cache_segment(uint32_t* cache, uint32_t seg) { - uint32_t prev[16]; uint32_t x[16]; uint32_t y[16]; - for (uint32_t i = 0u; i < 16u; ++i) prev[i] = 0u; - for (uint32_t j = 0u; j < MH_SEGMENT_LINES; ++j) { - x[0] = 0x61707865u ^ prev[0]; x[1] = 0x3320646eu ^ prev[1]; x[2] = 0x79622d32u ^ prev[2]; x[3] = 0x6b206574u ^ prev[3]; - x[4] = 0x3067619fu ^ prev[4]; - x[5] = 0x3c269176u ^ prev[5]; - x[6] = 0x84a03b03u ^ prev[6]; - x[7] = 0xf8c63294u ^ prev[7]; - x[8] = 0xff977c5bu ^ prev[8]; - x[9] = 0xe60def3eu ^ prev[9]; - x[10] = 0x63630141u ^ prev[10]; - x[11] = 0xb8fbcb58u ^ prev[11]; - x[12] = seg ^ prev[12]; x[13] = j ^ prev[13]; x[14] = 0x49676e65u ^ prev[14]; x[15] = 0x756d4d48u ^ prev[15]; - mh_chacha_block(x, y); - uint32_t* line = cache + ((seg * MH_SEGMENT_LINES + j) * 16u); - for (uint32_t i = 0u; i < 16u; ++i) { line[i] = y[i]; prev[i] = y[i]; } - } -} - -// M_r: per word (s ^ (RC + rk)) * MUL, then a column round and a diagonal round with the seed-drawn rotations. -IGNEUM_HD void mh_mixer(uint32_t* s, uint32_t rk) { - s[0] = (s[0] ^ (0xbab68293u + rk)) * 0x42146205u; - s[1] = (s[1] ^ (0xcc162340u + rk)) * 0x52cbe0fbu; - s[2] = (s[2] ^ (0x6ce151ccu + rk)) * 0x7ecf4a03u; - s[3] = (s[3] ^ (0xe62b8997u + rk)) * 0x6728907fu; - s[4] = (s[4] ^ (0xc9c80297u + rk)) * 0xd81d9751u; - s[5] = (s[5] ^ (0xf74a1654u + rk)) * 0x132952c3u; - s[6] = (s[6] ^ (0x3d704af5u + rk)) * 0xf60de277u; - s[7] = (s[7] ^ (0x3cf522b7u + rk)) * 0x05358035u; - s[8] = (s[8] ^ (0x2b9cac04u + rk)) * 0xbaf6499du; - s[9] = (s[9] ^ (0xa880ac10u + rk)) * 0xe4db9667u; - s[10] = (s[10] ^ (0x13e5dd1du + rk)) * 0x3e98f45du; - s[11] = (s[11] ^ (0x6fc3e233u + rk)) * 0xd0004eddu; - s[12] = (s[12] ^ (0x2d83eeacu + rk)) * 0x2691630du; - s[13] = (s[13] ^ (0x9006e8bfu + rk)) * 0x9beb3bcfu; - s[14] = (s[14] ^ (0x2c4b5362u + rk)) * 0xab310379u; - s[15] = (s[15] ^ (0x31b49ee2u + rk)) * 0x99cfb423u; - MH_QR(s[0], s[4], s[8], s[12], 20u, 20u, 19u, 4u) MH_QR(s[1], s[5], s[9], s[13], 20u, 20u, 19u, 4u) - MH_QR(s[2], s[6], s[10], s[14], 20u, 20u, 19u, 4u) MH_QR(s[3], s[7], s[11], s[15], 20u, 20u, 19u, 4u) - MH_QR(s[0], s[5], s[10], s[15], 26u, 3u, 3u, 27u) MH_QR(s[1], s[6], s[11], s[12], 26u, 3u, 3u, 27u) - MH_QR(s[2], s[7], s[8], s[13], 26u, 3u, 3u, 27u) MH_QR(s[3], s[4], s[9], s[14], 26u, 3u, 3u, 27u) -} - -// Item t: 16 words. s = (K, t * MUL[i] + RC[i]); 8 rounds of 8 x mixer + cache line s[0] & mask; 8 x final mixer. -IGNEUM_HD void mh_item(const uint32_t* cache, uint32_t t, uint32_t* s) { - s[0] = 0x3067619fu; - s[1] = 0x3c269176u; - s[2] = 0x84a03b03u; - s[3] = 0xf8c63294u; - s[4] = 0xff977c5bu; - s[5] = 0xe60def3eu; - s[6] = 0x63630141u; - s[7] = 0xb8fbcb58u; - s[8] = t * 0x42146205u + 0xbab68293u; - s[9] = t * 0x52cbe0fbu + 0xcc162340u; - s[10] = t * 0x7ecf4a03u + 0x6ce151ccu; - s[11] = t * 0x6728907fu + 0xe62b8997u; - s[12] = t * 0xd81d9751u + 0xc9c80297u; - s[13] = t * 0x132952c3u + 0xf74a1654u; - s[14] = t * 0xf60de277u + 0x3d704af5u; - s[15] = t * 0x05358035u + 0x3cf522b7u; - for (uint32_t r = 0u; r < 8u; ++r) { - for (uint32_t j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (r * 8u + j + 1u)); - const uint32_t* line = cache + ((s[0] & MH_CACHE_LINE_MASK) * 16u); - for (uint32_t i = 0u; i < 16u; ++i) s[i] ^= line[i]; - } - for (uint32_t j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (64u + j + 1u)); -} -// dataset[w] without the dataset: derive item w >> 4 and take word w & 15. -IGNEUM_HD uint32_t mh_word(const uint32_t* cache, uint32_t w) { uint32_t s[16]; mh_item(cache, w >> 4u, s); return s[w & 15u]; } diff --git a/proto-cuda/packs-ca4/mx8_mm512/memhard.metal b/proto-cuda/packs-ca4/mx8_mm512/memhard.metal deleted file mode 100644 index 01b263d6d..000000000 --- a/proto-cuda/packs-ca4/mx8_mm512/memhard.metal +++ /dev/null @@ -1,107 +0,0 @@ -#include -using namespace metal; -// Memory-hard dataset core (MEMHARD.md). Cache: 2^26 words in 2^16 segments of 64 chained ChaCha12 lines. -// Item: 8 rounds of 8 x seed-parameterised mixer + one 64-byte cache read, then 8 x final mixer (class v3, mixer multiplier 8, -// docs/plans/mixer-x4.md: the round key of application j of round r is 0x9E3779B9 * (r * m + j + 1)). All parameters are literals. -#define MH_CACHE_LINE_MASK 0x003fffffu -#define MH_SEGMENT_LINES 64u -#define MH_QR(a, b, c, d, r1, r2, r3, r4) { a += b; d ^= a; d = mh_rotl(d, r1); c += d; b ^= c; b = mh_rotl(b, r2); a += b; d ^= a; d = mh_rotl(d, r3); c += d; b ^= c; b = mh_rotl(b, r4); } -inline uint mh_rotl(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 at every call site - -// y = ChaCha12 core(x) + x -inline void mh_chacha_block(const thread uint* x, thread uint* y) { - for (uint i = 0u; i < 16u; ++i) y[i] = x[i]; - for (uint r = 0u; r < 6u; ++r) { - MH_QR(y[0], y[4], y[8], y[12], 16u, 12u, 8u, 7u) MH_QR(y[1], y[5], y[9], y[13], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[6], y[10], y[14], 16u, 12u, 8u, 7u) MH_QR(y[3], y[7], y[11], y[15], 16u, 12u, 8u, 7u) - MH_QR(y[0], y[5], y[10], y[15], 16u, 12u, 8u, 7u) MH_QR(y[1], y[6], y[11], y[12], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[7], y[8], y[13], 16u, 12u, 8u, 7u) MH_QR(y[3], y[4], y[9], y[14], 16u, 12u, 8u, 7u) - } - for (uint i = 0u; i < 16u; ++i) y[i] += x[i]; -} - -// One cache segment: 64 chained lines written at cache[seg * 1024]. in_j = prev ^ (sigma || K || seg || j || tag), prev_0 = 0. -inline void mh_cache_segment(device uint* cache, uint seg) { - uint prev[16]; uint x[16]; uint y[16]; - for (uint i = 0u; i < 16u; ++i) prev[i] = 0u; - for (uint j = 0u; j < MH_SEGMENT_LINES; ++j) { - x[0] = 0x61707865u ^ prev[0]; x[1] = 0x3320646eu ^ prev[1]; x[2] = 0x79622d32u ^ prev[2]; x[3] = 0x6b206574u ^ prev[3]; - x[4] = 0x3067619fu ^ prev[4]; - x[5] = 0x3c269176u ^ prev[5]; - x[6] = 0x84a03b03u ^ prev[6]; - x[7] = 0xf8c63294u ^ prev[7]; - x[8] = 0xff977c5bu ^ prev[8]; - x[9] = 0xe60def3eu ^ prev[9]; - x[10] = 0x63630141u ^ prev[10]; - x[11] = 0xb8fbcb58u ^ prev[11]; - x[12] = seg ^ prev[12]; x[13] = j ^ prev[13]; x[14] = 0x49676e65u ^ prev[14]; x[15] = 0x756d4d48u ^ prev[15]; - mh_chacha_block(x, y); - device uint* line = cache + ((seg * MH_SEGMENT_LINES + j) * 16u); - for (uint i = 0u; i < 16u; ++i) { line[i] = y[i]; prev[i] = y[i]; } - } -} - -// M_r: per word (s ^ (RC + rk)) * MUL, then a column round and a diagonal round with the seed-drawn rotations. -inline void mh_mixer(thread uint* s, uint rk) { - s[0] = (s[0] ^ (0xbab68293u + rk)) * 0x42146205u; - s[1] = (s[1] ^ (0xcc162340u + rk)) * 0x52cbe0fbu; - s[2] = (s[2] ^ (0x6ce151ccu + rk)) * 0x7ecf4a03u; - s[3] = (s[3] ^ (0xe62b8997u + rk)) * 0x6728907fu; - s[4] = (s[4] ^ (0xc9c80297u + rk)) * 0xd81d9751u; - s[5] = (s[5] ^ (0xf74a1654u + rk)) * 0x132952c3u; - s[6] = (s[6] ^ (0x3d704af5u + rk)) * 0xf60de277u; - s[7] = (s[7] ^ (0x3cf522b7u + rk)) * 0x05358035u; - s[8] = (s[8] ^ (0x2b9cac04u + rk)) * 0xbaf6499du; - s[9] = (s[9] ^ (0xa880ac10u + rk)) * 0xe4db9667u; - s[10] = (s[10] ^ (0x13e5dd1du + rk)) * 0x3e98f45du; - s[11] = (s[11] ^ (0x6fc3e233u + rk)) * 0xd0004eddu; - s[12] = (s[12] ^ (0x2d83eeacu + rk)) * 0x2691630du; - s[13] = (s[13] ^ (0x9006e8bfu + rk)) * 0x9beb3bcfu; - s[14] = (s[14] ^ (0x2c4b5362u + rk)) * 0xab310379u; - s[15] = (s[15] ^ (0x31b49ee2u + rk)) * 0x99cfb423u; - MH_QR(s[0], s[4], s[8], s[12], 20u, 20u, 19u, 4u) MH_QR(s[1], s[5], s[9], s[13], 20u, 20u, 19u, 4u) - MH_QR(s[2], s[6], s[10], s[14], 20u, 20u, 19u, 4u) MH_QR(s[3], s[7], s[11], s[15], 20u, 20u, 19u, 4u) - MH_QR(s[0], s[5], s[10], s[15], 26u, 3u, 3u, 27u) MH_QR(s[1], s[6], s[11], s[12], 26u, 3u, 3u, 27u) - MH_QR(s[2], s[7], s[8], s[13], 26u, 3u, 3u, 27u) MH_QR(s[3], s[4], s[9], s[14], 26u, 3u, 3u, 27u) -} - -// Item t: 16 words. s = (K, t * MUL[i] + RC[i]); 8 rounds of 8 x mixer + cache line s[0] & mask; 8 x final mixer. -inline void mh_item(device const uint* cache, uint t, thread uint* s) { - s[0] = 0x3067619fu; - s[1] = 0x3c269176u; - s[2] = 0x84a03b03u; - s[3] = 0xf8c63294u; - s[4] = 0xff977c5bu; - s[5] = 0xe60def3eu; - s[6] = 0x63630141u; - s[7] = 0xb8fbcb58u; - s[8] = t * 0x42146205u + 0xbab68293u; - s[9] = t * 0x52cbe0fbu + 0xcc162340u; - s[10] = t * 0x7ecf4a03u + 0x6ce151ccu; - s[11] = t * 0x6728907fu + 0xe62b8997u; - s[12] = t * 0xd81d9751u + 0xc9c80297u; - s[13] = t * 0x132952c3u + 0xf74a1654u; - s[14] = t * 0xf60de277u + 0x3d704af5u; - s[15] = t * 0x05358035u + 0x3cf522b7u; - for (uint r = 0u; r < 8u; ++r) { - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (r * 8u + j + 1u)); - device const uint* line = cache + ((s[0] & MH_CACHE_LINE_MASK) * 16u); - for (uint i = 0u; i < 16u; ++i) s[i] ^= line[i]; - } - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (64u + j + 1u)); -} -// dataset[w] without the dataset: derive item w >> 4 and take word w & 15. -inline uint mh_word(device const uint* cache, uint w) { uint s[16]; mh_item(cache, w >> 4u, s); return s[w & 15u]; } - -// One thread per segment (2^16 threads). -kernel void igneum_cache_fill(device uint* cache [[buffer(0)]], uint gid [[thread_position_in_grid]]) { - mh_cache_segment(cache, gid); -} -// One thread per 64-byte item (dataset words / 16 threads). -kernel void igneum_build(device const uint* cache [[buffer(0)]], device uint* dataset [[buffer(1)]], - uint gid [[thread_position_in_grid]]) { - uint s[16]; - mh_item(cache, gid, s); - device uint* d = dataset + gid * 16u; - for (uint i = 0u; i < 16u; ++i) d[i] = s[i]; -} diff --git a/proto-cuda/packs-ca4/mx8_mm512/program.h b/proto-cuda/packs-ca4/mx8_mm512/program.h deleted file mode 100644 index e78095175..000000000 --- a/proto-cuda/packs-ca4/mx8_mm512/program.h +++ /dev/null @@ -1,73 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// Program metadata for host.cu plus the launch wrappers defined in kernel.cu. -// Also included by proto-opencl/host.c (C99), which defines IGNEUM_NO_CUDA first and reads only the macros. -#pragma once -#ifdef __cplusplus -#include -#else -#include -#endif -#ifndef IGNEUM_NO_CUDA -#include -#endif - -#define IGNEUM_SEED_STRING "igneum-genesis" -#define IGNEUM_SEED_BYTES_HEX "69676e65756d2d67656e65736973" -#define IGNEUM_GENERATOR 2 -#define IGNEUM_PROGRAM_ATTEMPT 0 -#define IGNEUM_PROGRAM_ID 0xd5492e93169cad0bull -#define IGNEUM_DAY_STRING "2026-10-03" -#define IGNEUM_DAY_BYTES_HEX "6461792f323032362d31302d3033" -#define IGNEUM_DAY0 0x3067619fu -#define IGNEUM_DAY1 0x3c269176u -#define IGNEUM_DATASET_LOG2 28 -#define IGNEUM_MASK 0x0fffffffu -#define IGNEUM_LANES 32 -#define IGNEUM_ITERATIONS 8 -#define IGNEUM_INSTR_COUNT 64 -#define IGNEUM_LOADS_PER_HASH 128 -#define IGNEUM_WIDE_LOADS_PER_HASH 0 -#define IGNEUM_OP_MIX "load=16 add=8 shfl=8 xor=6 mad=5 mul=5 mulhi=5 sub=4 rotl=3 rotr=3 or=1" -// Class v3 construction (Counter ASIC 2.0, 5 October 2026, docs/plans/mixer-x4.md): version 2 loads; the dataset item -// derivation applies the mixer IGNEUM_MIXER_MULT times per round (memhard.h), and the cache follows the growth rule. -#define IGNEUM_LOAD_CLASS "mx8+mm512" -#define IGNEUM_CLASS_MIXER_MULT 8 -#define IGNEUM_CACHE_GROWTH 1 // 1: cache words = 2^(26 + doublings(day)), doublings = floor(log2(1 + day / 1460)) -#define IGNEUM_LOAD_SLOTS 16 -#define IGNEUM_LOAD_MIX { 100, 0, 0 } -#define IGNEUM_LOAD_WIDTH_COUNTS { 16, 0, 0 } // loads of 4, 16, 64 bytes per program -#define IGNEUM_BYTES_PER_HASH 512 -#define IGNEUM_FOLD_ROT 11 -#define IGNEUM_FOLD_MUL 0x9e3779b1u -// Latency-shadow block (Counter ASIC 3.0 item 8, docs/analysis/latency-shadow-2026-10-06.md): NOT the lottery hash. A block of -// IGNEUM_SHADOW_INSTRS ALU instructions (the ten non-load families) runs IGNEUM_SHADOW_REPS times at the end of every -// iteration; the 16 loads, the acceptance rule and the base program are the class's without the shadow. -#define IGNEUM_SHADOW_INSTRS 0 -#define IGNEUM_SHADOW_REPS 0 -#define IGNEUM_SHADOW_INSTRS_PER_HASH 0 -#define IGNEUM_SHADOW_OP_MIX "" -// Counter ASIC 4.0 research (experimental): IGNEUM_MM8_TILES int8 mma.m8n8k16 u8 tiles per iteration after the shadow block. -#define IGNEUM_MM8_TILES 512 -#define IGNEUM_MM8_TILES_PER_HASH 4096 -// 0 = closed-form dataset (ds_elem), 1 = memory-hard cache construction (MEMHARD.md, memhard.h) -#define IGNEUM_DATASET_MODE 1 - -#define IGNEUM_SEEDW_INIT { 0x67a9a7beu, 0x1a155b25u, 0xfddfb732u, 0x4b5af2e8u, 0xc55caf33u, 0xa27c13b7u, 0x06628a48u, 0x03852469u } -#define IGNEUM_KEY_INIT { 0x3067619fu, 0x3c269176u, 0x84a03b03u, 0xf8c63294u, 0xff977c5bu, 0xe60def3eu, 0x63630141u, 0xb8fbcb58u } -#define IGNEUM_CACHE_LOG2_WORDS 26 -#define IGNEUM_CACHE_SEGMENT_LOG2_LINES 6 -#define IGNEUM_CACHE_SEGMENTS 65536u -#define IGNEUM_ITEM_ROUNDS 8 -#define IGNEUM_MIXER_MULT 8 // mixer applications per round and after the last read (class v3, docs/plans/mixer-x4.md) -#define IGNEUM_MIX_ROT_INIT { 20u, 20u, 19u, 4u, 26u, 3u, 3u, 27u } -#define IGNEUM_MIX_MUL_INIT { 0x42146205u, 0x52cbe0fbu, 0x7ecf4a03u, 0x6728907fu, 0xd81d9751u, 0x132952c3u, 0xf60de277u, 0x05358035u, 0xbaf6499du, 0xe4db9667u, 0x3e98f45du, 0xd0004eddu, 0x2691630du, 0x9beb3bcfu, 0xab310379u, 0x99cfb423u } -#define IGNEUM_MIX_RC_INIT { 0xbab68293u, 0xcc162340u, 0x6ce151ccu, 0xe62b8997u, 0xc9c80297u, 0xf74a1654u, 0x3d704af5u, 0x3cf522b7u, 0x2b9cac04u, 0xa880ac10u, 0x13e5dd1du, 0x6fc3e233u, 0x2d83eeacu, 0x9006e8bfu, 0x2c4b5362u, 0x31b49ee2u } - -#ifndef IGNEUM_NO_CUDA -// Defined in kernel.cu. All launch on the default stream and return cudaGetLastError(). -cudaError_t igneum_launch_cache_fill(uint32_t* cache, uint32_t nSegments); -cudaError_t igneum_launch_build(uint32_t* ds, const uint32_t* cache, uint32_t nItems); -cudaError_t igneum_launch_hash(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask, - uint32_t nonces, uint32_t blockWarps); -cudaError_t igneum_hash_info(int* numRegs, int* blocksPerSM, uint32_t blockWarps); -#endif diff --git a/proto-cuda/packs-ca4/mx8_mm512/program.json b/proto-cuda/packs-ca4/mx8_mm512/program.json deleted file mode 100644 index 78be02895..000000000 --- a/proto-cuda/packs-ca4/mx8_mm512/program.json +++ /dev/null @@ -1,134 +0,0 @@ -{ - "format": "igneum-program-pack-3", - "generator": 2, - "attempt": 0, - "program_id": "0xd5492e93169cad0b", - "program_id_derivation": "FNV-1a 64 over 'igneum-program/' || generator_le32 || seed_words as little-endian bytes || attempt_le32", - "dataset_mode": "memory-hard", - "seed": "igneum-genesis", - "seed_bytes": "69676e65756d2d67656e65736973", - "seed_words": ["0x67a9a7be", "0x1a155b25", "0xfddfb732", "0x4b5af2e8", "0xc55caf33", "0xa27c13b7", "0x06628a48", "0x03852469"], - "seed_derivation": "seed_words = FNV-1a 64 over seed_bytes (attempt 0) or seed_bytes || attempt_le32 (attempt k >= 1), basis ^ (salt * 0x9E3779B97F4A7C15) for salt 0..3, then h ^= h>>33; h *= 0xff51afd7ed558ccd; h ^= h>>33; words[2*salt] = low 32, words[2*salt+1] = high 32", - "generator_rule": "version 2: exactly 16 load slots drawn first from instructions 1..63 (partial Fisher-Yates), the other 48 ops from the ten non-load weights (sum 75); a load's source is drawn from the registers other than dst written by an earlier instruction and not read by a load since; the candidate must pass the acceptance rule of spec 01 section 1.4.6 (static: no cyclically stale load source, every register has an injecting write; dynamic: 64 units on the seed-keyed closed-form dataset with no constant register bit, no lane-constant load site, under 164 saturated final values, every output bit within 136 of 1024, distinct addresses above 245760), else the next attempt of the seed is tried", - "lanes": 32, - "registers": 8, - "iterations": 8, - "instruction_count": 64, - "loads_per_hash": 128, - "load_class": "mx8+mm512", - "mixer_mult": 8, - "cache_growth": true, - "mixer": "class v3 (Counter ASIC 2.0, 5 October 2026, docs/plans/mixer-x4.md): every mixer application of the item derivation is 8 applications with round keys (r * 8 + j + 1) * 0x9E3779B9, the 8 dependent cache reads per item unchanged; cache growth rule option C: cache words = 2^(26 + doublings(day)), dataset words = 2^(genesis_log2 + doublings(day)), doublings(day) = floor(log2(1 + day / 1460)) for day = days since genesis", - "load_slots": 16, - "load_mix_percent_4_16_64": [100, 0, 0], - "load_width_counts_4_16_64": [16, 0, 0], - "bytes_per_hash": 512, - "wide_load": "read-width experiment (5 October 2026, docs/plans/read-width.md), NOT the lottery hash: a load of W words (width field, 4 or 16) reads dataset[b .. b + W) with b = (src & mask) & ~(W - 1) and folds every word into dst: x = dst ^ w[0]; for j in 1..W: x = (rotl(x, 11) * 0x9e3779b1) ^ w[j]; dst = x; width 1 is the plain load; the width is drawn per instruction from the class mix with one extra below(100) draw after the nine of version 2, and the program id is FNV-1a 64 over 'igneum-program-rw/' || generator_le32 || seed words || attempt_le32 || mix[3] || load_slots", - "op_mix": {"load": 16, "add": 8, "shfl": 8, "xor": 6, "mad": 5, "mul": 5, "mulhi": 5, "sub": 4, "rotl": 3, "rotr": 3, "or": 1}, - "register_init": "for i in 0..7: x = nonce ^ seed_words[i]; x += 0x9e3779b9 * (i+1) (mod 2^32); x = splitmix32(x); r[i] = x ^ seed_words[(i+1) & 7]", - "splitmix32": "x ^= x>>16; x *= 0x7feb352d; x ^= x>>15; x *= 0x846ca68b; x ^= x>>16", - "iteration": "sel = r0 sampled once at the top of each iteration, then all instructions in order", - "output": "lo = r0 ^ rotl(r1,7) ^ rotl(r2,14) ^ rotl(r3,21); hi = r4 ^ rotl(r5,9) ^ rotl(r6,18) ^ rotl(r7,27); out = (hi << 32) | lo", - "op_semantics": { - "add": "dst = dst + src + (bit `bit` of sel ? imm2 : imm)", - "sub": "dst = dst - src", - "mul": "dst = dst * src (low 32)", - "mulhi": "dst = high 32 bits of dst * src", - "xor": "dst = dst ^ src", - "or": "dst = dst | src", - "rotl": "dst = rotl(dst, rot), rot in 1..31", - "rotr": "dst = rotr(dst, src & 31)", - "mad": "dst = src * src2 + dst", - "shfl": "dst = dst ^ (src of lane (lane ^ mask)), mask in {1,2,4,8,16}, within the 32-lane warp", - "load": "dst = dst ^ dataset[src & dataset.mask]", - "wload": "base = (src of lane 0 & dataset.mask) & ~31; dst = dst ^ dataset[base + lane] (warp-coalesced 128-byte load, lever b, only when --wide-frac > 0)" - }, - "dataset": { - "log2_words": 28, - "bytes": 1073741824, - "mask": "0x0fffffff", - "day": "2026-10-03", - "day_bytes": "6461792f323032362d31302d3033", - "day_words_from": "seed_words_from_bytes(day_bytes)", - "d0": "0x3067619f", - "d1": "0x3c269176", - "mode": "memory-hard", - "spec": "proto-metal/MEMHARD.md", - "key": ["0x3067619f", "0x3c269176", "0x84a03b03", "0xf8c63294", "0xff977c5b", "0xe60def3e", "0x63630141", "0xb8fbcb58"], - "key_derivation": "the 8 words of seed_words_from_bytes(day_bytes); d0, d1 are key[0], key[1]", - "cache": {"log2_words": 26, "bytes": 268435456, "line_words": 16, "segment_lines": 64, "segments": 65536, "block": "ChaCha12 core + feed-forward, rotations 16 12 8 7", "sigma": ["0x61707865", "0x3320646e", "0x79622d32", "0x6b206574"], "tag": ["0x49676e65", "0x756d4d48"], "chain": "in_j = prev_line ^ (sigma[0..3] || key[0..7] || seg || j || tag[0..1]); line_j = block(in_j); prev_0 = 0"}, - "mixer": {"draw": "SplitMix64 seeded with key[0] | key[1] << 32: rot[0..7] = 1 + next() % 31, mul[0..15] = low32(next()) | 1, rc[0..15] = low32(next())", "rot": [20, 20, 19, 4, 26, 3, 3, 27], "mul": ["0x42146205", "0x52cbe0fb", "0x7ecf4a03", "0x6728907f", "0xd81d9751", "0x132952c3", "0xf60de277", "0x05358035", "0xbaf6499d", "0xe4db9667", "0x3e98f45d", "0xd0004edd", "0x2691630d", "0x9beb3bcf", "0xab310379", "0x99cfb423"], "rc": ["0xbab68293", "0xcc162340", "0x6ce151cc", "0xe62b8997", "0xc9c80297", "0xf74a1654", "0x3d704af5", "0x3cf522b7", "0x2b9cac04", "0xa880ac10", "0x13e5dd1d", "0x6fc3e233", "0x2d83eeac", "0x9006e8bf", "0x2c4b5362", "0x31b49ee2"], "round": "for i in 0..15: s[i] = (s[i] ^ (rc[i] + (r+1) * 0x9E3779B9)) * mul[i]; then quarter rounds on columns (0,4,8,12) (1,5,9,13) (2,6,10,14) (3,7,11,15) with rot[0..3] and diagonals (0,5,10,15) (1,6,11,12) (2,7,8,13) (3,4,9,14) with rot[4..7]", "quarter_round": "a += b; d ^= a; d = rotl(d, r1); c += d; b ^= c; b = rotl(b, r2); a += b; d ^= a; d = rotl(d, r3); c += d; b ^= c; b = rotl(b, r4)"}, - "mixer_mult": 8, - "item": "s[0..7] = key; s[8+i] = t * mul[i] + rc[i] for i in 0..7; for r in 0..7: for j in 0..7: s = M(s, rk = (r * 8 + j + 1) * 0x9E3779B9); line = s[0] & 0x003fffff; s[i] ^= cache[line * 16 + i]; then for j in 0..7: s = M(s, rk = (64 + j + 1) * 0x9E3779B9); item(t) = s", - "word": "dataset[w] = item(w >> 4)[w & 15]" - }, - "shadow": {"instrs": 0, "reps": 0, "instrs_per_hash": 0, "op_mix": {}, "rule": "Counter ASIC 3.0 item 8 (docs/analysis/latency-shadow-2026-10-06.md): after the 64 base instructions the program stream draws instrs more ALU instructions (the non-load table, nine draws each, the source as on an ALU slot); the block runs reps times at the end of every iteration with the iteration's sel; the acceptance rule interprets the base program only", "program_id_suffix": "'shadow/' || instrs_le16 || reps_le16", "instructions": [ - ]}, - "mm8_tiles": {"tiles": 512, "tiles_per_hash": 4096, "rule": "Counter ASIC 4.0 research (docs/analysis/counter-asic-4-research.md 15.2): after the shadow draws the stream draws tiles descriptors a, b, c (below 8 each) and c2 (below 7, skipping c); each tile is the PTX mma.m8n8k16 u8 product of the 32 lanes' r[a] (8 x 16 bytes) and r[b] (16 x 8), lane l adds C[l >> 2][2 (l & 3)] into r[c] and C[l >> 2][2 (l & 3) + 1] into r[c2] modulo 2^32; the block runs once per iteration after the shadow", "program_id_suffix": "'mm8/' || tiles_le16", "descriptors": [[6,5,2,3],[1,1,4,5],[2,5,0,7],[1,5,2,3],[2,0,2,6],[2,6,7,3],[1,4,1,3],[4,6,0,4],[1,1,1,5],[3,7,4,6],[3,0,0,4],[2,3,0,6],[3,1,6,0],[1,5,2,5],[3,3,3,4],[1,0,5,4],[7,2,0,4],[0,5,1,7],[1,1,2,0],[3,3,2,4],[3,0,6,1],[0,3,6,0],[1,7,6,4],[1,2,0,3],[3,4,0,2],[5,0,3,7],[1,0,7,3],[3,2,0,5],[2,4,6,5],[3,4,1,5],[6,5,6,3],[1,7,4,3],[7,6,3,0],[7,1,4,6],[1,0,5,1],[2,4,1,3],[1,7,7,1],[1,7,0,2],[3,6,2,4],[6,2,2,5],[6,1,4,1],[6,0,6,2],[7,4,5,1],[6,1,6,0],[0,6,4,1],[6,5,6,5],[6,6,1,0],[7,3,3,1],[7,7,6,3],[4,5,1,7],[1,7,4,5],[3,4,0,3],[2,5,7,4],[7,7,1,3],[7,6,6,5],[1,7,3,7],[4,0,7,3],[4,3,6,7],[2,3,6,0],[6,7,5,7],[6,0,2,4],[2,1,2,6],[4,6,1,6],[1,6,0,3],[0,5,5,3],[0,1,2,4],[3,5,1,3],[6,2,3,4],[7,1,1,2],[6,7,5,4],[1,7,1,5],[3,0,1,0],[4,0,3,6],[4,1,0,2],[5,1,3,4],[4,4,7,3],[5,6,1,7],[0,6,3,2],[2,5,0,5],[7,4,4,1],[6,7,3,4],[4,3,1,6],[6,0,1,7],[2,2,1,6],[0,6,0,3],[1,3,4,1],[3,4,2,3],[4,4,3,7],[6,4,5,3],[4,2,2,1],[6,5,3,6],[6,5,5,4],[2,6,2,3],[2,2,5,7],[2,4,7,3],[5,4,1,6],[0,6,3,1],[0,5,1,2],[2,7,3,2],[1,4,0,2],[2,4,2,5],[2,4,6,7],[5,7,6,3],[4,7,4,3],[2,0,3,7],[6,1,0,5],[4,1,0,5],[2,1,4,2],[4,6,1,2],[2,7,5,3],[1,0,0,2],[6,5,4,6],[5,7,4,2],[6,3,4,1],[2,1,7,5],[1,4,1,4],[7,2,5,0],[7,6,7,6],[0,1,2,1],[4,3,5,3],[5,3,7,2],[2,7,2,1],[0,1,3,0],[5,2,3,5],[5,1,2,5],[7,2,7,2],[6,3,3,2],[6,7,5,1],[6,7,4,1],[3,2,6,3],[0,4,2,1],[3,5,3,1],[0,7,0,6],[2,6,0,1],[5,6,3,1],[0,2,4,5],[3,1,1,5],[7,0,4,6],[2,2,4,2],[0,4,2,7],[4,6,1,6],[5,5,1,2],[2,2,6,2],[4,3,7,1],[2,0,1,7],[7,2,6,5],[2,7,2,7],[3,1,3,0],[2,1,0,6],[4,6,7,1],[3,6,7,0],[1,4,2,4],[3,1,5,1],[0,3,2,3],[7,3,0,4],[1,3,7,5],[6,3,2,3],[6,4,4,5],[6,5,0,4],[1,0,5,2],[0,1,5,7],[4,0,7,0],[6,6,6,1],[5,3,7,3],[0,7,3,2],[6,5,1,5],[6,5,4,0],[4,5,3,7],[3,6,0,7],[6,2,4,3],[6,5,0,4],[6,3,2,3],[7,3,5,2],[0,0,2,4],[6,6,5,6],[1,6,3,4],[2,4,7,5],[1,7,7,2],[4,2,7,3],[5,2,5,1],[0,4,2,1],[3,5,2,4],[1,1,4,3],[3,1,0,7],[7,4,0,7],[5,0,6,3],[2,5,7,6],[3,0,0,7],[1,4,5,0],[7,2,5,4],[1,5,2,0],[2,1,2,5],[5,0,2,4],[2,1,4,6],[4,1,0,5],[2,7,2,4],[5,6,2,5],[5,3,6,5],[1,0,7,1],[2,5,0,4],[5,7,3,2],[7,2,2,1],[2,5,5,6],[0,5,1,6],[4,7,3,7],[3,7,1,3],[0,6,7,3],[7,7,1,4],[0,1,6,5],[4,4,6,2],[7,1,2,5],[1,5,5,6],[2,4,5,3],[2,2,3,5],[4,6,5,7],[7,3,2,6],[6,2,3,4],[5,7,0,6],[7,1,1,0],[2,1,3,1],[1,7,4,5],[7,1,2,1],[0,6,2,1],[2,2,7,0],[1,1,6,2],[2,7,6,4],[2,1,7,5],[3,5,5,6],[7,1,3,1],[1,4,1,4],[0,3,5,6],[1,3,4,0],[5,4,0,1],[7,2,0,4],[2,0,2,1],[3,4,4,1],[1,7,0,5],[3,5,6,3],[2,0,0,2],[5,7,1,0],[2,7,4,6],[0,1,3,7],[0,2,2,7],[4,4,1,5],[7,6,0,7],[4,4,2,0],[0,6,1,2],[3,6,2,4],[7,5,7,3],[4,4,0,5],[0,6,0,1],[0,7,4,3],[0,2,5,4],[5,5,7,4],[6,7,6,3],[5,0,0,3],[2,1,3,4],[4,3,6,0],[4,1,7,3],[5,5,3,0],[3,2,3,0],[0,6,3,2],[6,4,2,5],[6,4,2,1],[7,2,1,4],[6,1,3,1],[0,0,7,0],[6,3,3,4],[1,0,6,5],[7,7,7,3],[6,2,7,0],[0,3,5,3],[6,0,0,3],[2,4,6,7],[1,0,7,6],[1,3,1,4],[1,5,6,2],[5,1,4,7],[0,0,6,7],[2,4,2,4],[7,2,3,5],[1,0,4,2],[0,1,5,1],[0,2,7,2],[1,5,0,2],[4,7,5,0],[1,0,2,0],[7,1,7,4],[0,7,4,3],[0,5,5,4],[3,2,6,7],[7,4,7,1],[6,7,6,3],[5,5,6,4],[5,7,1,6],[0,7,1,3],[6,1,5,6],[4,5,3,5],[6,0,3,1],[6,0,2,3],[6,4,4,2],[6,4,6,3],[7,6,6,4],[3,2,3,6],[6,5,2,4],[4,5,0,2],[3,1,0,2],[4,3,5,0],[2,1,2,6],[2,1,1,4],[6,1,2,3],[4,6,3,2],[1,3,4,1],[2,6,7,1],[0,3,2,1],[7,1,6,2],[2,4,1,6],[1,6,3,6],[2,1,1,0],[0,2,6,3],[5,2,6,2],[7,4,1,3],[2,4,7,1],[4,3,2,3],[5,7,2,0],[6,2,7,2],[6,1,7,6],[1,5,3,0],[0,1,1,6],[0,3,7,1],[6,7,4,0],[4,4,4,1],[4,4,6,3],[4,6,4,2],[2,2,0,1],[7,1,1,2],[4,0,2,3],[1,4,0,3],[3,7,2,7],[5,1,7,6],[5,0,7,5],[6,6,7,5],[4,7,0,4],[0,4,4,6],[2,5,4,6],[7,4,3,1],[6,0,7,3],[5,3,3,5],[0,4,4,5],[6,1,4,7],[1,5,3,5],[6,4,4,5],[4,1,6,0],[7,3,1,4],[4,1,1,5],[4,5,0,1],[7,6,0,4],[7,3,7,5],[4,7,5,7],[2,7,3,5],[4,4,0,2],[2,2,5,6],[2,7,3,0],[7,6,5,0],[6,4,0,5],[6,6,5,4],[7,7,2,5],[4,1,7,5],[7,0,3,5],[1,0,6,0],[2,5,7,4],[5,4,2,6],[2,7,6,2],[4,6,2,4],[1,5,0,4],[6,5,6,2],[3,6,0,3],[6,4,6,4],[6,3,2,4],[3,0,2,4],[2,0,6,2],[3,5,1,2],[5,2,2,4],[1,2,0,2],[3,1,1,0],[6,0,7,5],[2,0,1,7],[3,7,2,4],[3,0,1,2],[6,4,5,3],[5,4,1,4],[6,3,4,1],[2,3,3,0],[3,6,5,0],[0,7,1,2],[0,7,2,4],[7,7,1,4],[5,7,3,2],[6,5,7,0],[6,3,3,5],[2,7,0,4],[7,6,2,1],[0,0,7,1],[7,4,1,0],[1,1,1,4],[6,6,3,6],[6,1,4,3],[5,7,2,4],[7,4,4,1],[3,4,4,7],[0,5,5,2],[2,0,5,1],[7,4,3,1],[2,7,6,2],[7,1,0,3],[4,7,5,1],[5,4,3,5],[6,6,1,2],[1,3,1,5],[5,0,0,6],[4,2,7,6],[4,4,3,7],[0,7,4,3],[1,7,6,0],[5,1,6,5],[2,7,6,5],[1,6,5,0],[6,6,6,5],[6,2,7,4],[3,3,7,0],[7,7,3,5],[6,4,1,0],[3,2,3,4],[3,0,5,0],[0,7,5,3],[7,5,6,3],[6,4,0,5],[0,7,3,4],[5,1,4,7],[4,7,3,2],[3,5,3,2],[5,1,1,4],[1,2,7,5],[1,7,5,2],[3,7,1,0],[7,5,7,0],[3,1,2,1],[6,7,4,7],[5,2,4,5],[7,1,1,3],[7,5,7,3],[0,3,1,7],[2,4,7,1],[5,7,7,1],[7,7,7,1],[7,1,7,5],[6,5,5,6],[7,7,1,5],[5,1,4,1],[6,7,1,0],[7,7,6,5],[0,6,6,2],[0,6,1,2],[2,0,3,2],[0,3,0,1],[1,7,0,5],[1,2,4,1],[1,0,1,6],[4,1,6,2],[2,0,0,1],[3,1,2,0],[2,2,7,1],[1,7,6,5],[2,7,6,1],[5,3,6,3],[7,6,6,0],[3,0,5,6],[3,1,4,6],[0,6,2,5],[7,6,7,1],[6,0,1,0],[2,4,6,7],[2,3,0,6],[7,0,7,6],[3,7,6,4],[6,4,1,2],[6,5,2,5],[2,5,4,5],[1,2,1,2],[5,2,6,4],[7,1,2,6],[0,0,0,6],[3,1,6,4],[6,1,7,6],[5,6,3,1],[2,2,7,2],[3,5,5,0],[3,2,2,0],[3,1,6,2],[1,6,1,5],[7,6,1,5],[4,4,1,0],[5,7,7,6],[7,5,3,4],[2,0,4,2],[4,0,0,1],[0,2,7,0],[5,6,4,7],[7,3,4,7],[4,4,6,0],[0,6,6,5],[2,4,1,3]]}, - "instructions": [ - {"i": 0, "op": "mad", "dst": 2, "src": 3, "src2": 4, "imm": "0xbf7b174d", "imm2": "0x337b762e", "rot": 17, "bit": 2, "mask": 2, "width": 1}, - {"i": 1, "op": "mad", "dst": 2, "src": 1, "src2": 1, "imm": "0xdd04a5da", "imm2": "0x42da7657", "rot": 15, "bit": 30, "mask": 16, "width": 1}, - {"i": 2, "op": "mad", "dst": 2, "src": 3, "src2": 2, "imm": "0x734003fa", "imm2": "0x5bb67700", "rot": 3, "bit": 20, "mask": 1, "width": 1}, - {"i": 3, "op": "xor", "dst": 3, "src": 5, "src2": 5, "imm": "0xc55a1b1c", "imm2": "0xa19720f3", "rot": 7, "bit": 8, "mask": 1, "width": 1}, - {"i": 4, "op": "load", "dst": 7, "src": 2, "src2": 5, "imm": "0xad572dd7", "imm2": "0x9ceb3ea7", "rot": 18, "bit": 30, "mask": 2, "width": 1}, - {"i": 5, "op": "load", "dst": 5, "src": 7, "src2": 3, "imm": "0x769a53be", "imm2": "0x80f9067e", "rot": 12, "bit": 22, "mask": 1, "width": 1}, - {"i": 6, "op": "shfl", "dst": 1, "src": 4, "src2": 0, "imm": "0xd3613d88", "imm2": "0x262fb219", "rot": 10, "bit": 30, "mask": 8, "width": 1}, - {"i": 7, "op": "shfl", "dst": 7, "src": 3, "src2": 4, "imm": "0xce38e42f", "imm2": "0xb868b818", "rot": 11, "bit": 8, "mask": 8, "width": 1}, - {"i": 8, "op": "mulhi", "dst": 1, "src": 5, "src2": 2, "imm": "0xa5eebca5", "imm2": "0x5703a72b", "rot": 13, "bit": 13, "mask": 16, "width": 1}, - {"i": 9, "op": "rotr", "dst": 6, "src": 3, "src2": 4, "imm": "0x17a5a9c7", "imm2": "0xdcfb93a1", "rot": 20, "bit": 27, "mask": 2, "width": 1}, - {"i": 10, "op": "or", "dst": 3, "src": 4, "src2": 1, "imm": "0xccb7d785", "imm2": "0xc335364c", "rot": 14, "bit": 12, "mask": 4, "width": 1}, - {"i": 11, "op": "load", "dst": 4, "src": 3, "src2": 2, "imm": "0x88cb9af3", "imm2": "0x4e7dc10d", "rot": 24, "bit": 17, "mask": 4, "width": 1}, - {"i": 12, "op": "mulhi", "dst": 0, "src": 4, "src2": 4, "imm": "0x45374321", "imm2": "0x3cd91989", "rot": 11, "bit": 4, "mask": 2, "width": 1}, - {"i": 13, "op": "add", "dst": 5, "src": 1, "src2": 2, "imm": "0xc7934706", "imm2": "0xd3177981", "rot": 16, "bit": 30, "mask": 2, "width": 1}, - {"i": 14, "op": "load", "dst": 0, "src": 4, "src2": 3, "imm": "0xfd7f56bb", "imm2": "0x65e14f52", "rot": 13, "bit": 22, "mask": 2, "width": 1}, - {"i": 15, "op": "sub", "dst": 2, "src": 4, "src2": 4, "imm": "0x35a80b49", "imm2": "0x060f2d13", "rot": 16, "bit": 20, "mask": 16, "width": 1}, - {"i": 16, "op": "load", "dst": 2, "src": 0, "src2": 2, "imm": "0xae0a32c2", "imm2": "0x4c2a4cfe", "rot": 8, "bit": 31, "mask": 16, "width": 1}, - {"i": 17, "op": "load", "dst": 7, "src": 2, "src2": 6, "imm": "0x82a84cc3", "imm2": "0x21a38d68", "rot": 15, "bit": 21, "mask": 2, "width": 1}, - {"i": 18, "op": "shfl", "dst": 7, "src": 3, "src2": 3, "imm": "0xa3818806", "imm2": "0x8f66b5c8", "rot": 14, "bit": 6, "mask": 4, "width": 1}, - {"i": 19, "op": "mul", "dst": 5, "src": 0, "src2": 1, "imm": "0xa00de107", "imm2": "0x77bfcaa5", "rot": 3, "bit": 10, "mask": 2, "width": 1}, - {"i": 20, "op": "shfl", "dst": 3, "src": 4, "src2": 7, "imm": "0x1d2b8cab", "imm2": "0x80b4f9a2", "rot": 14, "bit": 25, "mask": 2, "width": 1}, - {"i": 21, "op": "shfl", "dst": 2, "src": 4, "src2": 5, "imm": "0x3ac915d2", "imm2": "0x5fba7bc2", "rot": 16, "bit": 1, "mask": 16, "width": 1}, - {"i": 22, "op": "mulhi", "dst": 6, "src": 2, "src2": 0, "imm": "0xdc3ec8fd", "imm2": "0x599e2fa3", "rot": 22, "bit": 3, "mask": 2, "width": 1}, - {"i": 23, "op": "load", "dst": 6, "src": 1, "src2": 5, "imm": "0x2a6b16d5", "imm2": "0xd73e396f", "rot": 28, "bit": 29, "mask": 2, "width": 1}, - {"i": 24, "op": "mul", "dst": 5, "src": 0, "src2": 7, "imm": "0x376d0223", "imm2": "0xe1c2169a", "rot": 4, "bit": 16, "mask": 16, "width": 1}, - {"i": 25, "op": "rotl", "dst": 5, "src": 7, "src2": 3, "imm": "0x78ad8c60", "imm2": "0x6f5b77d5", "rot": 19, "bit": 11, "mask": 16, "width": 1}, - {"i": 26, "op": "shfl", "dst": 7, "src": 6, "src2": 5, "imm": "0x93915b9f", "imm2": "0x1e61fb6b", "rot": 28, "bit": 23, "mask": 2, "width": 1}, - {"i": 27, "op": "xor", "dst": 0, "src": 5, "src2": 7, "imm": "0x6378fe15", "imm2": "0x66c78f42", "rot": 12, "bit": 31, "mask": 8, "width": 1}, - {"i": 28, "op": "xor", "dst": 0, "src": 4, "src2": 7, "imm": "0x20a57fda", "imm2": "0x088c848e", "rot": 16, "bit": 13, "mask": 4, "width": 1}, - {"i": 29, "op": "sub", "dst": 3, "src": 0, "src2": 2, "imm": "0x49d95fd5", "imm2": "0x1a5f946a", "rot": 6, "bit": 12, "mask": 1, "width": 1}, - {"i": 30, "op": "mul", "dst": 5, "src": 1, "src2": 7, "imm": "0x0a816217", "imm2": "0x405c4f73", "rot": 13, "bit": 27, "mask": 4, "width": 1}, - {"i": 31, "op": "load", "dst": 7, "src": 2, "src2": 2, "imm": "0x09ed045e", "imm2": "0xd69c4715", "rot": 5, "bit": 9, "mask": 2, "width": 1}, - {"i": 32, "op": "load", "dst": 1, "src": 0, "src2": 6, "imm": "0xeb79ea49", "imm2": "0xcc587f5a", "rot": 6, "bit": 8, "mask": 16, "width": 1}, - {"i": 33, "op": "xor", "dst": 5, "src": 6, "src2": 1, "imm": "0x3027401e", "imm2": "0x5f20c27e", "rot": 18, "bit": 9, "mask": 2, "width": 1}, - {"i": 34, "op": "load", "dst": 5, "src": 1, "src2": 3, "imm": "0x0e1cab07", "imm2": "0x09356c5b", "rot": 19, "bit": 31, "mask": 1, "width": 1}, - {"i": 35, "op": "mulhi", "dst": 0, "src": 5, "src2": 2, "imm": "0x90e31357", "imm2": "0xabd32484", "rot": 26, "bit": 5, "mask": 8, "width": 1}, - {"i": 36, "op": "shfl", "dst": 5, "src": 2, "src2": 5, "imm": "0xee9a955f", "imm2": "0x31b3faed", "rot": 8, "bit": 24, "mask": 4, "width": 1}, - {"i": 37, "op": "load", "dst": 7, "src": 0, "src2": 7, "imm": "0x3ba2f832", "imm2": "0x1160dcd3", "rot": 4, "bit": 29, "mask": 1, "width": 1}, - {"i": 38, "op": "add", "dst": 3, "src": 1, "src2": 7, "imm": "0x75ba2fad", "imm2": "0x230c005c", "rot": 4, "bit": 27, "mask": 1, "width": 1}, - {"i": 39, "op": "shfl", "dst": 1, "src": 5, "src2": 3, "imm": "0xdbf37e75", "imm2": "0xb5ac1969", "rot": 30, "bit": 13, "mask": 4, "width": 1}, - {"i": 40, "op": "xor", "dst": 2, "src": 5, "src2": 5, "imm": "0x47f136c5", "imm2": "0x06ce9153", "rot": 19, "bit": 10, "mask": 2, "width": 1}, - {"i": 41, "op": "mad", "dst": 3, "src": 6, "src2": 3, "imm": "0xce13eff8", "imm2": "0x04cc1d55", "rot": 3, "bit": 1, "mask": 4, "width": 1}, - {"i": 42, "op": "sub", "dst": 6, "src": 7, "src2": 1, "imm": "0x6a65ab71", "imm2": "0x8fbc1bcd", "rot": 4, "bit": 1, "mask": 8, "width": 1}, - {"i": 43, "op": "xor", "dst": 7, "src": 0, "src2": 7, "imm": "0xdaeb4928", "imm2": "0xc0423027", "rot": 24, "bit": 11, "mask": 8, "width": 1}, - {"i": 44, "op": "load", "dst": 1, "src": 7, "src2": 7, "imm": "0x778f01c9", "imm2": "0x28cedcea", "rot": 12, "bit": 4, "mask": 16, "width": 1}, - {"i": 45, "op": "mul", "dst": 2, "src": 3, "src2": 2, "imm": "0xf4264f1b", "imm2": "0x0f627d56", "rot": 5, "bit": 28, "mask": 8, "width": 1}, - {"i": 46, "op": "mulhi", "dst": 1, "src": 5, "src2": 1, "imm": "0xffb2147a", "imm2": "0xccde9b05", "rot": 13, "bit": 9, "mask": 2, "width": 1}, - {"i": 47, "op": "sub", "dst": 4, "src": 3, "src2": 5, "imm": "0x73b36234", "imm2": "0x3f5d5997", "rot": 7, "bit": 18, "mask": 2, "width": 1}, - {"i": 48, "op": "rotr", "dst": 2, "src": 6, "src2": 3, "imm": "0x3a4d9aa9", "imm2": "0x212bec7b", "rot": 4, "bit": 29, "mask": 16, "width": 1}, - {"i": 49, "op": "load", "dst": 3, "src": 5, "src2": 2, "imm": "0x626f11df", "imm2": "0x56cd5bfd", "rot": 7, "bit": 1, "mask": 1, "width": 1}, - {"i": 50, "op": "add", "dst": 1, "src": 5, "src2": 6, "imm": "0x81b8bc2c", "imm2": "0x1907970c", "rot": 28, "bit": 7, "mask": 4, "width": 1}, - {"i": 51, "op": "mul", "dst": 0, "src": 2, "src2": 2, "imm": "0xa8848b30", "imm2": "0xef6ac348", "rot": 9, "bit": 15, "mask": 8, "width": 1}, - {"i": 52, "op": "add", "dst": 0, "src": 2, "src2": 0, "imm": "0x4f92b968", "imm2": "0x699fd448", "rot": 22, "bit": 6, "mask": 4, "width": 1}, - {"i": 53, "op": "add", "dst": 1, "src": 0, "src2": 2, "imm": "0x2bb965af", "imm2": "0x77b1520d", "rot": 2, "bit": 12, "mask": 8, "width": 1}, - {"i": 54, "op": "rotl", "dst": 7, "src": 1, "src2": 0, "imm": "0x553e678b", "imm2": "0x3cc8eae0", "rot": 14, "bit": 20, "mask": 2, "width": 1}, - {"i": 55, "op": "add", "dst": 3, "src": 7, "src2": 2, "imm": "0x7b0fe07a", "imm2": "0xa54c55a0", "rot": 10, "bit": 1, "mask": 1, "width": 1}, - {"i": 56, "op": "load", "dst": 6, "src": 7, "src2": 4, "imm": "0x01eba9aa", "imm2": "0x2758c0f7", "rot": 14, "bit": 15, "mask": 4, "width": 1}, - {"i": 57, "op": "rotr", "dst": 1, "src": 5, "src2": 2, "imm": "0x1f5267b3", "imm2": "0x236f5a27", "rot": 2, "bit": 31, "mask": 16, "width": 1}, - {"i": 58, "op": "load", "dst": 5, "src": 4, "src2": 3, "imm": "0xa9954a9b", "imm2": "0x6a54d4e8", "rot": 11, "bit": 10, "mask": 16, "width": 1}, - {"i": 59, "op": "load", "dst": 6, "src": 2, "src2": 4, "imm": "0x9923ff88", "imm2": "0x9357254e", "rot": 16, "bit": 1, "mask": 16, "width": 1}, - {"i": 60, "op": "mad", "dst": 3, "src": 5, "src2": 0, "imm": "0xc0cc51a6", "imm2": "0x3fd7701b", "rot": 20, "bit": 1, "mask": 4, "width": 1}, - {"i": 61, "op": "add", "dst": 5, "src": 7, "src2": 7, "imm": "0xaf9dd72d", "imm2": "0xad7493e7", "rot": 7, "bit": 31, "mask": 16, "width": 1}, - {"i": 62, "op": "add", "dst": 4, "src": 6, "src2": 2, "imm": "0x89841d87", "imm2": "0x1e07c3d9", "rot": 6, "bit": 27, "mask": 1, "width": 1}, - {"i": 63, "op": "rotl", "dst": 5, "src": 4, "src2": 2, "imm": "0xae210f8d", "imm2": "0x8e499ba4", "rot": 19, "bit": 9, "mask": 1, "width": 1} - ] -} diff --git a/proto-cuda/packs-ca4/mx8_mm512/program.metal b/proto-cuda/packs-ca4/mx8_mm512/program.metal deleted file mode 100644 index 2628b83cb..000000000 --- a/proto-cuda/packs-ca4/mx8_mm512/program.metal +++ /dev/null @@ -1,641 +0,0 @@ -#include -using namespace metal; -// int8 tile reference (Counter ASIC 4.0 research): A = the 32 lanes' av as 8 x 16 bytes (lane l holds row l >> 2, bytes 4 (l & 3)..), -// B = bv as 16 x 8 (lane l holds column l >> 2); lane l gets C[l >> 2][2 (l & 3)] and C[l >> 2][2 (l & 3) + 1]; the PTX mma.m8n8k16 u8 layout. -static inline void mm8_ref(uint av, uint bv, uint lane, thread uint& d0, thread uint& d1) { - uint row = lane >> 2, col0 = 2u * (lane & 3u); - uint acc0 = 0u, acc1 = 0u; - for (uint j = 0u; j < 4u; ++j) { - uint aw = simd_shuffle(av, (ushort)(4u * row + j)); - uint bw0 = simd_shuffle(bv, (ushort)(4u * col0 + j)); - uint bw1 = simd_shuffle(bv, (ushort)(4u * (col0 + 1u) + j)); - for (uint t = 0u; t < 4u; ++t) { - uint ab = (aw >> (8u * t)) & 0xffu; - acc0 += ab * ((bw0 >> (8u * t)) & 0xffu); - acc1 += ab * ((bw1 >> (8u * t)) & 0xffu); - } - } - d0 = acc0; d1 = acc1; -} - - -#define MASK 0x0fffffffu -constant uint SEEDW[8] = { 0x67a9a7beu, 0x1a155b25u, 0xfddfb732u, 0x4b5af2e8u, 0xc55caf33u, 0xa27c13b7u, 0x06628a48u, 0x03852469u }; - -inline uint splitmix32(uint x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -inline uint rotl_imm(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 -inline uint rotr_var(uint x, uint n) { n &= 31u; return (x >> n) | (x << ((32u - n) & 31u)); } -inline uint ds_elem(uint i, uint d0, uint d1) { - uint x = i ^ d0; - x *= 0x9E3779B1u; x ^= x >> 15; - x += d1; - x *= 0x85EBCA77u; x ^= x >> 13; - x *= 0xC2B2AE3Du; x ^= x >> 16; - return x; -} - -kernel void igneum_hash(device const uint* dataset [[buffer(0)]], - device ulong* out [[buffer(1)]], - constant uint& baseNonce [[buffer(2)]], - uint gid [[thread_position_in_grid]]) { - uint nonce = baseNonce + gid; - uint r0, r1, r2, r3, r4, r5, r6, r7; - uint lane = gid & 31u; - { uint x = nonce ^ SEEDW[0]; x += 0x9e3779b9u * 1u; x = splitmix32(x); r0 = x ^ SEEDW[1]; } - { uint x = nonce ^ SEEDW[1]; x += 0x9e3779b9u * 2u; x = splitmix32(x); r1 = x ^ SEEDW[2]; } - { uint x = nonce ^ SEEDW[2]; x += 0x9e3779b9u * 3u; x = splitmix32(x); r2 = x ^ SEEDW[3]; } - { uint x = nonce ^ SEEDW[3]; x += 0x9e3779b9u * 4u; x = splitmix32(x); r3 = x ^ SEEDW[4]; } - { uint x = nonce ^ SEEDW[4]; x += 0x9e3779b9u * 5u; x = splitmix32(x); r4 = x ^ SEEDW[5]; } - { uint x = nonce ^ SEEDW[5]; x += 0x9e3779b9u * 6u; x = splitmix32(x); r5 = x ^ SEEDW[6]; } - { uint x = nonce ^ SEEDW[6]; x += 0x9e3779b9u * 7u; x = splitmix32(x); r6 = x ^ SEEDW[7]; } - { uint x = nonce ^ SEEDW[7]; x += 0x9e3779b9u * 8u; x = splitmix32(x); r7 = x ^ SEEDW[0]; } - - for (uint it = 0u; it < 8u; ++it) { - uint sel = r0; - r2 = r3 * r4 + r2; // 0 - r2 = r1 * r1 + r2; // 1 - r2 = r3 * r2 + r2; // 2 - r3 = r3 ^ r5; // 3 - r7 = r7 ^ dataset[r2 & MASK]; // 4 - r5 = r5 ^ dataset[r7 & MASK]; // 5 - r1 = r1 ^ simd_shuffle_xor(r4, (ushort)8); // 6 - r7 = r7 ^ simd_shuffle_xor(r3, (ushort)8); // 7 - r1 = mulhi(r1, r5); // 8 - r6 = rotr_var(r6, r3); // 9 - r3 = r3 | r4; // 10 - r4 = r4 ^ dataset[r3 & MASK]; // 11 - r0 = mulhi(r0, r4); // 12 - r5 = r5 + r1 + select(0xc7934706u, 0xd3177981u, ((sel >> 30u) & 1u) != 0u); // 13 - r0 = r0 ^ dataset[r4 & MASK]; // 14 - r2 = r2 - r4; // 15 - r2 = r2 ^ dataset[r0 & MASK]; // 16 - r7 = r7 ^ dataset[r2 & MASK]; // 17 - r7 = r7 ^ simd_shuffle_xor(r3, (ushort)4); // 18 - r5 = r5 * r0; // 19 - r3 = r3 ^ simd_shuffle_xor(r4, (ushort)2); // 20 - r2 = r2 ^ simd_shuffle_xor(r4, (ushort)16); // 21 - r6 = mulhi(r6, r2); // 22 - r6 = r6 ^ dataset[r1 & MASK]; // 23 - r5 = r5 * r0; // 24 - r5 = rotl_imm(r5, 19u); // 25 - r7 = r7 ^ simd_shuffle_xor(r6, (ushort)2); // 26 - r0 = r0 ^ r5; // 27 - r0 = r0 ^ r4; // 28 - r3 = r3 - r0; // 29 - r5 = r5 * r1; // 30 - r7 = r7 ^ dataset[r2 & MASK]; // 31 - r1 = r1 ^ dataset[r0 & MASK]; // 32 - r5 = r5 ^ r6; // 33 - r5 = r5 ^ dataset[r1 & MASK]; // 34 - r0 = mulhi(r0, r5); // 35 - r5 = r5 ^ simd_shuffle_xor(r2, (ushort)4); // 36 - r7 = r7 ^ dataset[r0 & MASK]; // 37 - r3 = r3 + r1 + select(0x75ba2fadu, 0x230c005cu, ((sel >> 27u) & 1u) != 0u); // 38 - r1 = r1 ^ simd_shuffle_xor(r5, (ushort)4); // 39 - r2 = r2 ^ r5; // 40 - r3 = r6 * r3 + r3; // 41 - r6 = r6 - r7; // 42 - r7 = r7 ^ r0; // 43 - r1 = r1 ^ dataset[r7 & MASK]; // 44 - r2 = r2 * r3; // 45 - r1 = mulhi(r1, r5); // 46 - r4 = r4 - r3; // 47 - r2 = rotr_var(r2, r6); // 48 - r3 = r3 ^ dataset[r5 & MASK]; // 49 - r1 = r1 + r5 + select(0x81b8bc2cu, 0x1907970cu, ((sel >> 7u) & 1u) != 0u); // 50 - r0 = r0 * r2; // 51 - r0 = r0 + r2 + select(0x4f92b968u, 0x699fd448u, ((sel >> 6u) & 1u) != 0u); // 52 - r1 = r1 + r0 + select(0x2bb965afu, 0x77b1520du, ((sel >> 12u) & 1u) != 0u); // 53 - r7 = rotl_imm(r7, 14u); // 54 - r3 = r3 + r7 + select(0x7b0fe07au, 0xa54c55a0u, ((sel >> 1u) & 1u) != 0u); // 55 - r6 = r6 ^ dataset[r7 & MASK]; // 56 - r1 = rotr_var(r1, r5); // 57 - r5 = r5 ^ dataset[r4 & MASK]; // 58 - r6 = r6 ^ dataset[r2 & MASK]; // 59 - r3 = r5 * r0 + r3; // 60 - r5 = r5 + r7 + select(0xaf9dd72du, 0xad7493e7u, ((sel >> 31u) & 1u) != 0u); // 61 - r4 = r4 + r6 + select(0x89841d87u, 0x1e07c3d9u, ((sel >> 27u) & 1u) != 0u); // 62 - r5 = rotl_imm(r5, 19u); // 63 - // int8 tile block (Counter ASIC 4.0 research, experimental): 512 mma.m8n8k16 u8 tiles per iteration, both outputs consumed - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t0 mm8 a=r6 b=r5 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t1 mm8 a=r1 b=r1 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t2 mm8 a=r2 b=r5 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t3 mm8 a=r1 b=r5 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t4 mm8 a=r2 b=r0 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t5 mm8 a=r2 b=r6 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t6 mm8 a=r1 b=r4 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t7 mm8 a=r4 b=r6 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t8 mm8 a=r1 b=r1 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t9 mm8 a=r3 b=r7 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t10 mm8 a=r3 b=r0 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t11 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t12 mm8 a=r3 b=r1 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t13 mm8 a=r1 b=r5 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t14 mm8 a=r3 b=r3 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t15 mm8 a=r1 b=r0 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t16 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t17 mm8 a=r0 b=r5 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t18 mm8 a=r1 b=r1 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t19 mm8 a=r3 b=r3 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t20 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t21 mm8 a=r0 b=r3 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t22 mm8 a=r1 b=r7 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t23 mm8 a=r1 b=r2 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t24 mm8 a=r3 b=r4 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t25 mm8 a=r5 b=r0 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t26 mm8 a=r1 b=r0 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t27 mm8 a=r3 b=r2 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t28 mm8 a=r2 b=r4 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t29 mm8 a=r3 b=r4 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t30 mm8 a=r6 b=r5 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t31 mm8 a=r1 b=r7 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t32 mm8 a=r7 b=r6 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t33 mm8 a=r7 b=r1 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t34 mm8 a=r1 b=r0 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t35 mm8 a=r2 b=r4 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t36 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t37 mm8 a=r1 b=r7 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t38 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t39 mm8 a=r6 b=r2 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t40 mm8 a=r6 b=r1 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t41 mm8 a=r6 b=r0 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t42 mm8 a=r7 b=r4 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t43 mm8 a=r6 b=r1 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t44 mm8 a=r0 b=r6 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t45 mm8 a=r6 b=r5 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t46 mm8 a=r6 b=r6 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t47 mm8 a=r7 b=r3 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t48 mm8 a=r7 b=r7 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t49 mm8 a=r4 b=r5 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t50 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t51 mm8 a=r3 b=r4 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t52 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t53 mm8 a=r7 b=r7 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t54 mm8 a=r7 b=r6 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t55 mm8 a=r1 b=r7 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t56 mm8 a=r4 b=r0 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t57 mm8 a=r4 b=r3 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t58 mm8 a=r2 b=r3 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t59 mm8 a=r6 b=r7 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t60 mm8 a=r6 b=r0 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t61 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t62 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t63 mm8 a=r1 b=r6 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t64 mm8 a=r0 b=r5 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t65 mm8 a=r0 b=r1 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t66 mm8 a=r3 b=r5 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t67 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t68 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t69 mm8 a=r6 b=r7 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t70 mm8 a=r1 b=r7 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t71 mm8 a=r3 b=r0 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t72 mm8 a=r4 b=r0 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t73 mm8 a=r4 b=r1 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t74 mm8 a=r5 b=r1 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t75 mm8 a=r4 b=r4 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t76 mm8 a=r5 b=r6 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t77 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t78 mm8 a=r2 b=r5 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t79 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t80 mm8 a=r6 b=r7 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t81 mm8 a=r4 b=r3 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t82 mm8 a=r6 b=r0 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t83 mm8 a=r2 b=r2 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t84 mm8 a=r0 b=r6 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t85 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t86 mm8 a=r3 b=r4 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t87 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t88 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t89 mm8 a=r4 b=r2 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t90 mm8 a=r6 b=r5 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t91 mm8 a=r6 b=r5 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t92 mm8 a=r2 b=r6 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t93 mm8 a=r2 b=r2 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t94 mm8 a=r2 b=r4 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t95 mm8 a=r5 b=r4 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t96 mm8 a=r0 b=r6 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t97 mm8 a=r0 b=r5 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t98 mm8 a=r2 b=r7 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t99 mm8 a=r1 b=r4 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t100 mm8 a=r2 b=r4 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t101 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t102 mm8 a=r5 b=r7 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t103 mm8 a=r4 b=r7 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t104 mm8 a=r2 b=r0 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t105 mm8 a=r6 b=r1 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t106 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t107 mm8 a=r2 b=r1 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t108 mm8 a=r4 b=r6 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t109 mm8 a=r2 b=r7 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t110 mm8 a=r1 b=r0 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t111 mm8 a=r6 b=r5 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t112 mm8 a=r5 b=r7 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t113 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t114 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t115 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t116 mm8 a=r7 b=r2 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t117 mm8 a=r7 b=r6 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t118 mm8 a=r0 b=r1 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t119 mm8 a=r4 b=r3 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t120 mm8 a=r5 b=r3 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t121 mm8 a=r2 b=r7 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t122 mm8 a=r0 b=r1 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t123 mm8 a=r5 b=r2 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t124 mm8 a=r5 b=r1 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t125 mm8 a=r7 b=r2 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t126 mm8 a=r6 b=r3 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t127 mm8 a=r6 b=r7 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t128 mm8 a=r6 b=r7 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t129 mm8 a=r3 b=r2 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t130 mm8 a=r0 b=r4 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t131 mm8 a=r3 b=r5 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t132 mm8 a=r0 b=r7 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t133 mm8 a=r2 b=r6 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t134 mm8 a=r5 b=r6 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t135 mm8 a=r0 b=r2 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t136 mm8 a=r3 b=r1 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t137 mm8 a=r7 b=r0 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t138 mm8 a=r2 b=r2 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t139 mm8 a=r0 b=r4 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t140 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t141 mm8 a=r5 b=r5 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t142 mm8 a=r2 b=r2 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t143 mm8 a=r4 b=r3 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t144 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t145 mm8 a=r7 b=r2 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t146 mm8 a=r2 b=r7 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t147 mm8 a=r3 b=r1 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t148 mm8 a=r2 b=r1 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t149 mm8 a=r4 b=r6 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t150 mm8 a=r3 b=r6 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t151 mm8 a=r1 b=r4 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t152 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t153 mm8 a=r0 b=r3 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t154 mm8 a=r7 b=r3 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t155 mm8 a=r1 b=r3 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t156 mm8 a=r6 b=r3 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t157 mm8 a=r6 b=r4 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t158 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t159 mm8 a=r1 b=r0 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t160 mm8 a=r0 b=r1 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t161 mm8 a=r4 b=r0 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t162 mm8 a=r6 b=r6 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t163 mm8 a=r5 b=r3 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t164 mm8 a=r0 b=r7 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t165 mm8 a=r6 b=r5 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t166 mm8 a=r6 b=r5 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t167 mm8 a=r4 b=r5 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t168 mm8 a=r3 b=r6 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t169 mm8 a=r6 b=r2 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t170 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t171 mm8 a=r6 b=r3 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t172 mm8 a=r7 b=r3 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t173 mm8 a=r0 b=r0 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t174 mm8 a=r6 b=r6 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t175 mm8 a=r1 b=r6 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t176 mm8 a=r2 b=r4 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t177 mm8 a=r1 b=r7 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t178 mm8 a=r4 b=r2 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t179 mm8 a=r5 b=r2 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t180 mm8 a=r0 b=r4 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t181 mm8 a=r3 b=r5 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t182 mm8 a=r1 b=r1 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t183 mm8 a=r3 b=r1 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t184 mm8 a=r7 b=r4 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t185 mm8 a=r5 b=r0 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t186 mm8 a=r2 b=r5 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t187 mm8 a=r3 b=r0 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t188 mm8 a=r1 b=r4 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t189 mm8 a=r7 b=r2 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t190 mm8 a=r1 b=r5 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t191 mm8 a=r2 b=r1 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t192 mm8 a=r5 b=r0 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t193 mm8 a=r2 b=r1 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t194 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t195 mm8 a=r2 b=r7 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t196 mm8 a=r5 b=r6 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t197 mm8 a=r5 b=r3 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t198 mm8 a=r1 b=r0 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t199 mm8 a=r2 b=r5 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t200 mm8 a=r5 b=r7 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t201 mm8 a=r7 b=r2 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t202 mm8 a=r2 b=r5 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t203 mm8 a=r0 b=r5 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t204 mm8 a=r4 b=r7 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t205 mm8 a=r3 b=r7 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t206 mm8 a=r0 b=r6 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t207 mm8 a=r7 b=r7 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t208 mm8 a=r0 b=r1 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t209 mm8 a=r4 b=r4 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t210 mm8 a=r7 b=r1 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t211 mm8 a=r1 b=r5 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t212 mm8 a=r2 b=r4 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t213 mm8 a=r2 b=r2 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t214 mm8 a=r4 b=r6 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t215 mm8 a=r7 b=r3 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t216 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t217 mm8 a=r5 b=r7 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t218 mm8 a=r7 b=r1 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t219 mm8 a=r2 b=r1 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t220 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t221 mm8 a=r7 b=r1 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t222 mm8 a=r0 b=r6 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t223 mm8 a=r2 b=r2 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t224 mm8 a=r1 b=r1 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t225 mm8 a=r2 b=r7 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t226 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t227 mm8 a=r3 b=r5 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t228 mm8 a=r7 b=r1 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t229 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t230 mm8 a=r0 b=r3 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t231 mm8 a=r1 b=r3 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t232 mm8 a=r5 b=r4 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t233 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t234 mm8 a=r2 b=r0 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t235 mm8 a=r3 b=r4 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t236 mm8 a=r1 b=r7 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t237 mm8 a=r3 b=r5 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t238 mm8 a=r2 b=r0 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t239 mm8 a=r5 b=r7 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t240 mm8 a=r2 b=r7 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t241 mm8 a=r0 b=r1 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t242 mm8 a=r0 b=r2 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t243 mm8 a=r4 b=r4 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t244 mm8 a=r7 b=r6 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t245 mm8 a=r4 b=r4 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t246 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t247 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t248 mm8 a=r7 b=r5 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t249 mm8 a=r4 b=r4 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t250 mm8 a=r0 b=r6 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t251 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t252 mm8 a=r0 b=r2 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t253 mm8 a=r5 b=r5 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t254 mm8 a=r6 b=r7 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t255 mm8 a=r5 b=r0 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t256 mm8 a=r2 b=r1 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t257 mm8 a=r4 b=r3 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t258 mm8 a=r4 b=r1 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t259 mm8 a=r5 b=r5 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t260 mm8 a=r3 b=r2 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t261 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t262 mm8 a=r6 b=r4 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t263 mm8 a=r6 b=r4 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t264 mm8 a=r7 b=r2 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t265 mm8 a=r6 b=r1 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t266 mm8 a=r0 b=r0 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t267 mm8 a=r6 b=r3 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t268 mm8 a=r1 b=r0 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t269 mm8 a=r7 b=r7 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t270 mm8 a=r6 b=r2 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t271 mm8 a=r0 b=r3 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t272 mm8 a=r6 b=r0 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t273 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t274 mm8 a=r1 b=r0 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t275 mm8 a=r1 b=r3 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t276 mm8 a=r1 b=r5 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t277 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t278 mm8 a=r0 b=r0 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t279 mm8 a=r2 b=r4 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t280 mm8 a=r7 b=r2 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t281 mm8 a=r1 b=r0 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t282 mm8 a=r0 b=r1 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t283 mm8 a=r0 b=r2 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t284 mm8 a=r1 b=r5 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t285 mm8 a=r4 b=r7 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t286 mm8 a=r1 b=r0 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t287 mm8 a=r7 b=r1 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t288 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t289 mm8 a=r0 b=r5 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t290 mm8 a=r3 b=r2 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t291 mm8 a=r7 b=r4 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t292 mm8 a=r6 b=r7 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t293 mm8 a=r5 b=r5 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t294 mm8 a=r5 b=r7 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t295 mm8 a=r0 b=r7 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t296 mm8 a=r6 b=r1 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t297 mm8 a=r4 b=r5 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t298 mm8 a=r6 b=r0 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t299 mm8 a=r6 b=r0 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t300 mm8 a=r6 b=r4 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t301 mm8 a=r6 b=r4 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t302 mm8 a=r7 b=r6 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t303 mm8 a=r3 b=r2 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t304 mm8 a=r6 b=r5 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t305 mm8 a=r4 b=r5 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t306 mm8 a=r3 b=r1 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t307 mm8 a=r4 b=r3 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t308 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t309 mm8 a=r2 b=r1 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t310 mm8 a=r6 b=r1 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t311 mm8 a=r4 b=r6 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t312 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t313 mm8 a=r2 b=r6 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t314 mm8 a=r0 b=r3 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t315 mm8 a=r7 b=r1 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t316 mm8 a=r2 b=r4 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t317 mm8 a=r1 b=r6 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t318 mm8 a=r2 b=r1 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t319 mm8 a=r0 b=r2 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t320 mm8 a=r5 b=r2 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t321 mm8 a=r7 b=r4 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t322 mm8 a=r2 b=r4 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t323 mm8 a=r4 b=r3 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t324 mm8 a=r5 b=r7 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t325 mm8 a=r6 b=r2 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t326 mm8 a=r6 b=r1 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t327 mm8 a=r1 b=r5 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t328 mm8 a=r0 b=r1 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t329 mm8 a=r0 b=r3 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t330 mm8 a=r6 b=r7 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t331 mm8 a=r4 b=r4 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t332 mm8 a=r4 b=r4 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t333 mm8 a=r4 b=r6 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t334 mm8 a=r2 b=r2 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t335 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t336 mm8 a=r4 b=r0 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t337 mm8 a=r1 b=r4 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t338 mm8 a=r3 b=r7 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t339 mm8 a=r5 b=r1 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t340 mm8 a=r5 b=r0 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t341 mm8 a=r6 b=r6 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t342 mm8 a=r4 b=r7 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t343 mm8 a=r0 b=r4 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t344 mm8 a=r2 b=r5 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t345 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t346 mm8 a=r6 b=r0 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t347 mm8 a=r5 b=r3 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t348 mm8 a=r0 b=r4 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t349 mm8 a=r6 b=r1 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t350 mm8 a=r1 b=r5 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t351 mm8 a=r6 b=r4 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t352 mm8 a=r4 b=r1 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t353 mm8 a=r7 b=r3 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t354 mm8 a=r4 b=r1 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t355 mm8 a=r4 b=r5 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t356 mm8 a=r7 b=r6 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t357 mm8 a=r7 b=r3 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t358 mm8 a=r4 b=r7 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t359 mm8 a=r2 b=r7 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t360 mm8 a=r4 b=r4 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t361 mm8 a=r2 b=r2 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t362 mm8 a=r2 b=r7 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t363 mm8 a=r7 b=r6 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t364 mm8 a=r6 b=r4 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t365 mm8 a=r6 b=r6 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t366 mm8 a=r7 b=r7 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t367 mm8 a=r4 b=r1 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t368 mm8 a=r7 b=r0 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t369 mm8 a=r1 b=r0 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t370 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t371 mm8 a=r5 b=r4 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t372 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t373 mm8 a=r4 b=r6 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t374 mm8 a=r1 b=r5 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t375 mm8 a=r6 b=r5 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t376 mm8 a=r3 b=r6 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t377 mm8 a=r6 b=r4 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t378 mm8 a=r6 b=r3 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t379 mm8 a=r3 b=r0 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t380 mm8 a=r2 b=r0 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t381 mm8 a=r3 b=r5 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t382 mm8 a=r5 b=r2 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t383 mm8 a=r1 b=r2 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t384 mm8 a=r3 b=r1 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t385 mm8 a=r6 b=r0 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t386 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t387 mm8 a=r3 b=r7 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t388 mm8 a=r3 b=r0 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t389 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t390 mm8 a=r5 b=r4 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t391 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t392 mm8 a=r2 b=r3 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t393 mm8 a=r3 b=r6 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t394 mm8 a=r0 b=r7 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t395 mm8 a=r0 b=r7 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t396 mm8 a=r7 b=r7 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t397 mm8 a=r5 b=r7 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t398 mm8 a=r6 b=r5 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t399 mm8 a=r6 b=r3 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t400 mm8 a=r2 b=r7 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t401 mm8 a=r7 b=r6 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t402 mm8 a=r0 b=r0 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t403 mm8 a=r7 b=r4 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t404 mm8 a=r1 b=r1 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t405 mm8 a=r6 b=r6 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t406 mm8 a=r6 b=r1 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t407 mm8 a=r5 b=r7 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t408 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t409 mm8 a=r3 b=r4 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t410 mm8 a=r0 b=r5 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t411 mm8 a=r2 b=r0 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t412 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t413 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t414 mm8 a=r7 b=r1 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t415 mm8 a=r4 b=r7 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t416 mm8 a=r5 b=r4 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t417 mm8 a=r6 b=r6 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t418 mm8 a=r1 b=r3 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t419 mm8 a=r5 b=r0 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t420 mm8 a=r4 b=r2 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t421 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t422 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t423 mm8 a=r1 b=r7 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t424 mm8 a=r5 b=r1 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t425 mm8 a=r2 b=r7 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t426 mm8 a=r1 b=r6 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t427 mm8 a=r6 b=r6 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t428 mm8 a=r6 b=r2 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t429 mm8 a=r3 b=r3 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t430 mm8 a=r7 b=r7 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t431 mm8 a=r6 b=r4 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t432 mm8 a=r3 b=r2 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t433 mm8 a=r3 b=r0 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t434 mm8 a=r0 b=r7 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t435 mm8 a=r7 b=r5 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t436 mm8 a=r6 b=r4 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t437 mm8 a=r0 b=r7 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t438 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t439 mm8 a=r4 b=r7 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t440 mm8 a=r3 b=r5 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t441 mm8 a=r5 b=r1 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t442 mm8 a=r1 b=r2 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t443 mm8 a=r1 b=r7 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t444 mm8 a=r3 b=r7 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t445 mm8 a=r7 b=r5 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t446 mm8 a=r3 b=r1 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t447 mm8 a=r6 b=r7 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t448 mm8 a=r5 b=r2 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t449 mm8 a=r7 b=r1 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t450 mm8 a=r7 b=r5 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t451 mm8 a=r0 b=r3 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t452 mm8 a=r2 b=r4 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t453 mm8 a=r5 b=r7 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t454 mm8 a=r7 b=r7 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t455 mm8 a=r7 b=r1 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t456 mm8 a=r6 b=r5 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t457 mm8 a=r7 b=r7 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t458 mm8 a=r5 b=r1 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t459 mm8 a=r6 b=r7 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t460 mm8 a=r7 b=r7 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t461 mm8 a=r0 b=r6 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t462 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t463 mm8 a=r2 b=r0 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t464 mm8 a=r0 b=r3 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t465 mm8 a=r1 b=r7 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t466 mm8 a=r1 b=r2 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t467 mm8 a=r1 b=r0 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t468 mm8 a=r4 b=r1 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t469 mm8 a=r2 b=r0 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t470 mm8 a=r3 b=r1 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t471 mm8 a=r2 b=r2 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t472 mm8 a=r1 b=r7 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t473 mm8 a=r2 b=r7 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t474 mm8 a=r5 b=r3 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t475 mm8 a=r7 b=r6 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t476 mm8 a=r3 b=r0 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t477 mm8 a=r3 b=r1 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t478 mm8 a=r0 b=r6 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t479 mm8 a=r7 b=r6 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t480 mm8 a=r6 b=r0 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t481 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t482 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t483 mm8 a=r7 b=r0 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t484 mm8 a=r3 b=r7 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t485 mm8 a=r6 b=r4 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t486 mm8 a=r6 b=r5 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t487 mm8 a=r2 b=r5 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t488 mm8 a=r1 b=r2 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t489 mm8 a=r5 b=r2 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t490 mm8 a=r7 b=r1 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t491 mm8 a=r0 b=r0 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t492 mm8 a=r3 b=r1 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t493 mm8 a=r6 b=r1 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t494 mm8 a=r5 b=r6 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t495 mm8 a=r2 b=r2 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t496 mm8 a=r3 b=r5 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t497 mm8 a=r3 b=r2 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t498 mm8 a=r3 b=r1 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t499 mm8 a=r1 b=r6 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t500 mm8 a=r7 b=r6 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t501 mm8 a=r4 b=r4 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t502 mm8 a=r5 b=r7 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t503 mm8 a=r7 b=r5 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t504 mm8 a=r2 b=r0 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t505 mm8 a=r4 b=r0 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t506 mm8 a=r0 b=r2 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t507 mm8 a=r5 b=r6 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t508 mm8 a=r7 b=r3 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t509 mm8 a=r4 b=r4 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t510 mm8 a=r0 b=r6 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t511 mm8 a=r2 b=r4 c=r1 c2=r3 - } - uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((ulong)hi << 32) | (ulong)lo; -} diff --git a/proto-cuda/packs-ca4/mx8_mm512/program_bound.metal b/proto-cuda/packs-ca4/mx8_mm512/program_bound.metal deleted file mode 100644 index fb3afad92..000000000 --- a/proto-cuda/packs-ca4/mx8_mm512/program_bound.metal +++ /dev/null @@ -1,643 +0,0 @@ -#include -using namespace metal; -// int8 tile reference (Counter ASIC 4.0 research): A = the 32 lanes' av as 8 x 16 bytes (lane l holds row l >> 2, bytes 4 (l & 3)..), -// B = bv as 16 x 8 (lane l holds column l >> 2); lane l gets C[l >> 2][2 (l & 3)] and C[l >> 2][2 (l & 3) + 1]; the PTX mma.m8n8k16 u8 layout. -static inline void mm8_ref(uint av, uint bv, uint lane, thread uint& d0, thread uint& d1) { - uint row = lane >> 2, col0 = 2u * (lane & 3u); - uint acc0 = 0u, acc1 = 0u; - for (uint j = 0u; j < 4u; ++j) { - uint aw = simd_shuffle(av, (ushort)(4u * row + j)); - uint bw0 = simd_shuffle(bv, (ushort)(4u * col0 + j)); - uint bw1 = simd_shuffle(bv, (ushort)(4u * (col0 + 1u) + j)); - for (uint t = 0u; t < 4u; ++t) { - uint ab = (aw >> (8u * t)) & 0xffu; - acc0 += ab * ((bw0 >> (8u * t)) & 0xffu); - acc1 += ab * ((bw1 >> (8u * t)) & 0xffu); - } - } - d0 = acc0; d1 = acc1; -} - - -#define MASK 0x0fffffffu -constant uint SEEDW[8] = { 0x67a9a7beu, 0x1a155b25u, 0xfddfb732u, 0x4b5af2e8u, 0xc55caf33u, 0xa27c13b7u, 0x06628a48u, 0x03852469u }; - -inline uint splitmix32(uint x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -inline uint rotl_imm(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 -inline uint rotr_var(uint x, uint n) { n &= 31u; return (x >> n) | (x << ((32u - n) & 31u)); } -inline uint ds_elem(uint i, uint d0, uint d1) { - uint x = i ^ d0; - x *= 0x9E3779B1u; x ^= x >> 15; - x += d1; - x *= 0x85EBCA77u; x ^= x >> 13; - x *= 0xC2B2AE3Du; x ^= x >> 16; - return x; -} - -// Header-bound variant: the init words come from buffer 3 (bind.rs), not from SEEDW. -kernel void igneum_hash_bound(device const uint* dataset [[buffer(0)]], - device ulong* out [[buffer(1)]], - constant uint& baseNonce [[buffer(2)]], - constant uint* initw [[buffer(3)]], - uint gid [[thread_position_in_grid]]) { - uint nonce = baseNonce + gid; - uint r0, r1, r2, r3, r4, r5, r6, r7; - uint lane = gid & 31u; - { uint x = nonce ^ initw[0]; x += 0x9e3779b9u * 1u; x = splitmix32(x); r0 = x ^ initw[1]; } - { uint x = nonce ^ initw[1]; x += 0x9e3779b9u * 2u; x = splitmix32(x); r1 = x ^ initw[2]; } - { uint x = nonce ^ initw[2]; x += 0x9e3779b9u * 3u; x = splitmix32(x); r2 = x ^ initw[3]; } - { uint x = nonce ^ initw[3]; x += 0x9e3779b9u * 4u; x = splitmix32(x); r3 = x ^ initw[4]; } - { uint x = nonce ^ initw[4]; x += 0x9e3779b9u * 5u; x = splitmix32(x); r4 = x ^ initw[5]; } - { uint x = nonce ^ initw[5]; x += 0x9e3779b9u * 6u; x = splitmix32(x); r5 = x ^ initw[6]; } - { uint x = nonce ^ initw[6]; x += 0x9e3779b9u * 7u; x = splitmix32(x); r6 = x ^ initw[7]; } - { uint x = nonce ^ initw[7]; x += 0x9e3779b9u * 8u; x = splitmix32(x); r7 = x ^ initw[0]; } - - for (uint it = 0u; it < 8u; ++it) { - uint sel = r0; - r2 = r3 * r4 + r2; // 0 - r2 = r1 * r1 + r2; // 1 - r2 = r3 * r2 + r2; // 2 - r3 = r3 ^ r5; // 3 - r7 = r7 ^ dataset[r2 & MASK]; // 4 - r5 = r5 ^ dataset[r7 & MASK]; // 5 - r1 = r1 ^ simd_shuffle_xor(r4, (ushort)8); // 6 - r7 = r7 ^ simd_shuffle_xor(r3, (ushort)8); // 7 - r1 = mulhi(r1, r5); // 8 - r6 = rotr_var(r6, r3); // 9 - r3 = r3 | r4; // 10 - r4 = r4 ^ dataset[r3 & MASK]; // 11 - r0 = mulhi(r0, r4); // 12 - r5 = r5 + r1 + select(0xc7934706u, 0xd3177981u, ((sel >> 30u) & 1u) != 0u); // 13 - r0 = r0 ^ dataset[r4 & MASK]; // 14 - r2 = r2 - r4; // 15 - r2 = r2 ^ dataset[r0 & MASK]; // 16 - r7 = r7 ^ dataset[r2 & MASK]; // 17 - r7 = r7 ^ simd_shuffle_xor(r3, (ushort)4); // 18 - r5 = r5 * r0; // 19 - r3 = r3 ^ simd_shuffle_xor(r4, (ushort)2); // 20 - r2 = r2 ^ simd_shuffle_xor(r4, (ushort)16); // 21 - r6 = mulhi(r6, r2); // 22 - r6 = r6 ^ dataset[r1 & MASK]; // 23 - r5 = r5 * r0; // 24 - r5 = rotl_imm(r5, 19u); // 25 - r7 = r7 ^ simd_shuffle_xor(r6, (ushort)2); // 26 - r0 = r0 ^ r5; // 27 - r0 = r0 ^ r4; // 28 - r3 = r3 - r0; // 29 - r5 = r5 * r1; // 30 - r7 = r7 ^ dataset[r2 & MASK]; // 31 - r1 = r1 ^ dataset[r0 & MASK]; // 32 - r5 = r5 ^ r6; // 33 - r5 = r5 ^ dataset[r1 & MASK]; // 34 - r0 = mulhi(r0, r5); // 35 - r5 = r5 ^ simd_shuffle_xor(r2, (ushort)4); // 36 - r7 = r7 ^ dataset[r0 & MASK]; // 37 - r3 = r3 + r1 + select(0x75ba2fadu, 0x230c005cu, ((sel >> 27u) & 1u) != 0u); // 38 - r1 = r1 ^ simd_shuffle_xor(r5, (ushort)4); // 39 - r2 = r2 ^ r5; // 40 - r3 = r6 * r3 + r3; // 41 - r6 = r6 - r7; // 42 - r7 = r7 ^ r0; // 43 - r1 = r1 ^ dataset[r7 & MASK]; // 44 - r2 = r2 * r3; // 45 - r1 = mulhi(r1, r5); // 46 - r4 = r4 - r3; // 47 - r2 = rotr_var(r2, r6); // 48 - r3 = r3 ^ dataset[r5 & MASK]; // 49 - r1 = r1 + r5 + select(0x81b8bc2cu, 0x1907970cu, ((sel >> 7u) & 1u) != 0u); // 50 - r0 = r0 * r2; // 51 - r0 = r0 + r2 + select(0x4f92b968u, 0x699fd448u, ((sel >> 6u) & 1u) != 0u); // 52 - r1 = r1 + r0 + select(0x2bb965afu, 0x77b1520du, ((sel >> 12u) & 1u) != 0u); // 53 - r7 = rotl_imm(r7, 14u); // 54 - r3 = r3 + r7 + select(0x7b0fe07au, 0xa54c55a0u, ((sel >> 1u) & 1u) != 0u); // 55 - r6 = r6 ^ dataset[r7 & MASK]; // 56 - r1 = rotr_var(r1, r5); // 57 - r5 = r5 ^ dataset[r4 & MASK]; // 58 - r6 = r6 ^ dataset[r2 & MASK]; // 59 - r3 = r5 * r0 + r3; // 60 - r5 = r5 + r7 + select(0xaf9dd72du, 0xad7493e7u, ((sel >> 31u) & 1u) != 0u); // 61 - r4 = r4 + r6 + select(0x89841d87u, 0x1e07c3d9u, ((sel >> 27u) & 1u) != 0u); // 62 - r5 = rotl_imm(r5, 19u); // 63 - // int8 tile block (Counter ASIC 4.0 research, experimental): 512 mma.m8n8k16 u8 tiles per iteration, both outputs consumed - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t0 mm8 a=r6 b=r5 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t1 mm8 a=r1 b=r1 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t2 mm8 a=r2 b=r5 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t3 mm8 a=r1 b=r5 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t4 mm8 a=r2 b=r0 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t5 mm8 a=r2 b=r6 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t6 mm8 a=r1 b=r4 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t7 mm8 a=r4 b=r6 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t8 mm8 a=r1 b=r1 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t9 mm8 a=r3 b=r7 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t10 mm8 a=r3 b=r0 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t11 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t12 mm8 a=r3 b=r1 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t13 mm8 a=r1 b=r5 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t14 mm8 a=r3 b=r3 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t15 mm8 a=r1 b=r0 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t16 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t17 mm8 a=r0 b=r5 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t18 mm8 a=r1 b=r1 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t19 mm8 a=r3 b=r3 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t20 mm8 a=r3 b=r0 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t21 mm8 a=r0 b=r3 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t22 mm8 a=r1 b=r7 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t23 mm8 a=r1 b=r2 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t24 mm8 a=r3 b=r4 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t25 mm8 a=r5 b=r0 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t26 mm8 a=r1 b=r0 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t27 mm8 a=r3 b=r2 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t28 mm8 a=r2 b=r4 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t29 mm8 a=r3 b=r4 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t30 mm8 a=r6 b=r5 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t31 mm8 a=r1 b=r7 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t32 mm8 a=r7 b=r6 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t33 mm8 a=r7 b=r1 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t34 mm8 a=r1 b=r0 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t35 mm8 a=r2 b=r4 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t36 mm8 a=r1 b=r7 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t37 mm8 a=r1 b=r7 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t38 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t39 mm8 a=r6 b=r2 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t40 mm8 a=r6 b=r1 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t41 mm8 a=r6 b=r0 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t42 mm8 a=r7 b=r4 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t43 mm8 a=r6 b=r1 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t44 mm8 a=r0 b=r6 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t45 mm8 a=r6 b=r5 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t46 mm8 a=r6 b=r6 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t47 mm8 a=r7 b=r3 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t48 mm8 a=r7 b=r7 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t49 mm8 a=r4 b=r5 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t50 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t51 mm8 a=r3 b=r4 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t52 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t53 mm8 a=r7 b=r7 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t54 mm8 a=r7 b=r6 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t55 mm8 a=r1 b=r7 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t56 mm8 a=r4 b=r0 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t57 mm8 a=r4 b=r3 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t58 mm8 a=r2 b=r3 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t59 mm8 a=r6 b=r7 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t60 mm8 a=r6 b=r0 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t61 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t62 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t63 mm8 a=r1 b=r6 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t64 mm8 a=r0 b=r5 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t65 mm8 a=r0 b=r1 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t66 mm8 a=r3 b=r5 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t67 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t68 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t69 mm8 a=r6 b=r7 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t70 mm8 a=r1 b=r7 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t71 mm8 a=r3 b=r0 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t72 mm8 a=r4 b=r0 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t73 mm8 a=r4 b=r1 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t74 mm8 a=r5 b=r1 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t75 mm8 a=r4 b=r4 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t76 mm8 a=r5 b=r6 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t77 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t78 mm8 a=r2 b=r5 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t79 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t80 mm8 a=r6 b=r7 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t81 mm8 a=r4 b=r3 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t82 mm8 a=r6 b=r0 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t83 mm8 a=r2 b=r2 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t84 mm8 a=r0 b=r6 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t85 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t86 mm8 a=r3 b=r4 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t87 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t88 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t89 mm8 a=r4 b=r2 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t90 mm8 a=r6 b=r5 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t91 mm8 a=r6 b=r5 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t92 mm8 a=r2 b=r6 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t93 mm8 a=r2 b=r2 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t94 mm8 a=r2 b=r4 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t95 mm8 a=r5 b=r4 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t96 mm8 a=r0 b=r6 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t97 mm8 a=r0 b=r5 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t98 mm8 a=r2 b=r7 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t99 mm8 a=r1 b=r4 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t100 mm8 a=r2 b=r4 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t101 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t102 mm8 a=r5 b=r7 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t103 mm8 a=r4 b=r7 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t104 mm8 a=r2 b=r0 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t105 mm8 a=r6 b=r1 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t106 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t107 mm8 a=r2 b=r1 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t108 mm8 a=r4 b=r6 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t109 mm8 a=r2 b=r7 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t110 mm8 a=r1 b=r0 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t111 mm8 a=r6 b=r5 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t112 mm8 a=r5 b=r7 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t113 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t114 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t115 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t116 mm8 a=r7 b=r2 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t117 mm8 a=r7 b=r6 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t118 mm8 a=r0 b=r1 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t119 mm8 a=r4 b=r3 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t120 mm8 a=r5 b=r3 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t121 mm8 a=r2 b=r7 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t122 mm8 a=r0 b=r1 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t123 mm8 a=r5 b=r2 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t124 mm8 a=r5 b=r1 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t125 mm8 a=r7 b=r2 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t126 mm8 a=r6 b=r3 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t127 mm8 a=r6 b=r7 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t128 mm8 a=r6 b=r7 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t129 mm8 a=r3 b=r2 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t130 mm8 a=r0 b=r4 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t131 mm8 a=r3 b=r5 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t132 mm8 a=r0 b=r7 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t133 mm8 a=r2 b=r6 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t134 mm8 a=r5 b=r6 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t135 mm8 a=r0 b=r2 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t136 mm8 a=r3 b=r1 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t137 mm8 a=r7 b=r0 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t138 mm8 a=r2 b=r2 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t139 mm8 a=r0 b=r4 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t140 mm8 a=r4 b=r6 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t141 mm8 a=r5 b=r5 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t142 mm8 a=r2 b=r2 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t143 mm8 a=r4 b=r3 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t144 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t145 mm8 a=r7 b=r2 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t146 mm8 a=r2 b=r7 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t147 mm8 a=r3 b=r1 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t148 mm8 a=r2 b=r1 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t149 mm8 a=r4 b=r6 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t150 mm8 a=r3 b=r6 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t151 mm8 a=r1 b=r4 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t152 mm8 a=r3 b=r1 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t153 mm8 a=r0 b=r3 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t154 mm8 a=r7 b=r3 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t155 mm8 a=r1 b=r3 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t156 mm8 a=r6 b=r3 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t157 mm8 a=r6 b=r4 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t158 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t159 mm8 a=r1 b=r0 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t160 mm8 a=r0 b=r1 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t161 mm8 a=r4 b=r0 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t162 mm8 a=r6 b=r6 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t163 mm8 a=r5 b=r3 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t164 mm8 a=r0 b=r7 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t165 mm8 a=r6 b=r5 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t166 mm8 a=r6 b=r5 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t167 mm8 a=r4 b=r5 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t168 mm8 a=r3 b=r6 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t169 mm8 a=r6 b=r2 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t170 mm8 a=r6 b=r5 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t171 mm8 a=r6 b=r3 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t172 mm8 a=r7 b=r3 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t173 mm8 a=r0 b=r0 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t174 mm8 a=r6 b=r6 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t175 mm8 a=r1 b=r6 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t176 mm8 a=r2 b=r4 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t177 mm8 a=r1 b=r7 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t178 mm8 a=r4 b=r2 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t179 mm8 a=r5 b=r2 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t180 mm8 a=r0 b=r4 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t181 mm8 a=r3 b=r5 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t182 mm8 a=r1 b=r1 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t183 mm8 a=r3 b=r1 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t184 mm8 a=r7 b=r4 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t185 mm8 a=r5 b=r0 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t186 mm8 a=r2 b=r5 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t187 mm8 a=r3 b=r0 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t188 mm8 a=r1 b=r4 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t189 mm8 a=r7 b=r2 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t190 mm8 a=r1 b=r5 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t191 mm8 a=r2 b=r1 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t192 mm8 a=r5 b=r0 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t193 mm8 a=r2 b=r1 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t194 mm8 a=r4 b=r1 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t195 mm8 a=r2 b=r7 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t196 mm8 a=r5 b=r6 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t197 mm8 a=r5 b=r3 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t198 mm8 a=r1 b=r0 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t199 mm8 a=r2 b=r5 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t200 mm8 a=r5 b=r7 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t201 mm8 a=r7 b=r2 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t202 mm8 a=r2 b=r5 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t203 mm8 a=r0 b=r5 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t204 mm8 a=r4 b=r7 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t205 mm8 a=r3 b=r7 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t206 mm8 a=r0 b=r6 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t207 mm8 a=r7 b=r7 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t208 mm8 a=r0 b=r1 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t209 mm8 a=r4 b=r4 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t210 mm8 a=r7 b=r1 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t211 mm8 a=r1 b=r5 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t212 mm8 a=r2 b=r4 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t213 mm8 a=r2 b=r2 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t214 mm8 a=r4 b=r6 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t215 mm8 a=r7 b=r3 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t216 mm8 a=r6 b=r2 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t217 mm8 a=r5 b=r7 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t218 mm8 a=r7 b=r1 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t219 mm8 a=r2 b=r1 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t220 mm8 a=r1 b=r7 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t221 mm8 a=r7 b=r1 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t222 mm8 a=r0 b=r6 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t223 mm8 a=r2 b=r2 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t224 mm8 a=r1 b=r1 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t225 mm8 a=r2 b=r7 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t226 mm8 a=r2 b=r1 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t227 mm8 a=r3 b=r5 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t228 mm8 a=r7 b=r1 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t229 mm8 a=r1 b=r4 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t230 mm8 a=r0 b=r3 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t231 mm8 a=r1 b=r3 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t232 mm8 a=r5 b=r4 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t233 mm8 a=r7 b=r2 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t234 mm8 a=r2 b=r0 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t235 mm8 a=r3 b=r4 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t236 mm8 a=r1 b=r7 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t237 mm8 a=r3 b=r5 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t238 mm8 a=r2 b=r0 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t239 mm8 a=r5 b=r7 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t240 mm8 a=r2 b=r7 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t241 mm8 a=r0 b=r1 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t242 mm8 a=r0 b=r2 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t243 mm8 a=r4 b=r4 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r0 += d0_; r7 += d1_; } // t244 mm8 a=r7 b=r6 c=r0 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t245 mm8 a=r4 b=r4 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t246 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t247 mm8 a=r3 b=r6 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t248 mm8 a=r7 b=r5 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t249 mm8 a=r4 b=r4 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t250 mm8 a=r0 b=r6 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t251 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t252 mm8 a=r0 b=r2 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t253 mm8 a=r5 b=r5 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t254 mm8 a=r6 b=r7 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t255 mm8 a=r5 b=r0 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t256 mm8 a=r2 b=r1 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t257 mm8 a=r4 b=r3 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t258 mm8 a=r4 b=r1 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t259 mm8 a=r5 b=r5 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t260 mm8 a=r3 b=r2 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t261 mm8 a=r0 b=r6 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t262 mm8 a=r6 b=r4 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t263 mm8 a=r6 b=r4 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t264 mm8 a=r7 b=r2 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t265 mm8 a=r6 b=r1 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t266 mm8 a=r0 b=r0 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t267 mm8 a=r6 b=r3 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t268 mm8 a=r1 b=r0 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t269 mm8 a=r7 b=r7 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t270 mm8 a=r6 b=r2 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t271 mm8 a=r0 b=r3 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t272 mm8 a=r6 b=r0 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t273 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t274 mm8 a=r1 b=r0 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t275 mm8 a=r1 b=r3 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t276 mm8 a=r1 b=r5 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t277 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t278 mm8 a=r0 b=r0 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t279 mm8 a=r2 b=r4 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r2, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t280 mm8 a=r7 b=r2 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t281 mm8 a=r1 b=r0 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t282 mm8 a=r0 b=r1 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t283 mm8 a=r0 b=r2 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t284 mm8 a=r1 b=r5 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t285 mm8 a=r4 b=r7 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t286 mm8 a=r1 b=r0 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t287 mm8 a=r7 b=r1 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t288 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t289 mm8 a=r0 b=r5 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t290 mm8 a=r3 b=r2 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t291 mm8 a=r7 b=r4 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t292 mm8 a=r6 b=r7 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r5, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t293 mm8 a=r5 b=r5 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t294 mm8 a=r5 b=r7 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t295 mm8 a=r0 b=r7 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t296 mm8 a=r6 b=r1 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t297 mm8 a=r4 b=r5 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t298 mm8 a=r6 b=r0 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t299 mm8 a=r6 b=r0 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t300 mm8 a=r6 b=r4 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t301 mm8 a=r6 b=r4 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t302 mm8 a=r7 b=r6 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t303 mm8 a=r3 b=r2 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t304 mm8 a=r6 b=r5 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t305 mm8 a=r4 b=r5 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t306 mm8 a=r3 b=r1 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t307 mm8 a=r4 b=r3 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t308 mm8 a=r2 b=r1 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t309 mm8 a=r2 b=r1 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t310 mm8 a=r6 b=r1 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t311 mm8 a=r4 b=r6 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t312 mm8 a=r1 b=r3 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r6, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t313 mm8 a=r2 b=r6 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t314 mm8 a=r0 b=r3 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t315 mm8 a=r7 b=r1 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t316 mm8 a=r2 b=r4 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t317 mm8 a=r1 b=r6 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r1, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t318 mm8 a=r2 b=r1 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t319 mm8 a=r0 b=r2 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t320 mm8 a=r5 b=r2 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t321 mm8 a=r7 b=r4 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t322 mm8 a=r2 b=r4 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r3, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t323 mm8 a=r4 b=r3 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t324 mm8 a=r5 b=r7 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t325 mm8 a=r6 b=r2 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t326 mm8 a=r6 b=r1 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t327 mm8 a=r1 b=r5 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r1, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t328 mm8 a=r0 b=r1 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t329 mm8 a=r0 b=r3 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r4 += d0_; r0 += d1_; } // t330 mm8 a=r6 b=r7 c=r4 c2=r0 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t331 mm8 a=r4 b=r4 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t332 mm8 a=r4 b=r4 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t333 mm8 a=r4 b=r6 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t334 mm8 a=r2 b=r2 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t335 mm8 a=r7 b=r1 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r2 += d0_; r3 += d1_; } // t336 mm8 a=r4 b=r0 c=r2 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r4, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t337 mm8 a=r1 b=r4 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r2 += d0_; r7 += d1_; } // t338 mm8 a=r3 b=r7 c=r2 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t339 mm8 a=r5 b=r1 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t340 mm8 a=r5 b=r0 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t341 mm8 a=r6 b=r6 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t342 mm8 a=r4 b=r7 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t343 mm8 a=r0 b=r4 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t344 mm8 a=r2 b=r5 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t345 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t346 mm8 a=r6 b=r0 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t347 mm8 a=r5 b=r3 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r4, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t348 mm8 a=r0 b=r4 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t349 mm8 a=r6 b=r1 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t350 mm8 a=r1 b=r5 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t351 mm8 a=r6 b=r4 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t352 mm8 a=r4 b=r1 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t353 mm8 a=r7 b=r3 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t354 mm8 a=r4 b=r1 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r5, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t355 mm8 a=r4 b=r5 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t356 mm8 a=r7 b=r6 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t357 mm8 a=r7 b=r3 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r5 += d0_; r7 += d1_; } // t358 mm8 a=r4 b=r7 c=r5 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t359 mm8 a=r2 b=r7 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t360 mm8 a=r4 b=r4 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t361 mm8 a=r2 b=r2 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t362 mm8 a=r2 b=r7 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t363 mm8 a=r7 b=r6 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t364 mm8 a=r6 b=r4 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r5 += d0_; r4 += d1_; } // t365 mm8 a=r6 b=r6 c=r5 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t366 mm8 a=r7 b=r7 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t367 mm8 a=r4 b=r1 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t368 mm8 a=r7 b=r0 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t369 mm8 a=r1 b=r0 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t370 mm8 a=r2 b=r5 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t371 mm8 a=r5 b=r4 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t372 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r6, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t373 mm8 a=r4 b=r6 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r5, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t374 mm8 a=r1 b=r5 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t375 mm8 a=r6 b=r5 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t376 mm8 a=r3 b=r6 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t377 mm8 a=r6 b=r4 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t378 mm8 a=r6 b=r3 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t379 mm8 a=r3 b=r0 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t380 mm8 a=r2 b=r0 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t381 mm8 a=r3 b=r5 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t382 mm8 a=r5 b=r2 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r0 += d0_; r2 += d1_; } // t383 mm8 a=r1 b=r2 c=r0 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t384 mm8 a=r3 b=r1 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t385 mm8 a=r6 b=r0 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t386 mm8 a=r2 b=r0 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t387 mm8 a=r3 b=r7 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t388 mm8 a=r3 b=r0 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t389 mm8 a=r6 b=r4 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t390 mm8 a=r5 b=r4 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t391 mm8 a=r6 b=r3 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r3 += d0_; r0 += d1_; } // t392 mm8 a=r2 b=r3 c=r3 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r6, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t393 mm8 a=r3 b=r6 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t394 mm8 a=r0 b=r7 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t395 mm8 a=r0 b=r7 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t396 mm8 a=r7 b=r7 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t397 mm8 a=r5 b=r7 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t398 mm8 a=r6 b=r5 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r3, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t399 mm8 a=r6 b=r3 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r0 += d0_; r4 += d1_; } // t400 mm8 a=r2 b=r7 c=r0 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t401 mm8 a=r7 b=r6 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t402 mm8 a=r0 b=r0 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t403 mm8 a=r7 b=r4 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r1, r1, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t404 mm8 a=r1 b=r1 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r3 += d0_; r6 += d1_; } // t405 mm8 a=r6 b=r6 c=r3 c2=r6 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t406 mm8 a=r6 b=r1 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r2 += d0_; r4 += d1_; } // t407 mm8 a=r5 b=r7 c=r2 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t408 mm8 a=r7 b=r4 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r4, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t409 mm8 a=r3 b=r4 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r5, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t410 mm8 a=r0 b=r5 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t411 mm8 a=r2 b=r0 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r4, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t412 mm8 a=r7 b=r4 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t413 mm8 a=r2 b=r7 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r0 += d0_; r3 += d1_; } // t414 mm8 a=r7 b=r1 c=r0 c2=r3 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r5 += d0_; r1 += d1_; } // t415 mm8 a=r4 b=r7 c=r5 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r4, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t416 mm8 a=r5 b=r4 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t417 mm8 a=r6 b=r6 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r3, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t418 mm8 a=r1 b=r3 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r0, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t419 mm8 a=r5 b=r0 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r2, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t420 mm8 a=r4 b=r2 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r3 += d0_; r7 += d1_; } // t421 mm8 a=r4 b=r4 c=r3 c2=r7 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r4 += d0_; r3 += d1_; } // t422 mm8 a=r0 b=r7 c=r4 c2=r3 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t423 mm8 a=r1 b=r7 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t424 mm8 a=r5 b=r1 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t425 mm8 a=r2 b=r7 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t426 mm8 a=r1 b=r6 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r6, r6, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t427 mm8 a=r6 b=r6 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r2, lane, d0_, d1_); r7 += d0_; r4 += d1_; } // t428 mm8 a=r6 b=r2 c=r7 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r3, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t429 mm8 a=r3 b=r3 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r3 += d0_; r5 += d1_; } // t430 mm8 a=r7 b=r7 c=r3 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t431 mm8 a=r6 b=r4 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t432 mm8 a=r3 b=r2 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t433 mm8 a=r3 b=r0 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r5 += d0_; r3 += d1_; } // t434 mm8 a=r0 b=r7 c=r5 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t435 mm8 a=r7 b=r5 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t436 mm8 a=r6 b=r4 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r7, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t437 mm8 a=r0 b=r7 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t438 mm8 a=r5 b=r1 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r7, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t439 mm8 a=r4 b=r7 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t440 mm8 a=r3 b=r5 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r1 += d0_; r4 += d1_; } // t441 mm8 a=r5 b=r1 c=r1 c2=r4 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t442 mm8 a=r1 b=r2 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r5 += d0_; r2 += d1_; } // t443 mm8 a=r1 b=r7 c=r5 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t444 mm8 a=r3 b=r7 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t445 mm8 a=r7 b=r5 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r2 += d0_; r1 += d1_; } // t446 mm8 a=r3 b=r1 c=r2 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t447 mm8 a=r6 b=r7 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t448 mm8 a=r5 b=r2 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t449 mm8 a=r7 b=r1 c=r1 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r7 += d0_; r3 += d1_; } // t450 mm8 a=r7 b=r5 c=r7 c2=r3 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r1 += d0_; r7 += d1_; } // t451 mm8 a=r0 b=r3 c=r1 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t452 mm8 a=r2 b=r4 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t453 mm8 a=r5 b=r7 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t454 mm8 a=r7 b=r7 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r7 += d0_; r5 += d1_; } // t455 mm8 a=r7 b=r1 c=r7 c2=r5 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t456 mm8 a=r6 b=r5 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t457 mm8 a=r7 b=r7 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r5, r1, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t458 mm8 a=r5 b=r1 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r7, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t459 mm8 a=r6 b=r7 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r7, r7, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t460 mm8 a=r7 b=r7 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t461 mm8 a=r0 b=r6 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t462 mm8 a=r0 b=r6 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r3 += d0_; r2 += d1_; } // t463 mm8 a=r2 b=r0 c=r3 c2=r2 - { uint d0_, d1_; mm8_ref(r0, r3, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t464 mm8 a=r0 b=r3 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r0 += d0_; r5 += d1_; } // t465 mm8 a=r1 b=r7 c=r0 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r4 += d0_; r1 += d1_; } // t466 mm8 a=r1 b=r2 c=r4 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r0, lane, d0_, d1_); r1 += d0_; r6 += d1_; } // t467 mm8 a=r1 b=r0 c=r1 c2=r6 - { uint d0_, d1_; mm8_ref(r4, r1, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t468 mm8 a=r4 b=r1 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t469 mm8 a=r2 b=r0 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t470 mm8 a=r3 b=r1 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t471 mm8 a=r2 b=r2 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r1, r7, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t472 mm8 a=r1 b=r7 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r7, lane, d0_, d1_); r6 += d0_; r1 += d1_; } // t473 mm8 a=r2 b=r7 c=r6 c2=r1 - { uint d0_, d1_; mm8_ref(r5, r3, lane, d0_, d1_); r6 += d0_; r3 += d1_; } // t474 mm8 a=r5 b=r3 c=r6 c2=r3 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t475 mm8 a=r7 b=r6 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r0, lane, d0_, d1_); r5 += d0_; r6 += d1_; } // t476 mm8 a=r3 b=r0 c=r5 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r4 += d0_; r6 += d1_; } // t477 mm8 a=r3 b=r1 c=r4 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t478 mm8 a=r0 b=r6 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r7 += d0_; r1 += d1_; } // t479 mm8 a=r7 b=r6 c=r7 c2=r1 - { uint d0_, d1_; mm8_ref(r6, r0, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t480 mm8 a=r6 b=r0 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r6 += d0_; r7 += d1_; } // t481 mm8 a=r2 b=r4 c=r6 c2=r7 - { uint d0_, d1_; mm8_ref(r2, r3, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t482 mm8 a=r2 b=r3 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r0, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t483 mm8 a=r7 b=r0 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r7, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t484 mm8 a=r3 b=r7 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r4, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t485 mm8 a=r6 b=r4 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r6, r5, lane, d0_, d1_); r2 += d0_; r5 += d1_; } // t486 mm8 a=r6 b=r5 c=r2 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r5, lane, d0_, d1_); r4 += d0_; r5 += d1_; } // t487 mm8 a=r2 b=r5 c=r4 c2=r5 - { uint d0_, d1_; mm8_ref(r1, r2, lane, d0_, d1_); r1 += d0_; r2 += d1_; } // t488 mm8 a=r1 b=r2 c=r1 c2=r2 - { uint d0_, d1_; mm8_ref(r5, r2, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t489 mm8 a=r5 b=r2 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r7, r1, lane, d0_, d1_); r2 += d0_; r6 += d1_; } // t490 mm8 a=r7 b=r1 c=r2 c2=r6 - { uint d0_, d1_; mm8_ref(r0, r0, lane, d0_, d1_); r0 += d0_; r6 += d1_; } // t491 mm8 a=r0 b=r0 c=r0 c2=r6 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r6 += d0_; r4 += d1_; } // t492 mm8 a=r3 b=r1 c=r6 c2=r4 - { uint d0_, d1_; mm8_ref(r6, r1, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t493 mm8 a=r6 b=r1 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r3 += d0_; r1 += d1_; } // t494 mm8 a=r5 b=r6 c=r3 c2=r1 - { uint d0_, d1_; mm8_ref(r2, r2, lane, d0_, d1_); r7 += d0_; r2 += d1_; } // t495 mm8 a=r2 b=r2 c=r7 c2=r2 - { uint d0_, d1_; mm8_ref(r3, r5, lane, d0_, d1_); r5 += d0_; r0 += d1_; } // t496 mm8 a=r3 b=r5 c=r5 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r2, lane, d0_, d1_); r2 += d0_; r0 += d1_; } // t497 mm8 a=r3 b=r2 c=r2 c2=r0 - { uint d0_, d1_; mm8_ref(r3, r1, lane, d0_, d1_); r6 += d0_; r2 += d1_; } // t498 mm8 a=r3 b=r1 c=r6 c2=r2 - { uint d0_, d1_; mm8_ref(r1, r6, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t499 mm8 a=r1 b=r6 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r7, r6, lane, d0_, d1_); r1 += d0_; r5 += d1_; } // t500 mm8 a=r7 b=r6 c=r1 c2=r5 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r1 += d0_; r0 += d1_; } // t501 mm8 a=r4 b=r4 c=r1 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r7, lane, d0_, d1_); r7 += d0_; r6 += d1_; } // t502 mm8 a=r5 b=r7 c=r7 c2=r6 - { uint d0_, d1_; mm8_ref(r7, r5, lane, d0_, d1_); r3 += d0_; r4 += d1_; } // t503 mm8 a=r7 b=r5 c=r3 c2=r4 - { uint d0_, d1_; mm8_ref(r2, r0, lane, d0_, d1_); r4 += d0_; r2 += d1_; } // t504 mm8 a=r2 b=r0 c=r4 c2=r2 - { uint d0_, d1_; mm8_ref(r4, r0, lane, d0_, d1_); r0 += d0_; r1 += d1_; } // t505 mm8 a=r4 b=r0 c=r0 c2=r1 - { uint d0_, d1_; mm8_ref(r0, r2, lane, d0_, d1_); r7 += d0_; r0 += d1_; } // t506 mm8 a=r0 b=r2 c=r7 c2=r0 - { uint d0_, d1_; mm8_ref(r5, r6, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t507 mm8 a=r5 b=r6 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r7, r3, lane, d0_, d1_); r4 += d0_; r7 += d1_; } // t508 mm8 a=r7 b=r3 c=r4 c2=r7 - { uint d0_, d1_; mm8_ref(r4, r4, lane, d0_, d1_); r6 += d0_; r0 += d1_; } // t509 mm8 a=r4 b=r4 c=r6 c2=r0 - { uint d0_, d1_; mm8_ref(r0, r6, lane, d0_, d1_); r6 += d0_; r5 += d1_; } // t510 mm8 a=r0 b=r6 c=r6 c2=r5 - { uint d0_, d1_; mm8_ref(r2, r4, lane, d0_, d1_); r1 += d0_; r3 += d1_; } // t511 mm8 a=r2 b=r4 c=r1 c2=r3 - } - uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((ulong)hi << 32) | (ulong)lo; -} diff --git a/proto-cuda/packs-ca4/mx8_mm512/vectors.h b/proto-cuda/packs-ca4/mx8_mm512/vectors.h deleted file mode 100644 index 560dedade..000000000 --- a/proto-cuda/packs-ca4/mx8_mm512/vectors.h +++ /dev/null @@ -1,57 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// Expected outputs: igneum-pow (Rust) CPU interpreter, generator v2, memory-hard dataset -#pragma once -#ifdef __cplusplus -#include -#else -#include -#endif - -#define IGNEUM_VEC_WARPS 3 -static const uint32_t IGNEUM_VEC_BASE[IGNEUM_VEC_WARPS] = { 0u, 4096u, 1000000u }; -static const uint64_t IGNEUM_VEC_OUT[IGNEUM_VEC_WARPS][32] = { - { // base nonce 0 - 0x9fb618fc7b6228c3ull, 0x7213e619e80bff2eull, 0x8908e869fc64e236ull, 0x53853d142eaa1e6cull, 0x8d5808d85bd32eaeull, 0xe4f65ecf58bac2a9ull, 0x4ea054f4156f4c70ull, 0xa1f08a2b3a1892e0ull, - 0xf44e93176c787eefull, 0xfb9c55a11549b45bull, 0x10b7acceda07b5d6ull, 0x4474ce4799a31f83ull, 0x197eb259ca50e2acull, 0x96e4dfb43bf258a8ull, 0x12c83c8e53a92bf9ull, 0x83ecf8570f5f1dc2ull, - 0xb946e6cd2443717bull, 0x60a888be09b52c72ull, 0x3b763ae6046e85a7ull, 0x6b93ec061ac325e3ull, 0x60db0f498f330cd0ull, 0xf522f6d924097107ull, 0x38b94f8f4453d839ull, 0x517d41d209a5ddebull, - 0x4858f3b1f2d71b82ull, 0x787a16b5e58c3fd9ull, 0xe78b4de4e009ec91ull, 0x0b8198730d366171ull, 0x9397dc5c1d63fa72ull, 0x15ca5f6540b987a3ull, 0x83a24efeb6311fb7ull, 0x714a3e83fec4a2beull - }, - { // base nonce 4096 - 0xeba4e6729f0e97b7ull, 0x74af8ebe7886a1caull, 0x7b92eb7acdaede21ull, 0x2b4909c0c88ffb31ull, 0xcd5a3d7fc34eab6full, 0x928be2f89abd8307ull, 0x1ee3a56655f8e701ull, 0xd2f307a6599e90feull, - 0x624b2ea35003d63cull, 0x831b70652ed75475ull, 0xc7130ca2e331789aull, 0xf2ea9c7c896c794bull, 0x360d08611e11f090ull, 0xc86b79a26ce04221ull, 0xc800bf3fd30e8091ull, 0x3cd7a142317a4fc7ull, - 0x5020b46b67593caeull, 0xf569cf2ad3352bd6ull, 0xcdd040840753678dull, 0x7f91ccac56ed6baaull, 0xf844b8e0342f813bull, 0xd7ccc48b5c99ea75ull, 0x3f801464920736cfull, 0x2cd1c9fcb4868010ull, - 0x8aeffc12d4739525ull, 0x42ca83cca80b44c3ull, 0x6148286c39ecf6dbull, 0xd5db07f58d639b0dull, 0x813dcc14cf534f42ull, 0xcba5207abd54f712ull, 0xbe48e582f2f5cb2aull, 0x09d56ce806b90b55ull - }, - { // base nonce 1000000 - 0xe73c99aacc6c825cull, 0xda827d89d90359f9ull, 0x4d224a3c6f1147e7ull, 0x6748b5fd24e70062ull, 0x9053e40978e90a50ull, 0x7347ab5b612bb430ull, 0xd7371fb2cdfea057ull, 0xbc24275fd6f92ff3ull, - 0x57daadd306115c53ull, 0xd5571774c6de8831ull, 0xce515d8d9a249a47ull, 0x1ec0e9dac751c9d2ull, 0xc35a8fa80881fdafull, 0x9bb227f408b1e2c4ull, 0xb11818ebab6cf5a4ull, 0x212b46a1bf67e747ull, - 0x96bd3432a7384a4full, 0x006c173b6f4854efull, 0x59c23df13369f803ull, 0x01528a936a9a0776ull, 0xcb38ceab8f869d43ull, 0x67a9b4af9ac66df2ull, 0x63b3dc84e4485cf8ull, 0xbfc1a38632fd9d2full, - 0xd2854da6e088992eull, 0xc755fd06911b0a51ull, 0x01a2c5a460546176ull, 0x0710d4a963fee5f6ull, 0x3ddbebf9a8d3fe7eull, 0xaf55162589b7f13cull, 0xb25f45be4e6bbeaaull, 0x88fb9a4c59780da2ull - } -}; - -// Dataset self-test: dataset[0..15] and dataset[IGNEUM_MASK] (268435455). -static const uint32_t IGNEUM_DS_HEAD[16] = { - 0xfdad4319u, 0x1a7b68e1u, 0xde6db608u, 0x13d73892u, 0xd17f447au, 0xb2221ccfu, 0x9db004bdu, 0x57d7d367u, - 0xdbc4cf34u, 0x697c009au, 0xc43af1d4u, 0x97f12b2eu, 0x74c37cd0u, 0xc651ea15u, 0x665a6d29u, 0x22330a2du -}; -static const uint32_t IGNEUM_DS_LAST_INDEX = 268435455u; -static const uint32_t IGNEUM_DS_LAST = 0xa83e7aa6u; -// 64 sampled dataset words (index, value) computed on the Mac. -#define IGNEUM_DS_SAMPLES 64 -static const uint32_t IGNEUM_DS_SAMPLE_INDEX[IGNEUM_DS_SAMPLES] = { - 59471966u, 217795994u, 208353206u, 42483309u, 172547758u, 148076330u, 183853158u, 214389424u, 267488061u, 169781097u, 184093494u, 153880993u, 84977930u, 46426879u, 3093825u, 225364072u, 44593546u, 260713159u, 168250303u, 52384140u, 223401610u, 45554030u, 95410555u, 175039924u, 79171087u, 267580473u, 24168642u, 37981670u, 171551130u, 195559979u, 204611762u, 140997658u, 138925853u, 86637313u, 20736778u, 219665210u, 160430336u, 264654675u, 8013395u, 228945585u, 213884386u, 104419827u, 44185464u, 142737231u, 99284897u, 132475900u, 61861762u, 132056166u, 262388043u, 91878046u, 117353561u, 124768597u, 71352993u, 190698941u, 46055428u, 55281366u, 165145231u, 106810753u, 171985651u, 232085256u, 159510492u, 40072060u, 209107596u, 39023794u -}; -static const uint32_t IGNEUM_DS_SAMPLE_VALUE[IGNEUM_DS_SAMPLES] = { - 0x3230bc7bu, 0x7fbfe2c9u, 0xb2690991u, 0x1745c7c5u, 0x0ab0ccafu, 0x1bf87d6bu, 0x160139fdu, 0x719817acu, 0x0155df4bu, 0xbe1e86c3u, 0x680bcd6cu, 0x79c3dc6cu, 0x181e7e5fu, 0x0713a109u, 0xc705dd9fu, 0x3933b7a8u, 0xdd1c0431u, 0x50522b30u, 0xa0020b38u, 0xbff39e96u, 0x21b67e18u, 0x740f8db3u, 0x2baba568u, 0x2c9bef83u, 0x0ad9b671u, 0xc4327869u, 0x7b4fd7d0u, 0x2c29965fu, 0xec56f15fu, 0x61111746u, 0x303a1d6eu, 0xbddcfd1au, 0xf829a355u, 0x6d5df2a9u, 0x01ab8e44u, 0x06d13507u, 0xda8dcfc6u, 0x01a703e1u, 0xafe7d2c1u, 0xc091c3a2u, 0xac1814feu, 0x6e6ff62au, 0x8fdf01bau, 0xdd3f7159u, 0xdfa0d75cu, 0x26684c35u, 0x7f441e63u, 0x88df2570u, 0x8aa4d5ebu, 0xcc816c05u, 0x434df890u, 0xcd392ad6u, 0x1ab4cb63u, 0x595926fau, 0x7cd76b41u, 0x20cb95c4u, 0x13cf823fu, 0xf9daf901u, 0xff9af40au, 0x2c7dfa51u, 0x871206dbu, 0x938c116cu, 0xb64bf199u, 0x5751f874u -}; -// Cache self-test (memory-hard mode): cache[0..15], the last 16 words, and FNV-1a 64 over all 2^26 words. -static const uint32_t IGNEUM_CACHE_HEAD[16] = { - 0x355a86d2u, 0x7957db1cu, 0xd21772afu, 0x6fc1e09bu, 0xd55ce61du, 0x6e6a278bu, 0xd3f543ceu, 0x223d8e82u, - 0x143ab337u, 0x2e9f05bdu, 0x2eb389bfu, 0x0c6e449eu, 0x5cfa4222u, 0xba6560feu, 0x8e3e1aa4u, 0xdbcc1d53u -}; -static const uint32_t IGNEUM_CACHE_LAST[16] = { - 0x41190d91u, 0xbd277957u, 0x22ddbb49u, 0x6986f207u, 0xdf69a4d6u, 0x26401a3au, 0x818230fbu, 0xc417122du, - 0x3597b211u, 0xb553ce55u, 0xcf39cc0du, 0x3b7fc43au, 0x3fd43b00u, 0x67e1c80eu, 0xffa7ea7du, 0xca2960abu -}; -static const uint64_t IGNEUM_CACHE_FNV64 = 0x48c4f5bf24166b2eull; diff --git a/proto-cuda/packs-ca4/mx8_mm512/vectors.json b/proto-cuda/packs-ca4/mx8_mm512/vectors.json deleted file mode 100644 index 05893747d..000000000 --- a/proto-cuda/packs-ca4/mx8_mm512/vectors.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "seed": "igneum-genesis", - "day": "2026-10-03", - "dataset_mode": "memory-hard", - "dataset_log2_words": 28, - "mask": "0x0fffffff", - "lanes": 32, - "source": "igneum-pow (Rust) CPU interpreter, generator v2, memory-hard dataset", - "warps": [ - {"base_nonce": 0, "expected": [ - "0x9fb618fc7b6228c3", "0x7213e619e80bff2e", "0x8908e869fc64e236", "0x53853d142eaa1e6c", "0x8d5808d85bd32eae", "0xe4f65ecf58bac2a9", "0x4ea054f4156f4c70", "0xa1f08a2b3a1892e0", - "0xf44e93176c787eef", "0xfb9c55a11549b45b", "0x10b7acceda07b5d6", "0x4474ce4799a31f83", "0x197eb259ca50e2ac", "0x96e4dfb43bf258a8", "0x12c83c8e53a92bf9", "0x83ecf8570f5f1dc2", - "0xb946e6cd2443717b", "0x60a888be09b52c72", "0x3b763ae6046e85a7", "0x6b93ec061ac325e3", "0x60db0f498f330cd0", "0xf522f6d924097107", "0x38b94f8f4453d839", "0x517d41d209a5ddeb", - "0x4858f3b1f2d71b82", "0x787a16b5e58c3fd9", "0xe78b4de4e009ec91", "0x0b8198730d366171", "0x9397dc5c1d63fa72", "0x15ca5f6540b987a3", "0x83a24efeb6311fb7", "0x714a3e83fec4a2be" - ]}, - {"base_nonce": 4096, "expected": [ - "0xeba4e6729f0e97b7", "0x74af8ebe7886a1ca", "0x7b92eb7acdaede21", "0x2b4909c0c88ffb31", "0xcd5a3d7fc34eab6f", "0x928be2f89abd8307", "0x1ee3a56655f8e701", "0xd2f307a6599e90fe", - "0x624b2ea35003d63c", "0x831b70652ed75475", "0xc7130ca2e331789a", "0xf2ea9c7c896c794b", "0x360d08611e11f090", "0xc86b79a26ce04221", "0xc800bf3fd30e8091", "0x3cd7a142317a4fc7", - "0x5020b46b67593cae", "0xf569cf2ad3352bd6", "0xcdd040840753678d", "0x7f91ccac56ed6baa", "0xf844b8e0342f813b", "0xd7ccc48b5c99ea75", "0x3f801464920736cf", "0x2cd1c9fcb4868010", - "0x8aeffc12d4739525", "0x42ca83cca80b44c3", "0x6148286c39ecf6db", "0xd5db07f58d639b0d", "0x813dcc14cf534f42", "0xcba5207abd54f712", "0xbe48e582f2f5cb2a", "0x09d56ce806b90b55" - ]}, - {"base_nonce": 1000000, "expected": [ - "0xe73c99aacc6c825c", "0xda827d89d90359f9", "0x4d224a3c6f1147e7", "0x6748b5fd24e70062", "0x9053e40978e90a50", "0x7347ab5b612bb430", "0xd7371fb2cdfea057", "0xbc24275fd6f92ff3", - "0x57daadd306115c53", "0xd5571774c6de8831", "0xce515d8d9a249a47", "0x1ec0e9dac751c9d2", "0xc35a8fa80881fdaf", "0x9bb227f408b1e2c4", "0xb11818ebab6cf5a4", "0x212b46a1bf67e747", - "0x96bd3432a7384a4f", "0x006c173b6f4854ef", "0x59c23df13369f803", "0x01528a936a9a0776", "0xcb38ceab8f869d43", "0x67a9b4af9ac66df2", "0x63b3dc84e4485cf8", "0xbfc1a38632fd9d2f", - "0xd2854da6e088992e", "0xc755fd06911b0a51", "0x01a2c5a460546176", "0x0710d4a963fee5f6", "0x3ddbebf9a8d3fe7e", "0xaf55162589b7f13c", "0xb25f45be4e6bbeaa", "0x88fb9a4c59780da2" - ]} - ], - "dataset_head": ["0xfdad4319", "0x1a7b68e1", "0xde6db608", "0x13d73892", "0xd17f447a", "0xb2221ccf", "0x9db004bd", "0x57d7d367", "0xdbc4cf34", "0x697c009a", "0xc43af1d4", "0x97f12b2e", "0x74c37cd0", "0xc651ea15", "0x665a6d29", "0x22330a2d"], - "dataset_last_index": 268435455, - "dataset_last": "0xa83e7aa6", - "dataset_samples": [{"index": 59471966, "value": "0x3230bc7b"}, {"index": 217795994, "value": "0x7fbfe2c9"}, {"index": 208353206, "value": "0xb2690991"}, {"index": 42483309, "value": "0x1745c7c5"}, {"index": 172547758, "value": "0x0ab0ccaf"}, {"index": 148076330, "value": "0x1bf87d6b"}, {"index": 183853158, "value": "0x160139fd"}, {"index": 214389424, "value": "0x719817ac"}, {"index": 267488061, "value": "0x0155df4b"}, {"index": 169781097, "value": "0xbe1e86c3"}, {"index": 184093494, "value": "0x680bcd6c"}, {"index": 153880993, "value": "0x79c3dc6c"}, {"index": 84977930, "value": "0x181e7e5f"}, {"index": 46426879, "value": "0x0713a109"}, {"index": 3093825, "value": "0xc705dd9f"}, {"index": 225364072, "value": "0x3933b7a8"}, {"index": 44593546, "value": "0xdd1c0431"}, {"index": 260713159, "value": "0x50522b30"}, {"index": 168250303, "value": "0xa0020b38"}, {"index": 52384140, "value": "0xbff39e96"}, {"index": 223401610, "value": "0x21b67e18"}, {"index": 45554030, "value": "0x740f8db3"}, {"index": 95410555, "value": "0x2baba568"}, {"index": 175039924, "value": "0x2c9bef83"}, {"index": 79171087, "value": "0x0ad9b671"}, {"index": 267580473, "value": "0xc4327869"}, {"index": 24168642, "value": "0x7b4fd7d0"}, {"index": 37981670, "value": "0x2c29965f"}, {"index": 171551130, "value": "0xec56f15f"}, {"index": 195559979, "value": "0x61111746"}, {"index": 204611762, "value": "0x303a1d6e"}, {"index": 140997658, "value": "0xbddcfd1a"}, {"index": 138925853, "value": "0xf829a355"}, {"index": 86637313, "value": "0x6d5df2a9"}, {"index": 20736778, "value": "0x01ab8e44"}, {"index": 219665210, "value": "0x06d13507"}, {"index": 160430336, "value": "0xda8dcfc6"}, {"index": 264654675, "value": "0x01a703e1"}, {"index": 8013395, "value": "0xafe7d2c1"}, {"index": 228945585, "value": "0xc091c3a2"}, {"index": 213884386, "value": "0xac1814fe"}, {"index": 104419827, "value": "0x6e6ff62a"}, {"index": 44185464, "value": "0x8fdf01ba"}, {"index": 142737231, "value": "0xdd3f7159"}, {"index": 99284897, "value": "0xdfa0d75c"}, {"index": 132475900, "value": "0x26684c35"}, {"index": 61861762, "value": "0x7f441e63"}, {"index": 132056166, "value": "0x88df2570"}, {"index": 262388043, "value": "0x8aa4d5eb"}, {"index": 91878046, "value": "0xcc816c05"}, {"index": 117353561, "value": "0x434df890"}, {"index": 124768597, "value": "0xcd392ad6"}, {"index": 71352993, "value": "0x1ab4cb63"}, {"index": 190698941, "value": "0x595926fa"}, {"index": 46055428, "value": "0x7cd76b41"}, {"index": 55281366, "value": "0x20cb95c4"}, {"index": 165145231, "value": "0x13cf823f"}, {"index": 106810753, "value": "0xf9daf901"}, {"index": 171985651, "value": "0xff9af40a"}, {"index": 232085256, "value": "0x2c7dfa51"}, {"index": 159510492, "value": "0x871206db"}, {"index": 40072060, "value": "0x938c116c"}, {"index": 209107596, "value": "0xb64bf199"}, {"index": 39023794, "value": "0x5751f874"}], - "cache_head": ["0x355a86d2", "0x7957db1c", "0xd21772af", "0x6fc1e09b", "0xd55ce61d", "0x6e6a278b", "0xd3f543ce", "0x223d8e82", "0x143ab337", "0x2e9f05bd", "0x2eb389bf", "0x0c6e449e", "0x5cfa4222", "0xba6560fe", "0x8e3e1aa4", "0xdbcc1d53"], - "cache_last_line": ["0x41190d91", "0xbd277957", "0x22ddbb49", "0x6986f207", "0xdf69a4d6", "0x26401a3a", "0x818230fb", "0xc417122d", "0x3597b211", "0xb553ce55", "0xcf39cc0d", "0x3b7fc43a", "0x3fd43b00", "0x67e1c80e", "0xffa7ea7d", "0xca2960ab"], - "cache_fnv1a64": "0x48c4f5bf24166b2e" -} diff --git a/proto-cuda/packs-ca4/mx8_sh256x27/kernel.cl b/proto-cuda/packs-ca4/mx8_sh256x27/kernel.cl deleted file mode 100644 index 72f7ca4d0..000000000 --- a/proto-cuda/packs-ca4/mx8_sh256x27/kernel.cl +++ /dev/null @@ -1,538 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// OpenCL C twin of the Metal kernel for the same seed (see proto-opencl/README.md, WAVEFRONT.md and program.metal). -// Built from source at runtime by proto-opencl/host.c, which passes these defines: -// IGNEUM_GROUP work-group size of igneum_hash, a multiple of 32 (default 32: one work-group = one 32-lane unit) -// IGNEUM_EXCHANGE 0 = local-memory exchange with a barrier (any device, any wave width; the default) -// 1 = sub_group_shuffle_xor (cl_khr_subgroup_shuffle), only with IGNEUM_GROUP 32 and a sub-group size of exactly 32 -// 2 = intel_sub_group_shuffle_xor (cl_intel_subgroups), same condition -// The verification unit is always 32 lanes. A 64-wide hardware wave (AMD GCN/CDNA, RDNA in wave64) runs two units; -// the exchange masks are 1, 2, 4, 8, 16, so every partner lane lies inside the lane's own aligned run of 32. -#ifndef IGNEUM_GROUP -#define IGNEUM_GROUP 32 -#endif -#ifndef IGNEUM_EXCHANGE -#define IGNEUM_EXCHANGE 0 -#endif -#ifdef __OPENCL_VERSION__ -#define IGNEUM_KERNEL_HASH __kernel __attribute__((reqd_work_group_size(IGNEUM_GROUP, 1, 1))) -#define IGNEUM_LOCAL_WORDS(name, n) __local uint name[n] -#if IGNEUM_EXCHANGE == 1 -#ifdef cl_khr_subgroups -#pragma OPENCL EXTENSION cl_khr_subgroups : enable -#endif -#ifdef cl_khr_subgroup_shuffle -#pragma OPENCL EXTENSION cl_khr_subgroup_shuffle : enable -#endif -#elif IGNEUM_EXCHANGE == 2 -#pragma OPENCL EXTENSION cl_intel_subgroups : enable -#endif -#else -// Not an OpenCL compiler: proto-opencl/emu compiles this file as C++ and supplies the built-ins and these two macros. -#include "emu_opencl.h" -#endif - -#if IGNEUM_EXCHANGE == 1 -#define IGNEUM_SHFL_XOR(dst, a, m) dst = sub_group_shuffle_xor((a), (uint)(m)) -#define IGNEUM_BCAST0(dst, a) dst = sub_group_broadcast((a), 0u) -#elif IGNEUM_EXCHANGE == 2 -#define IGNEUM_SHFL_XOR(dst, a, m) dst = intel_sub_group_shuffle_xor((a), (uint)(m)) -#define IGNEUM_BCAST0(dst, a) dst = sub_group_broadcast((a), 0u) -#else -// Local-memory exchange. Two buffers of IGNEUM_GROUP words alternate (xk counts exchanges), so one barrier per -// exchange is enough: a lane can only overwrite buffer b at exchange k+2 after passing barrier k+1, and every lane -// reaches barrier k+1 only after its read of buffer b at exchange k. The partner lid ^ m stays inside the lane's -// aligned run of 32 because m < 32. Control flow is uniform, so every work-item reaches every barrier. -#define IGNEUM_SHFL_XOR(dst, a, m) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid ^ (uint)(m))]; xk += 1u; } -#define IGNEUM_BCAST0(dst, a) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid & ~31u)]; xk += 1u; } -#endif - -static inline uint splitmix32(uint x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -// n is a literal in 1..31 at every call site. OpenCL rotate() rotates left by n modulo 32. -static inline uint rotl_imm(uint x, uint n) { return rotate(x, n); } -// Right rotation by n modulo 32 as a left rotation by (32 - n) modulo 32; n == 0 gives x. -static inline uint rotr_var(uint x, uint n) { return rotate(x, (0u - n) & 31u); } -static inline uint ds_elem(uint i, uint d0, uint d1) { - uint x = i ^ d0; - x *= 0x9E3779B1u; x ^= x >> 15; - x += d1; - x *= 0x85EBCA77u; x ^= x >> 13; - x *= 0xC2B2AE3Du; x ^= x >> 16; - return x; -} - -// Memory-hard dataset core (MEMHARD.md). Cache: 2^26 words in 2^16 segments of 64 chained ChaCha12 lines. -// Item: 8 rounds of 8 x seed-parameterised mixer + one 64-byte cache read, then 8 x final mixer (class v3, mixer multiplier 8, -// docs/plans/mixer-x4.md: the round key of application j of round r is 0x9E3779B9 * (r * m + j + 1)). All parameters are literals. -#define MH_CACHE_LINE_MASK 0x003fffffu -#define MH_SEGMENT_LINES 64u -#define MH_QR(a, b, c, d, r1, r2, r3, r4) { a += b; d ^= a; d = mh_rotl(d, r1); c += d; b ^= c; b = mh_rotl(b, r2); a += b; d ^= a; d = mh_rotl(d, r3); c += d; b ^= c; b = mh_rotl(b, r4); } -static inline uint mh_rotl(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 at every call site - -// y = ChaCha12 core(x) + x -static inline void mh_chacha_block(const uint* x, uint* y) { - for (uint i = 0u; i < 16u; ++i) y[i] = x[i]; - for (uint r = 0u; r < 6u; ++r) { - MH_QR(y[0], y[4], y[8], y[12], 16u, 12u, 8u, 7u) MH_QR(y[1], y[5], y[9], y[13], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[6], y[10], y[14], 16u, 12u, 8u, 7u) MH_QR(y[3], y[7], y[11], y[15], 16u, 12u, 8u, 7u) - MH_QR(y[0], y[5], y[10], y[15], 16u, 12u, 8u, 7u) MH_QR(y[1], y[6], y[11], y[12], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[7], y[8], y[13], 16u, 12u, 8u, 7u) MH_QR(y[3], y[4], y[9], y[14], 16u, 12u, 8u, 7u) - } - for (uint i = 0u; i < 16u; ++i) y[i] += x[i]; -} - -// One cache segment: 64 chained lines written at cache[seg * 1024]. in_j = prev ^ (sigma || K || seg || j || tag), prev_0 = 0. -static inline void mh_cache_segment(__global uint* cache, uint seg) { - uint prev[16]; uint x[16]; uint y[16]; - for (uint i = 0u; i < 16u; ++i) prev[i] = 0u; - for (uint j = 0u; j < MH_SEGMENT_LINES; ++j) { - x[0] = 0x61707865u ^ prev[0]; x[1] = 0x3320646eu ^ prev[1]; x[2] = 0x79622d32u ^ prev[2]; x[3] = 0x6b206574u ^ prev[3]; - x[4] = 0x3067619fu ^ prev[4]; - x[5] = 0x3c269176u ^ prev[5]; - x[6] = 0x84a03b03u ^ prev[6]; - x[7] = 0xf8c63294u ^ prev[7]; - x[8] = 0xff977c5bu ^ prev[8]; - x[9] = 0xe60def3eu ^ prev[9]; - x[10] = 0x63630141u ^ prev[10]; - x[11] = 0xb8fbcb58u ^ prev[11]; - x[12] = seg ^ prev[12]; x[13] = j ^ prev[13]; x[14] = 0x49676e65u ^ prev[14]; x[15] = 0x756d4d48u ^ prev[15]; - mh_chacha_block(x, y); - __global uint* line = cache + ((seg * MH_SEGMENT_LINES + j) * 16u); - for (uint i = 0u; i < 16u; ++i) { line[i] = y[i]; prev[i] = y[i]; } - } -} - -// M_r: per word (s ^ (RC + rk)) * MUL, then a column round and a diagonal round with the seed-drawn rotations. -static inline void mh_mixer(uint* s, uint rk) { - s[0] = (s[0] ^ (0xbab68293u + rk)) * 0x42146205u; - s[1] = (s[1] ^ (0xcc162340u + rk)) * 0x52cbe0fbu; - s[2] = (s[2] ^ (0x6ce151ccu + rk)) * 0x7ecf4a03u; - s[3] = (s[3] ^ (0xe62b8997u + rk)) * 0x6728907fu; - s[4] = (s[4] ^ (0xc9c80297u + rk)) * 0xd81d9751u; - s[5] = (s[5] ^ (0xf74a1654u + rk)) * 0x132952c3u; - s[6] = (s[6] ^ (0x3d704af5u + rk)) * 0xf60de277u; - s[7] = (s[7] ^ (0x3cf522b7u + rk)) * 0x05358035u; - s[8] = (s[8] ^ (0x2b9cac04u + rk)) * 0xbaf6499du; - s[9] = (s[9] ^ (0xa880ac10u + rk)) * 0xe4db9667u; - s[10] = (s[10] ^ (0x13e5dd1du + rk)) * 0x3e98f45du; - s[11] = (s[11] ^ (0x6fc3e233u + rk)) * 0xd0004eddu; - s[12] = (s[12] ^ (0x2d83eeacu + rk)) * 0x2691630du; - s[13] = (s[13] ^ (0x9006e8bfu + rk)) * 0x9beb3bcfu; - s[14] = (s[14] ^ (0x2c4b5362u + rk)) * 0xab310379u; - s[15] = (s[15] ^ (0x31b49ee2u + rk)) * 0x99cfb423u; - MH_QR(s[0], s[4], s[8], s[12], 20u, 20u, 19u, 4u) MH_QR(s[1], s[5], s[9], s[13], 20u, 20u, 19u, 4u) - MH_QR(s[2], s[6], s[10], s[14], 20u, 20u, 19u, 4u) MH_QR(s[3], s[7], s[11], s[15], 20u, 20u, 19u, 4u) - MH_QR(s[0], s[5], s[10], s[15], 26u, 3u, 3u, 27u) MH_QR(s[1], s[6], s[11], s[12], 26u, 3u, 3u, 27u) - MH_QR(s[2], s[7], s[8], s[13], 26u, 3u, 3u, 27u) MH_QR(s[3], s[4], s[9], s[14], 26u, 3u, 3u, 27u) -} - -// Item t: 16 words. s = (K, t * MUL[i] + RC[i]); 8 rounds of 8 x mixer + cache line s[0] & mask; 8 x final mixer. -static inline void mh_item(__global const uint* cache, uint t, uint* s) { - s[0] = 0x3067619fu; - s[1] = 0x3c269176u; - s[2] = 0x84a03b03u; - s[3] = 0xf8c63294u; - s[4] = 0xff977c5bu; - s[5] = 0xe60def3eu; - s[6] = 0x63630141u; - s[7] = 0xb8fbcb58u; - s[8] = t * 0x42146205u + 0xbab68293u; - s[9] = t * 0x52cbe0fbu + 0xcc162340u; - s[10] = t * 0x7ecf4a03u + 0x6ce151ccu; - s[11] = t * 0x6728907fu + 0xe62b8997u; - s[12] = t * 0xd81d9751u + 0xc9c80297u; - s[13] = t * 0x132952c3u + 0xf74a1654u; - s[14] = t * 0xf60de277u + 0x3d704af5u; - s[15] = t * 0x05358035u + 0x3cf522b7u; - for (uint r = 0u; r < 8u; ++r) { - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (r * 8u + j + 1u)); - __global const uint* line = cache + ((s[0] & MH_CACHE_LINE_MASK) * 16u); - for (uint i = 0u; i < 16u; ++i) s[i] ^= line[i]; - } - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (64u + j + 1u)); -} -// dataset[w] without the dataset: derive item w >> 4 and take word w & 15. -static inline uint mh_word(__global const uint* cache, uint w) { uint s[16]; mh_item(cache, w >> 4u, s); return s[w & 15u]; } - -// Memory-hard dataset (MEMHARD.md). One work-item per cache segment; one work-item per 64-byte dataset item. -// The same constants as memhard.h in this pack (one emitter, three dialects). -__kernel void igneum_cache_fill(__global uint* cache, uint nSegments) { - uint seg = (uint)get_global_id(0); - if (seg < nSegments) mh_cache_segment(cache, seg); -} -__kernel void igneum_build(__global uint* ds, __global const uint* cache, uint nItems) { - uint t = (uint)get_global_id(0); - if (t < nItems) { - uint s[16]; - mh_item(cache, t, s); - __global uint* d = ds + ((ulong)t * 16u); - for (uint i = 0u; i < 16u; ++i) d[i] = s[i]; - } -} - -// One hash per work-item. IGNEUM_GROUP is a multiple of 32; lane = lid & 31 and every exchange stays inside the -// lane's own aligned run of 32 work-items, exactly like simd_shuffle_xor inside a 32-wide Metal SIMD group and -// __shfl_xor_sync inside a CUDA warp. Control flow is uniform (no branches at all). -IGNEUM_KERNEL_HASH void igneum_hash(__global const uint* ds, __global ulong* out, uint baseNonce, uint mask) { - uint gid = (uint)get_global_id(0); - uint lid = (uint)get_local_id(0); - uint nonce = baseNonce + gid; - uint r0, r1, r2, r3, r4, r5, r6, r7; -#if IGNEUM_EXCHANGE == 0 - IGNEUM_LOCAL_WORDS(xch, 2 * IGNEUM_GROUP); - uint xk = 0u; -#else - (void)lid; -#endif - { uint x = nonce ^ 0x67a9a7beu; x += 0x9e3779b9u; x = splitmix32(x); r0 = x ^ 0x1a155b25u; } // SEEDW[0], 0x9e3779b9u * 1u, SEEDW[1] - { uint x = nonce ^ 0x1a155b25u; x += 0x3c6ef372u; x = splitmix32(x); r1 = x ^ 0xfddfb732u; } // SEEDW[1], 0x9e3779b9u * 2u, SEEDW[2] - { uint x = nonce ^ 0xfddfb732u; x += 0xdaa66d2bu; x = splitmix32(x); r2 = x ^ 0x4b5af2e8u; } // SEEDW[2], 0x9e3779b9u * 3u, SEEDW[3] - { uint x = nonce ^ 0x4b5af2e8u; x += 0x78dde6e4u; x = splitmix32(x); r3 = x ^ 0xc55caf33u; } // SEEDW[3], 0x9e3779b9u * 4u, SEEDW[4] - { uint x = nonce ^ 0xc55caf33u; x += 0x1715609du; x = splitmix32(x); r4 = x ^ 0xa27c13b7u; } // SEEDW[4], 0x9e3779b9u * 5u, SEEDW[5] - { uint x = nonce ^ 0xa27c13b7u; x += 0xb54cda56u; x = splitmix32(x); r5 = x ^ 0x06628a48u; } // SEEDW[5], 0x9e3779b9u * 6u, SEEDW[6] - { uint x = nonce ^ 0x06628a48u; x += 0x5384540fu; x = splitmix32(x); r6 = x ^ 0x03852469u; } // SEEDW[6], 0x9e3779b9u * 7u, SEEDW[7] - { uint x = nonce ^ 0x03852469u; x += 0xf1bbcdc8u; x = splitmix32(x); r7 = x ^ 0x67a9a7beu; } // SEEDW[7], 0x9e3779b9u * 8u, SEEDW[0] - - for (uint it = 0u; it < 8u; ++it) { - uint sel = r0; - r2 = r3 * r4 + r2; // 0 mad - r2 = r1 * r1 + r2; // 1 mad - r2 = r3 * r2 + r2; // 2 mad - r3 = r3 ^ r5; // 3 xor - r7 = r7 ^ ds[r2 & mask]; // 4 load - r5 = r5 ^ ds[r7 & mask]; // 5 load - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 8u); r1 = r1 ^ t_; } // 6 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 8u); r7 = r7 ^ t_; } // 7 shfl - r1 = mul_hi(r1, r5); // 8 mulhi - r6 = rotr_var(r6, r3); // 9 rotr - r3 = r3 | r4; // 10 or - r4 = r4 ^ ds[r3 & mask]; // 11 load - r0 = mul_hi(r0, r4); // 12 mulhi - r5 = r5 + r1 + ((((sel >> 30u) & 1u) != 0u) ? 0xd3177981u : 0xc7934706u); // 13 add - r0 = r0 ^ ds[r4 & mask]; // 14 load - r2 = r2 - r4; // 15 sub - r2 = r2 ^ ds[r0 & mask]; // 16 load - r7 = r7 ^ ds[r2 & mask]; // 17 load - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 4u); r7 = r7 ^ t_; } // 18 shfl - r5 = r5 * r0; // 19 mul - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 2u); r3 = r3 ^ t_; } // 20 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 16u); r2 = r2 ^ t_; } // 21 shfl - r6 = mul_hi(r6, r2); // 22 mulhi - r6 = r6 ^ ds[r1 & mask]; // 23 load - r5 = r5 * r0; // 24 mul - r5 = rotl_imm(r5, 19u); // 25 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 2u); r7 = r7 ^ t_; } // 26 shfl - r0 = r0 ^ r5; // 27 xor - r0 = r0 ^ r4; // 28 xor - r3 = r3 - r0; // 29 sub - r5 = r5 * r1; // 30 mul - r7 = r7 ^ ds[r2 & mask]; // 31 load - r1 = r1 ^ ds[r0 & mask]; // 32 load - r5 = r5 ^ r6; // 33 xor - r5 = r5 ^ ds[r1 & mask]; // 34 load - r0 = mul_hi(r0, r5); // 35 mulhi - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 4u); r5 = r5 ^ t_; } // 36 shfl - r7 = r7 ^ ds[r0 & mask]; // 37 load - r3 = r3 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0x230c005cu : 0x75ba2fadu); // 38 add - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 4u); r1 = r1 ^ t_; } // 39 shfl - r2 = r2 ^ r5; // 40 xor - r3 = r6 * r3 + r3; // 41 mad - r6 = r6 - r7; // 42 sub - r7 = r7 ^ r0; // 43 xor - r1 = r1 ^ ds[r7 & mask]; // 44 load - r2 = r2 * r3; // 45 mul - r1 = mul_hi(r1, r5); // 46 mulhi - r4 = r4 - r3; // 47 sub - r2 = rotr_var(r2, r6); // 48 rotr - r3 = r3 ^ ds[r5 & mask]; // 49 load - r1 = r1 + r5 + ((((sel >> 7u) & 1u) != 0u) ? 0x1907970cu : 0x81b8bc2cu); // 50 add - r0 = r0 * r2; // 51 mul - r0 = r0 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x699fd448u : 0x4f92b968u); // 52 add - r1 = r1 + r0 + ((((sel >> 12u) & 1u) != 0u) ? 0x77b1520du : 0x2bb965afu); // 53 add - r7 = rotl_imm(r7, 14u); // 54 rotl - r3 = r3 + r7 + ((((sel >> 1u) & 1u) != 0u) ? 0xa54c55a0u : 0x7b0fe07au); // 55 add - r6 = r6 ^ ds[r7 & mask]; // 56 load - r1 = rotr_var(r1, r5); // 57 rotr - r5 = r5 ^ ds[r4 & mask]; // 58 load - r6 = r6 ^ ds[r2 & mask]; // 59 load - r3 = r5 * r0 + r3; // 60 mad - r5 = r5 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0xad7493e7u : 0xaf9dd72du); // 61 add - r4 = r4 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x1e07c3d9u : 0x89841d87u); // 62 add - r5 = rotl_imm(r5, 19u); // 63 rotl - // latency-shadow block (Counter ASIC 3.0 item 8): 256 ALU instructions x 27 passes after instruction 63, no load - for (uint sh = 0u; sh < 27u; ++sh) { - r5 = r5 + r2 + ((((sel >> 18u) & 1u) != 0u) ? 0x8bc12da9u : 0x92199f99u); // s0 add - r0 = r0 + r7 + ((((sel >> 26u) & 1u) != 0u) ? 0x63079e5au : 0x8d72d3adu); // s1 add - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 2u); r6 = r6 ^ t_; } // s2 shfl - r4 = r4 - r2; // s3 sub - r7 = r7 + r0 + ((((sel >> 31u) & 1u) != 0u) ? 0x5d4c7a60u : 0xb21b4babu); // s4 add - r0 = rotl_imm(r0, 11u); // s5 rotl - r0 = r0 + r1 + ((((sel >> 16u) & 1u) != 0u) ? 0xc88e2942u : 0x2fe0e98bu); // s6 add - r7 = r7 ^ r1; // s7 xor - r1 = r6 * r5 + r1; // s8 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 4u); r6 = r6 ^ t_; } // s9 shfl - r1 = r2 * r2 + r1; // s10 mad - r5 = r0 * r3 + r5; // s11 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 4u); r2 = r2 ^ t_; } // s12 shfl - r1 = r1 + r5 + ((((sel >> 25u) & 1u) != 0u) ? 0xbc48c63eu : 0xbdf8f9a5u); // s13 add - r1 = rotl_imm(r1, 29u); // s14 rotl - r1 = r1 - r4; // s15 sub - r7 = r7 | r1; // s16 or - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 8u); r2 = r2 ^ t_; } // s17 shfl - r7 = r7 ^ r4; // s18 xor - r6 = r6 * r1; // s19 mul - r5 = r6 * r0 + r5; // s20 mad - r3 = r3 - r1; // s21 sub - r6 = r6 * r0; // s22 mul - r2 = r2 + r0 + ((((sel >> 1u) & 1u) != 0u) ? 0x96e8f127u : 0x45c37cecu); // s23 add - r6 = r6 - r4; // s24 sub - r7 = r3 * r4 + r7; // s25 mad - r3 = rotl_imm(r3, 9u); // s26 rotl - r2 = r2 - r1; // s27 sub - r6 = mul_hi(r6, r0); // s28 mulhi - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 2u); r2 = r2 ^ t_; } // s29 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 1u); r1 = r1 ^ t_; } // s30 shfl - r1 = r1 + r6 + ((((sel >> 1u) & 1u) != 0u) ? 0xb1cdb2abu : 0x37985632u); // s31 add - r0 = rotr_var(r0, r1); // s32 rotr - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 2u); r3 = r3 ^ t_; } // s33 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 4u); r0 = r0 ^ t_; } // s34 shfl - r7 = r7 * r0; // s35 mul - r3 = r3 + r1 + ((((sel >> 0u) & 1u) != 0u) ? 0x856e0180u : 0x804e777eu); // s36 add - r1 = r1 ^ r6; // s37 xor - r2 = r2 * r7; // s38 mul - r6 = r6 + r5 + ((((sel >> 2u) & 1u) != 0u) ? 0xe9bd0cc4u : 0x0b06c8a7u); // s39 add - r5 = r5 * r7; // s40 mul - r2 = mul_hi(r2, r3); // s41 mulhi - r2 = r2 ^ r0; // s42 xor - r0 = rotl_imm(r0, 4u); // s43 rotl - r4 = mul_hi(r4, r3); // s44 mulhi - r6 = r6 + r7 + ((((sel >> 12u) & 1u) != 0u) ? 0xcdcb63f6u : 0xee822e17u); // s45 add - r3 = r2 * r0 + r3; // s46 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 8u); r4 = r4 ^ t_; } // s47 shfl - r6 = mul_hi(r6, r5); // s48 mulhi - r0 = r0 - r2; // s49 sub - r3 = r3 - r5; // s50 sub - r1 = rotr_var(r1, r4); // s51 rotr - r6 = r7 * r7 + r6; // s52 mad - r5 = r5 ^ r3; // s53 xor - r1 = r1 - r0; // s54 sub - r5 = r5 - r6; // s55 sub - r3 = r3 + r2 + ((((sel >> 10u) & 1u) != 0u) ? 0xd4758987u : 0x3dfad1b6u); // s56 add - r4 = r4 + r1 + ((((sel >> 0u) & 1u) != 0u) ? 0x0602d6beu : 0xfca75bc2u); // s57 add - r5 = mul_hi(r5, r6); // s58 mulhi - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 2u); r2 = r2 ^ t_; } // s59 shfl - r2 = rotl_imm(r2, 9u); // s60 rotl - r4 = r4 | r6; // s61 or - r6 = rotr_var(r6, r4); // s62 rotr - r2 = r2 * r4; // s63 mul - r0 = r0 ^ r5; // s64 xor - r2 = r2 ^ r7; // s65 xor - r2 = r2 + r1 + ((((sel >> 6u) & 1u) != 0u) ? 0x8596687au : 0xd71c02ffu); // s66 add - r1 = rotl_imm(r1, 21u); // s67 rotl - r3 = r3 * r2; // s68 mul - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 2u); r7 = r7 ^ t_; } // s69 shfl - r3 = r3 * r2; // s70 mul - r0 = r2 * r5 + r0; // s71 mad - r6 = r6 + r7 + ((((sel >> 0u) & 1u) != 0u) ? 0x08ac5733u : 0x3c6fe15du); // s72 add - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 8u); r3 = r3 ^ t_; } // s73 shfl - r3 = r3 * r6; // s74 mul - r6 = r6 ^ r7; // s75 xor - r3 = r3 | r0; // s76 or - r2 = r2 + r4 + ((((sel >> 1u) & 1u) != 0u) ? 0xc100b495u : 0x295d5faeu); // s77 add - r3 = mul_hi(r3, r7); // s78 mulhi - r4 = r4 | r1; // s79 or - r4 = rotr_var(r4, r3); // s80 rotr - r4 = r4 + r3 + ((((sel >> 15u) & 1u) != 0u) ? 0x5cc59530u : 0x274a9221u); // s81 add - r7 = r7 + r3 + ((((sel >> 5u) & 1u) != 0u) ? 0x141479ecu : 0xc8651f8eu); // s82 add - r3 = rotr_var(r3, r6); // s83 rotr - r2 = r4 * r7 + r2; // s84 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 16u); r2 = r2 ^ t_; } // s85 shfl - r3 = r3 ^ r2; // s86 xor - { uint t_; IGNEUM_SHFL_XOR(t_, r7, 2u); r5 = r5 ^ t_; } // s87 shfl - r0 = r0 ^ r4; // s88 xor - r3 = r3 + r2 + ((((sel >> 18u) & 1u) != 0u) ? 0x883c0c92u : 0x53f915c2u); // s89 add - r7 = rotr_var(r7, r5); // s90 rotr - r3 = r3 + r1 + ((((sel >> 31u) & 1u) != 0u) ? 0x3cc5bd20u : 0xe2be00a5u); // s91 add - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 1u); r7 = r7 ^ t_; } // s92 shfl - r6 = rotr_var(r6, r2); // s93 rotr - r7 = rotl_imm(r7, 14u); // s94 rotl - r4 = r5 * r5 + r4; // s95 mad - r2 = r4 * r5 + r2; // s96 mad - r1 = r1 ^ r0; // s97 xor - r5 = r5 * r4; // s98 mul - r2 = r2 - r0; // s99 sub - r7 = rotl_imm(r7, 30u); // s100 rotl - r5 = r5 + r6 + ((((sel >> 17u) & 1u) != 0u) ? 0xbfd646cbu : 0x423fd9c9u); // s101 add - r7 = r7 + r6 + ((((sel >> 11u) & 1u) != 0u) ? 0x19b74a43u : 0x8d3c011du); // s102 add - r5 = rotl_imm(r5, 6u); // s103 rotl - r0 = r0 * r4; // s104 mul - r0 = r0 | r5; // s105 or - r0 = r0 + r1 + ((((sel >> 15u) & 1u) != 0u) ? 0x7dcb7e18u : 0x8ce14721u); // s106 add - r0 = rotl_imm(r0, 15u); // s107 rotl - r4 = r4 - r2; // s108 sub - r2 = mul_hi(r2, r0); // s109 mulhi - r1 = mul_hi(r1, r0); // s110 mulhi - r0 = r0 + r2 + ((((sel >> 28u) & 1u) != 0u) ? 0x3c179ad8u : 0x2d9fc6b9u); // s111 add - r2 = mul_hi(r2, r0); // s112 mulhi - r6 = r6 - r3; // s113 sub - r7 = r6 * r3 + r7; // s114 mad - r5 = rotr_var(r5, r1); // s115 rotr - r6 = rotr_var(r6, r4); // s116 rotr - r2 = r2 + r1 + ((((sel >> 22u) & 1u) != 0u) ? 0x47359729u : 0x502138e2u); // s117 add - r3 = r2 * r0 + r3; // s118 mad - r1 = r1 * r3; // s119 mul - r2 = r0 * r4 + r2; // s120 mad - r0 = r0 * r3; // s121 mul - r2 = mul_hi(r2, r0); // s122 mulhi - r5 = r5 * r6; // s123 mul - r4 = r2 * r4 + r4; // s124 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 2u); r4 = r4 ^ t_; } // s125 shfl - r2 = r2 ^ r0; // s126 xor - r1 = rotl_imm(r1, 7u); // s127 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 4u); r7 = r7 ^ t_; } // s128 shfl - r6 = rotl_imm(r6, 17u); // s129 rotl - r2 = r2 + r5 + ((((sel >> 15u) & 1u) != 0u) ? 0x6c57e4e7u : 0x33dff776u); // s130 add - r0 = r0 | r3; // s131 or - r5 = rotr_var(r5, r0); // s132 rotr - r2 = r2 + r3 + ((((sel >> 30u) & 1u) != 0u) ? 0xe8212c0cu : 0x5db25b34u); // s133 add - r4 = r4 ^ r3; // s134 xor - r6 = r6 ^ r1; // s135 xor - r1 = r1 - r7; // s136 sub - r2 = r6 * r2 + r2; // s137 mad - r0 = mul_hi(r0, r5); // s138 mulhi - r2 = r2 + r7 + ((((sel >> 10u) & 1u) != 0u) ? 0xd44ff710u : 0x218d4090u); // s139 add - r1 = rotr_var(r1, r4); // s140 rotr - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 1u); r3 = r3 ^ t_; } // s141 shfl - r7 = r6 * r2 + r7; // s142 mad - r2 = r2 * r3; // s143 mul - r7 = r7 + r4 + ((((sel >> 29u) & 1u) != 0u) ? 0xb98942fau : 0xf4b1a8deu); // s144 add - r7 = rotl_imm(r7, 15u); // s145 rotl - r7 = r7 ^ r5; // s146 xor - r4 = r7 * r4 + r4; // s147 mad - r6 = rotr_var(r6, r5); // s148 rotr - r1 = r2 * r4 + r1; // s149 mad - r1 = r1 + r7 + ((((sel >> 17u) & 1u) != 0u) ? 0x0d48ba42u : 0x2bef10f2u); // s150 add - r5 = r5 ^ r4; // s151 xor - r7 = r7 + r0 + ((((sel >> 30u) & 1u) != 0u) ? 0xc98dea9cu : 0xdc5cc080u); // s152 add - r4 = r4 * r6; // s153 mul - r0 = r0 * r2; // s154 mul - r6 = r6 ^ r5; // s155 xor - r4 = rotr_var(r4, r2); // s156 rotr - r1 = rotl_imm(r1, 11u); // s157 rotl - r5 = r5 + r0 + ((((sel >> 11u) & 1u) != 0u) ? 0x6c752dcbu : 0x18197438u); // s158 add - r4 = r1 * r5 + r4; // s159 mad - r4 = rotl_imm(r4, 26u); // s160 rotl - r3 = r0 * r7 + r3; // s161 mad - r3 = rotr_var(r3, r1); // s162 rotr - r4 = r4 + r0 + ((((sel >> 3u) & 1u) != 0u) ? 0x8e481727u : 0xf1c46574u); // s163 add - r0 = mul_hi(r0, r4); // s164 mulhi - r2 = r2 + r6 + ((((sel >> 20u) & 1u) != 0u) ? 0x550ab406u : 0xac578137u); // s165 add - r0 = rotl_imm(r0, 13u); // s166 rotl - r3 = r3 + r1 + ((((sel >> 14u) & 1u) != 0u) ? 0xaebb5966u : 0xa33e6706u); // s167 add - r3 = r3 | r5; // s168 or - r6 = rotr_var(r6, r2); // s169 rotr - r4 = r4 ^ r6; // s170 xor - r6 = r6 - r1; // s171 sub - r7 = rotl_imm(r7, 22u); // s172 rotl - r5 = rotl_imm(r5, 15u); // s173 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r0, 8u); r7 = r7 ^ t_; } // s174 shfl - r0 = r0 ^ r5; // s175 xor - r7 = rotl_imm(r7, 6u); // s176 rotl - r7 = r7 - r0; // s177 sub - r3 = rotl_imm(r3, 30u); // s178 rotl - r7 = r6 * r1 + r7; // s179 mad - r6 = rotl_imm(r6, 9u); // s180 rotl - r2 = r2 ^ r4; // s181 xor - r2 = r2 ^ r7; // s182 xor - r7 = r7 ^ r2; // s183 xor - r1 = r1 + r2 + ((((sel >> 21u) & 1u) != 0u) ? 0x5bb7550fu : 0xd94d55acu); // s184 add - r3 = r3 | r5; // s185 or - r6 = mul_hi(r6, r3); // s186 mulhi - r4 = r4 | r0; // s187 or - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 2u); r7 = r7 ^ t_; } // s188 shfl - r6 = r6 + r5 + ((((sel >> 6u) & 1u) != 0u) ? 0x2a354e2du : 0x89e747feu); // s189 add - r1 = r1 ^ r4; // s190 xor - r7 = mul_hi(r7, r4); // s191 mulhi - r2 = rotl_imm(r2, 26u); // s192 rotl - r5 = rotl_imm(r5, 8u); // s193 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 16u); r4 = r4 ^ t_; } // s194 shfl - r4 = r4 ^ r5; // s195 xor - r1 = r1 * r3; // s196 mul - r5 = r5 + r2 + ((((sel >> 7u) & 1u) != 0u) ? 0x10cfdc71u : 0xea03e8e7u); // s197 add - r7 = r7 + r5 + ((((sel >> 15u) & 1u) != 0u) ? 0x66e148d3u : 0xa7ee0102u); // s198 add - r5 = r5 - r2; // s199 sub - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 2u); r5 = r5 ^ t_; } // s200 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 8u); r7 = r7 ^ t_; } // s201 shfl - r4 = rotr_var(r4, r5); // s202 rotr - r7 = r7 ^ r5; // s203 xor - r7 = r7 ^ r0; // s204 xor - r6 = r6 + r2 + ((((sel >> 10u) & 1u) != 0u) ? 0x8558b619u : 0x8379a4deu); // s205 add - r4 = rotl_imm(r4, 13u); // s206 rotl - r1 = mul_hi(r1, r3); // s207 mulhi - r1 = r4 * r4 + r1; // s208 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r0, 4u); r2 = r2 ^ t_; } // s209 shfl - r7 = r7 + r0 + ((((sel >> 11u) & 1u) != 0u) ? 0xf4049c4cu : 0xaa8cb14eu); // s210 add - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 16u); r7 = r7 ^ t_; } // s211 shfl - r1 = r1 ^ r0; // s212 xor - r7 = rotl_imm(r7, 3u); // s213 rotl - r4 = rotr_var(r4, r7); // s214 rotr - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 16u); r3 = r3 ^ t_; } // s215 shfl - r5 = r5 | r1; // s216 or - r1 = r1 - r2; // s217 sub - r6 = r6 - r5; // s218 sub - r6 = rotl_imm(r6, 4u); // s219 rotl - r2 = mul_hi(r2, r0); // s220 mulhi - r2 = r2 | r0; // s221 or - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 8u); r5 = r5 ^ t_; } // s222 shfl - r5 = mul_hi(r5, r6); // s223 mulhi - r0 = r0 - r6; // s224 sub - r7 = rotl_imm(r7, 23u); // s225 rotl - r4 = r4 | r2; // s226 or - r2 = r2 * r4; // s227 mul - r3 = rotl_imm(r3, 12u); // s228 rotl - r0 = rotr_var(r0, r4); // s229 rotr - r0 = r0 + r6 + ((((sel >> 16u) & 1u) != 0u) ? 0x2a7fecb2u : 0x1d176220u); // s230 add - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 4u); r1 = r1 ^ t_; } // s231 shfl - r2 = r2 * r1; // s232 mul - r7 = r0 * r1 + r7; // s233 mad - r5 = rotl_imm(r5, 22u); // s234 rotl - r4 = rotr_var(r4, r6); // s235 rotr - r0 = r5 * r1 + r0; // s236 mad - r6 = r6 ^ r4; // s237 xor - r4 = r4 ^ r6; // s238 xor - r6 = rotl_imm(r6, 18u); // s239 rotl - r4 = r4 + r7 + ((((sel >> 25u) & 1u) != 0u) ? 0xadce39f3u : 0x17dafb4du); // s240 add - r0 = r0 - r7; // s241 sub - r1 = rotr_var(r1, r6); // s242 rotr - r3 = r3 + r0 + ((((sel >> 29u) & 1u) != 0u) ? 0x0e7033b6u : 0xf2f77d26u); // s243 add - r2 = r7 * r4 + r2; // s244 mad - r4 = r0 * r6 + r4; // s245 mad - r5 = r5 * r7; // s246 mul - r3 = r3 + r5 + ((((sel >> 31u) & 1u) != 0u) ? 0x7b2ff6b7u : 0xfed2da4eu); // s247 add - r0 = r0 + r7 + ((((sel >> 0u) & 1u) != 0u) ? 0xb5fad7b1u : 0x03b2891cu); // s248 add - r5 = mul_hi(r5, r4); // s249 mulhi - r5 = r5 + r2 + ((((sel >> 11u) & 1u) != 0u) ? 0xa7e71c2fu : 0x042cd6e3u); // s250 add - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 1u); r6 = r6 ^ t_; } // s251 shfl - r6 = r6 ^ r1; // s252 xor - r2 = r2 * r5; // s253 mul - r0 = r0 + r6 + ((((sel >> 16u) & 1u) != 0u) ? 0xb83d78deu : 0x784a302bu); // s254 add - r2 = r2 - r4; // s255 sub - } - } - uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((ulong)hi << 32) | (ulong)lo; -} - -#if IGNEUM_EXCHANGE != 0 -// Reports the sub-group size this device uses for a work-group of IGNEUM_GROUP items. host.c runs it only when the -// per-kernel query (clGetKernelSubGroupInfoKHR on igneum_hash) is unavailable; that query is preferred because a -// compiler may pick a different wave width per kernel (RDNA: wave32 or wave64). See WAVEFRONT.md. -IGNEUM_KERNEL_HASH void igneum_probe_subgroup(__global uint* out) { - if (get_local_id(0) == 0u) { out[0] = get_sub_group_size(); out[1] = get_num_sub_groups(); } -} -#endif diff --git a/proto-cuda/packs-ca4/mx8_sh256x27/kernel.cu b/proto-cuda/packs-ca4/mx8_sh256x27/kernel.cu deleted file mode 100644 index 62d29f6ff..000000000 --- a/proto-cuda/packs-ca4/mx8_sh256x27/kernel.cu +++ /dev/null @@ -1,423 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// Bit-exact twin of the Metal kernel for the same seed (see proto-cuda/CHECKLIST.md and program.metal). -// Compiled ahead of time by nvcc together with proto-cuda/host.cu. No NVRTC. -#include -#include -#include "program.h" -#include "memhard.h" - -__device__ __forceinline__ uint32_t splitmix32(uint32_t x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -// n is a literal in 1..31 at every call site, so both shift amounts are in 1..31. -__device__ __forceinline__ uint32_t rotl_imm(uint32_t x, uint32_t n) { return (x << n) | (x >> (32u - n)); } -// n is masked to 0..31; the second shift amount is masked too, so n == 0 gives x. -__device__ __forceinline__ uint32_t rotr_var(uint32_t x, uint32_t n) { n &= 31u; return (x >> n) | (x << ((32u - n) & 31u)); } -__device__ __forceinline__ uint32_t ds_elem(uint32_t i, uint32_t d0, uint32_t d1) { - uint32_t x = i ^ d0; - x *= 0x9E3779B1u; x ^= x >> 15; - x += d1; - x *= 0x85EBCA77u; x ^= x >> 13; - x *= 0xC2B2AE3Du; x ^= x >> 16; - return x; -} - -// Memory-hard dataset (MEMHARD.md). One thread per cache segment; one thread per 64-byte dataset item. -// The core functions (mh_cache_segment, mh_item) are in memhard.h and are also compiled for the host. -__global__ void igneum_cache_fill(uint32_t* cache, uint32_t nSegments) { - uint32_t seg = blockIdx.x * blockDim.x + threadIdx.x; - if (seg < nSegments) mh_cache_segment(cache, seg); -} -__global__ void igneum_build(uint32_t* ds, const uint32_t* cache, uint32_t nItems) { - uint32_t t = blockIdx.x * blockDim.x + threadIdx.x; - if (t < nItems) { - uint32_t s[16]; - mh_item(cache, t, s); - uint32_t* d = ds + (size_t)t * 16u; - for (uint32_t i = 0u; i < 16u; ++i) d[i] = s[i]; - } -} - -// One hash per thread. blockDim.x is a multiple of 32; lane = threadIdx.x & 31 and every -// __shfl_xor_sync stays inside the lane's own warp, exactly like simd_shuffle_xor inside a -// 32-wide Metal SIMD group. Control flow is uniform, so the full 0xffffffff member mask is valid. -__global__ void igneum_hash(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask) { - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t nonce = baseNonce + gid; - uint32_t r0, r1, r2, r3, r4, r5, r6, r7; - { uint32_t x = nonce ^ 0x67a9a7beu; x += 0x9e3779b9u; x = splitmix32(x); r0 = x ^ 0x1a155b25u; } // SEEDW[0], 0x9e3779b9u * 1u, SEEDW[1] - { uint32_t x = nonce ^ 0x1a155b25u; x += 0x3c6ef372u; x = splitmix32(x); r1 = x ^ 0xfddfb732u; } // SEEDW[1], 0x9e3779b9u * 2u, SEEDW[2] - { uint32_t x = nonce ^ 0xfddfb732u; x += 0xdaa66d2bu; x = splitmix32(x); r2 = x ^ 0x4b5af2e8u; } // SEEDW[2], 0x9e3779b9u * 3u, SEEDW[3] - { uint32_t x = nonce ^ 0x4b5af2e8u; x += 0x78dde6e4u; x = splitmix32(x); r3 = x ^ 0xc55caf33u; } // SEEDW[3], 0x9e3779b9u * 4u, SEEDW[4] - { uint32_t x = nonce ^ 0xc55caf33u; x += 0x1715609du; x = splitmix32(x); r4 = x ^ 0xa27c13b7u; } // SEEDW[4], 0x9e3779b9u * 5u, SEEDW[5] - { uint32_t x = nonce ^ 0xa27c13b7u; x += 0xb54cda56u; x = splitmix32(x); r5 = x ^ 0x06628a48u; } // SEEDW[5], 0x9e3779b9u * 6u, SEEDW[6] - { uint32_t x = nonce ^ 0x06628a48u; x += 0x5384540fu; x = splitmix32(x); r6 = x ^ 0x03852469u; } // SEEDW[6], 0x9e3779b9u * 7u, SEEDW[7] - { uint32_t x = nonce ^ 0x03852469u; x += 0xf1bbcdc8u; x = splitmix32(x); r7 = x ^ 0x67a9a7beu; } // SEEDW[7], 0x9e3779b9u * 8u, SEEDW[0] - - for (uint32_t it = 0u; it < 8u; ++it) { - uint32_t sel = r0; - r2 = r3 * r4 + r2; // 0 mad - r2 = r1 * r1 + r2; // 1 mad - r2 = r3 * r2 + r2; // 2 mad - r3 = r3 ^ r5; // 3 xor - r7 = r7 ^ ds[r2 & mask]; // 4 load - r5 = r5 ^ ds[r7 & mask]; // 5 load - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r4, 8); // 6 shfl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r3, 8); // 7 shfl - r1 = __umulhi(r1, r5); // 8 mulhi - r6 = rotr_var(r6, r3); // 9 rotr - r3 = r3 | r4; // 10 or - r4 = r4 ^ ds[r3 & mask]; // 11 load - r0 = __umulhi(r0, r4); // 12 mulhi - r5 = r5 + r1 + ((((sel >> 30u) & 1u) != 0u) ? 0xd3177981u : 0xc7934706u); // 13 add - r0 = r0 ^ ds[r4 & mask]; // 14 load - r2 = r2 - r4; // 15 sub - r2 = r2 ^ ds[r0 & mask]; // 16 load - r7 = r7 ^ ds[r2 & mask]; // 17 load - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r3, 4); // 18 shfl - r5 = r5 * r0; // 19 mul - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r4, 2); // 20 shfl - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r4, 16); // 21 shfl - r6 = __umulhi(r6, r2); // 22 mulhi - r6 = r6 ^ ds[r1 & mask]; // 23 load - r5 = r5 * r0; // 24 mul - r5 = rotl_imm(r5, 19u); // 25 rotl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r6, 2); // 26 shfl - r0 = r0 ^ r5; // 27 xor - r0 = r0 ^ r4; // 28 xor - r3 = r3 - r0; // 29 sub - r5 = r5 * r1; // 30 mul - r7 = r7 ^ ds[r2 & mask]; // 31 load - r1 = r1 ^ ds[r0 & mask]; // 32 load - r5 = r5 ^ r6; // 33 xor - r5 = r5 ^ ds[r1 & mask]; // 34 load - r0 = __umulhi(r0, r5); // 35 mulhi - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r2, 4); // 36 shfl - r7 = r7 ^ ds[r0 & mask]; // 37 load - r3 = r3 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0x230c005cu : 0x75ba2fadu); // 38 add - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r5, 4); // 39 shfl - r2 = r2 ^ r5; // 40 xor - r3 = r6 * r3 + r3; // 41 mad - r6 = r6 - r7; // 42 sub - r7 = r7 ^ r0; // 43 xor - r1 = r1 ^ ds[r7 & mask]; // 44 load - r2 = r2 * r3; // 45 mul - r1 = __umulhi(r1, r5); // 46 mulhi - r4 = r4 - r3; // 47 sub - r2 = rotr_var(r2, r6); // 48 rotr - r3 = r3 ^ ds[r5 & mask]; // 49 load - r1 = r1 + r5 + ((((sel >> 7u) & 1u) != 0u) ? 0x1907970cu : 0x81b8bc2cu); // 50 add - r0 = r0 * r2; // 51 mul - r0 = r0 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x699fd448u : 0x4f92b968u); // 52 add - r1 = r1 + r0 + ((((sel >> 12u) & 1u) != 0u) ? 0x77b1520du : 0x2bb965afu); // 53 add - r7 = rotl_imm(r7, 14u); // 54 rotl - r3 = r3 + r7 + ((((sel >> 1u) & 1u) != 0u) ? 0xa54c55a0u : 0x7b0fe07au); // 55 add - r6 = r6 ^ ds[r7 & mask]; // 56 load - r1 = rotr_var(r1, r5); // 57 rotr - r5 = r5 ^ ds[r4 & mask]; // 58 load - r6 = r6 ^ ds[r2 & mask]; // 59 load - r3 = r5 * r0 + r3; // 60 mad - r5 = r5 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0xad7493e7u : 0xaf9dd72du); // 61 add - r4 = r4 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x1e07c3d9u : 0x89841d87u); // 62 add - r5 = rotl_imm(r5, 19u); // 63 rotl - // latency-shadow block (Counter ASIC 3.0 item 8): 256 ALU instructions x 27 passes after instruction 63, no load - for (uint32_t sh = 0u; sh < 27u; ++sh) { - r5 = r5 + r2 + ((((sel >> 18u) & 1u) != 0u) ? 0x8bc12da9u : 0x92199f99u); // s0 add - r0 = r0 + r7 + ((((sel >> 26u) & 1u) != 0u) ? 0x63079e5au : 0x8d72d3adu); // s1 add - r6 = r6 ^ __shfl_xor_sync(0xffffffffu, r3, 2); // s2 shfl - r4 = r4 - r2; // s3 sub - r7 = r7 + r0 + ((((sel >> 31u) & 1u) != 0u) ? 0x5d4c7a60u : 0xb21b4babu); // s4 add - r0 = rotl_imm(r0, 11u); // s5 rotl - r0 = r0 + r1 + ((((sel >> 16u) & 1u) != 0u) ? 0xc88e2942u : 0x2fe0e98bu); // s6 add - r7 = r7 ^ r1; // s7 xor - r1 = r6 * r5 + r1; // s8 mad - r6 = r6 ^ __shfl_xor_sync(0xffffffffu, r1, 4); // s9 shfl - r1 = r2 * r2 + r1; // s10 mad - r5 = r0 * r3 + r5; // s11 mad - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r6, 4); // s12 shfl - r1 = r1 + r5 + ((((sel >> 25u) & 1u) != 0u) ? 0xbc48c63eu : 0xbdf8f9a5u); // s13 add - r1 = rotl_imm(r1, 29u); // s14 rotl - r1 = r1 - r4; // s15 sub - r7 = r7 | r1; // s16 or - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r4, 8); // s17 shfl - r7 = r7 ^ r4; // s18 xor - r6 = r6 * r1; // s19 mul - r5 = r6 * r0 + r5; // s20 mad - r3 = r3 - r1; // s21 sub - r6 = r6 * r0; // s22 mul - r2 = r2 + r0 + ((((sel >> 1u) & 1u) != 0u) ? 0x96e8f127u : 0x45c37cecu); // s23 add - r6 = r6 - r4; // s24 sub - r7 = r3 * r4 + r7; // s25 mad - r3 = rotl_imm(r3, 9u); // s26 rotl - r2 = r2 - r1; // s27 sub - r6 = __umulhi(r6, r0); // s28 mulhi - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r4, 2); // s29 shfl - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r6, 1); // s30 shfl - r1 = r1 + r6 + ((((sel >> 1u) & 1u) != 0u) ? 0xb1cdb2abu : 0x37985632u); // s31 add - r0 = rotr_var(r0, r1); // s32 rotr - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r4, 2); // s33 shfl - r0 = r0 ^ __shfl_xor_sync(0xffffffffu, r3, 4); // s34 shfl - r7 = r7 * r0; // s35 mul - r3 = r3 + r1 + ((((sel >> 0u) & 1u) != 0u) ? 0x856e0180u : 0x804e777eu); // s36 add - r1 = r1 ^ r6; // s37 xor - r2 = r2 * r7; // s38 mul - r6 = r6 + r5 + ((((sel >> 2u) & 1u) != 0u) ? 0xe9bd0cc4u : 0x0b06c8a7u); // s39 add - r5 = r5 * r7; // s40 mul - r2 = __umulhi(r2, r3); // s41 mulhi - r2 = r2 ^ r0; // s42 xor - r0 = rotl_imm(r0, 4u); // s43 rotl - r4 = __umulhi(r4, r3); // s44 mulhi - r6 = r6 + r7 + ((((sel >> 12u) & 1u) != 0u) ? 0xcdcb63f6u : 0xee822e17u); // s45 add - r3 = r2 * r0 + r3; // s46 mad - r4 = r4 ^ __shfl_xor_sync(0xffffffffu, r3, 8); // s47 shfl - r6 = __umulhi(r6, r5); // s48 mulhi - r0 = r0 - r2; // s49 sub - r3 = r3 - r5; // s50 sub - r1 = rotr_var(r1, r4); // s51 rotr - r6 = r7 * r7 + r6; // s52 mad - r5 = r5 ^ r3; // s53 xor - r1 = r1 - r0; // s54 sub - r5 = r5 - r6; // s55 sub - r3 = r3 + r2 + ((((sel >> 10u) & 1u) != 0u) ? 0xd4758987u : 0x3dfad1b6u); // s56 add - r4 = r4 + r1 + ((((sel >> 0u) & 1u) != 0u) ? 0x0602d6beu : 0xfca75bc2u); // s57 add - r5 = __umulhi(r5, r6); // s58 mulhi - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r1, 2); // s59 shfl - r2 = rotl_imm(r2, 9u); // s60 rotl - r4 = r4 | r6; // s61 or - r6 = rotr_var(r6, r4); // s62 rotr - r2 = r2 * r4; // s63 mul - r0 = r0 ^ r5; // s64 xor - r2 = r2 ^ r7; // s65 xor - r2 = r2 + r1 + ((((sel >> 6u) & 1u) != 0u) ? 0x8596687au : 0xd71c02ffu); // s66 add - r1 = rotl_imm(r1, 21u); // s67 rotl - r3 = r3 * r2; // s68 mul - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r5, 2); // s69 shfl - r3 = r3 * r2; // s70 mul - r0 = r2 * r5 + r0; // s71 mad - r6 = r6 + r7 + ((((sel >> 0u) & 1u) != 0u) ? 0x08ac5733u : 0x3c6fe15du); // s72 add - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r2, 8); // s73 shfl - r3 = r3 * r6; // s74 mul - r6 = r6 ^ r7; // s75 xor - r3 = r3 | r0; // s76 or - r2 = r2 + r4 + ((((sel >> 1u) & 1u) != 0u) ? 0xc100b495u : 0x295d5faeu); // s77 add - r3 = __umulhi(r3, r7); // s78 mulhi - r4 = r4 | r1; // s79 or - r4 = rotr_var(r4, r3); // s80 rotr - r4 = r4 + r3 + ((((sel >> 15u) & 1u) != 0u) ? 0x5cc59530u : 0x274a9221u); // s81 add - r7 = r7 + r3 + ((((sel >> 5u) & 1u) != 0u) ? 0x141479ecu : 0xc8651f8eu); // s82 add - r3 = rotr_var(r3, r6); // s83 rotr - r2 = r4 * r7 + r2; // s84 mad - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r5, 16); // s85 shfl - r3 = r3 ^ r2; // s86 xor - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r7, 2); // s87 shfl - r0 = r0 ^ r4; // s88 xor - r3 = r3 + r2 + ((((sel >> 18u) & 1u) != 0u) ? 0x883c0c92u : 0x53f915c2u); // s89 add - r7 = rotr_var(r7, r5); // s90 rotr - r3 = r3 + r1 + ((((sel >> 31u) & 1u) != 0u) ? 0x3cc5bd20u : 0xe2be00a5u); // s91 add - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r2, 1); // s92 shfl - r6 = rotr_var(r6, r2); // s93 rotr - r7 = rotl_imm(r7, 14u); // s94 rotl - r4 = r5 * r5 + r4; // s95 mad - r2 = r4 * r5 + r2; // s96 mad - r1 = r1 ^ r0; // s97 xor - r5 = r5 * r4; // s98 mul - r2 = r2 - r0; // s99 sub - r7 = rotl_imm(r7, 30u); // s100 rotl - r5 = r5 + r6 + ((((sel >> 17u) & 1u) != 0u) ? 0xbfd646cbu : 0x423fd9c9u); // s101 add - r7 = r7 + r6 + ((((sel >> 11u) & 1u) != 0u) ? 0x19b74a43u : 0x8d3c011du); // s102 add - r5 = rotl_imm(r5, 6u); // s103 rotl - r0 = r0 * r4; // s104 mul - r0 = r0 | r5; // s105 or - r0 = r0 + r1 + ((((sel >> 15u) & 1u) != 0u) ? 0x7dcb7e18u : 0x8ce14721u); // s106 add - r0 = rotl_imm(r0, 15u); // s107 rotl - r4 = r4 - r2; // s108 sub - r2 = __umulhi(r2, r0); // s109 mulhi - r1 = __umulhi(r1, r0); // s110 mulhi - r0 = r0 + r2 + ((((sel >> 28u) & 1u) != 0u) ? 0x3c179ad8u : 0x2d9fc6b9u); // s111 add - r2 = __umulhi(r2, r0); // s112 mulhi - r6 = r6 - r3; // s113 sub - r7 = r6 * r3 + r7; // s114 mad - r5 = rotr_var(r5, r1); // s115 rotr - r6 = rotr_var(r6, r4); // s116 rotr - r2 = r2 + r1 + ((((sel >> 22u) & 1u) != 0u) ? 0x47359729u : 0x502138e2u); // s117 add - r3 = r2 * r0 + r3; // s118 mad - r1 = r1 * r3; // s119 mul - r2 = r0 * r4 + r2; // s120 mad - r0 = r0 * r3; // s121 mul - r2 = __umulhi(r2, r0); // s122 mulhi - r5 = r5 * r6; // s123 mul - r4 = r2 * r4 + r4; // s124 mad - r4 = r4 ^ __shfl_xor_sync(0xffffffffu, r2, 2); // s125 shfl - r2 = r2 ^ r0; // s126 xor - r1 = rotl_imm(r1, 7u); // s127 rotl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r2, 4); // s128 shfl - r6 = rotl_imm(r6, 17u); // s129 rotl - r2 = r2 + r5 + ((((sel >> 15u) & 1u) != 0u) ? 0x6c57e4e7u : 0x33dff776u); // s130 add - r0 = r0 | r3; // s131 or - r5 = rotr_var(r5, r0); // s132 rotr - r2 = r2 + r3 + ((((sel >> 30u) & 1u) != 0u) ? 0xe8212c0cu : 0x5db25b34u); // s133 add - r4 = r4 ^ r3; // s134 xor - r6 = r6 ^ r1; // s135 xor - r1 = r1 - r7; // s136 sub - r2 = r6 * r2 + r2; // s137 mad - r0 = __umulhi(r0, r5); // s138 mulhi - r2 = r2 + r7 + ((((sel >> 10u) & 1u) != 0u) ? 0xd44ff710u : 0x218d4090u); // s139 add - r1 = rotr_var(r1, r4); // s140 rotr - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r6, 1); // s141 shfl - r7 = r6 * r2 + r7; // s142 mad - r2 = r2 * r3; // s143 mul - r7 = r7 + r4 + ((((sel >> 29u) & 1u) != 0u) ? 0xb98942fau : 0xf4b1a8deu); // s144 add - r7 = rotl_imm(r7, 15u); // s145 rotl - r7 = r7 ^ r5; // s146 xor - r4 = r7 * r4 + r4; // s147 mad - r6 = rotr_var(r6, r5); // s148 rotr - r1 = r2 * r4 + r1; // s149 mad - r1 = r1 + r7 + ((((sel >> 17u) & 1u) != 0u) ? 0x0d48ba42u : 0x2bef10f2u); // s150 add - r5 = r5 ^ r4; // s151 xor - r7 = r7 + r0 + ((((sel >> 30u) & 1u) != 0u) ? 0xc98dea9cu : 0xdc5cc080u); // s152 add - r4 = r4 * r6; // s153 mul - r0 = r0 * r2; // s154 mul - r6 = r6 ^ r5; // s155 xor - r4 = rotr_var(r4, r2); // s156 rotr - r1 = rotl_imm(r1, 11u); // s157 rotl - r5 = r5 + r0 + ((((sel >> 11u) & 1u) != 0u) ? 0x6c752dcbu : 0x18197438u); // s158 add - r4 = r1 * r5 + r4; // s159 mad - r4 = rotl_imm(r4, 26u); // s160 rotl - r3 = r0 * r7 + r3; // s161 mad - r3 = rotr_var(r3, r1); // s162 rotr - r4 = r4 + r0 + ((((sel >> 3u) & 1u) != 0u) ? 0x8e481727u : 0xf1c46574u); // s163 add - r0 = __umulhi(r0, r4); // s164 mulhi - r2 = r2 + r6 + ((((sel >> 20u) & 1u) != 0u) ? 0x550ab406u : 0xac578137u); // s165 add - r0 = rotl_imm(r0, 13u); // s166 rotl - r3 = r3 + r1 + ((((sel >> 14u) & 1u) != 0u) ? 0xaebb5966u : 0xa33e6706u); // s167 add - r3 = r3 | r5; // s168 or - r6 = rotr_var(r6, r2); // s169 rotr - r4 = r4 ^ r6; // s170 xor - r6 = r6 - r1; // s171 sub - r7 = rotl_imm(r7, 22u); // s172 rotl - r5 = rotl_imm(r5, 15u); // s173 rotl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r0, 8); // s174 shfl - r0 = r0 ^ r5; // s175 xor - r7 = rotl_imm(r7, 6u); // s176 rotl - r7 = r7 - r0; // s177 sub - r3 = rotl_imm(r3, 30u); // s178 rotl - r7 = r6 * r1 + r7; // s179 mad - r6 = rotl_imm(r6, 9u); // s180 rotl - r2 = r2 ^ r4; // s181 xor - r2 = r2 ^ r7; // s182 xor - r7 = r7 ^ r2; // s183 xor - r1 = r1 + r2 + ((((sel >> 21u) & 1u) != 0u) ? 0x5bb7550fu : 0xd94d55acu); // s184 add - r3 = r3 | r5; // s185 or - r6 = __umulhi(r6, r3); // s186 mulhi - r4 = r4 | r0; // s187 or - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r1, 2); // s188 shfl - r6 = r6 + r5 + ((((sel >> 6u) & 1u) != 0u) ? 0x2a354e2du : 0x89e747feu); // s189 add - r1 = r1 ^ r4; // s190 xor - r7 = __umulhi(r7, r4); // s191 mulhi - r2 = rotl_imm(r2, 26u); // s192 rotl - r5 = rotl_imm(r5, 8u); // s193 rotl - r4 = r4 ^ __shfl_xor_sync(0xffffffffu, r5, 16); // s194 shfl - r4 = r4 ^ r5; // s195 xor - r1 = r1 * r3; // s196 mul - r5 = r5 + r2 + ((((sel >> 7u) & 1u) != 0u) ? 0x10cfdc71u : 0xea03e8e7u); // s197 add - r7 = r7 + r5 + ((((sel >> 15u) & 1u) != 0u) ? 0x66e148d3u : 0xa7ee0102u); // s198 add - r5 = r5 - r2; // s199 sub - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r1, 2); // s200 shfl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r1, 8); // s201 shfl - r4 = rotr_var(r4, r5); // s202 rotr - r7 = r7 ^ r5; // s203 xor - r7 = r7 ^ r0; // s204 xor - r6 = r6 + r2 + ((((sel >> 10u) & 1u) != 0u) ? 0x8558b619u : 0x8379a4deu); // s205 add - r4 = rotl_imm(r4, 13u); // s206 rotl - r1 = __umulhi(r1, r3); // s207 mulhi - r1 = r4 * r4 + r1; // s208 mad - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r0, 4); // s209 shfl - r7 = r7 + r0 + ((((sel >> 11u) & 1u) != 0u) ? 0xf4049c4cu : 0xaa8cb14eu); // s210 add - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r1, 16); // s211 shfl - r1 = r1 ^ r0; // s212 xor - r7 = rotl_imm(r7, 3u); // s213 rotl - r4 = rotr_var(r4, r7); // s214 rotr - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r2, 16); // s215 shfl - r5 = r5 | r1; // s216 or - r1 = r1 - r2; // s217 sub - r6 = r6 - r5; // s218 sub - r6 = rotl_imm(r6, 4u); // s219 rotl - r2 = __umulhi(r2, r0); // s220 mulhi - r2 = r2 | r0; // s221 or - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r2, 8); // s222 shfl - r5 = __umulhi(r5, r6); // s223 mulhi - r0 = r0 - r6; // s224 sub - r7 = rotl_imm(r7, 23u); // s225 rotl - r4 = r4 | r2; // s226 or - r2 = r2 * r4; // s227 mul - r3 = rotl_imm(r3, 12u); // s228 rotl - r0 = rotr_var(r0, r4); // s229 rotr - r0 = r0 + r6 + ((((sel >> 16u) & 1u) != 0u) ? 0x2a7fecb2u : 0x1d176220u); // s230 add - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r2, 4); // s231 shfl - r2 = r2 * r1; // s232 mul - r7 = r0 * r1 + r7; // s233 mad - r5 = rotl_imm(r5, 22u); // s234 rotl - r4 = rotr_var(r4, r6); // s235 rotr - r0 = r5 * r1 + r0; // s236 mad - r6 = r6 ^ r4; // s237 xor - r4 = r4 ^ r6; // s238 xor - r6 = rotl_imm(r6, 18u); // s239 rotl - r4 = r4 + r7 + ((((sel >> 25u) & 1u) != 0u) ? 0xadce39f3u : 0x17dafb4du); // s240 add - r0 = r0 - r7; // s241 sub - r1 = rotr_var(r1, r6); // s242 rotr - r3 = r3 + r0 + ((((sel >> 29u) & 1u) != 0u) ? 0x0e7033b6u : 0xf2f77d26u); // s243 add - r2 = r7 * r4 + r2; // s244 mad - r4 = r0 * r6 + r4; // s245 mad - r5 = r5 * r7; // s246 mul - r3 = r3 + r5 + ((((sel >> 31u) & 1u) != 0u) ? 0x7b2ff6b7u : 0xfed2da4eu); // s247 add - r0 = r0 + r7 + ((((sel >> 0u) & 1u) != 0u) ? 0xb5fad7b1u : 0x03b2891cu); // s248 add - r5 = __umulhi(r5, r4); // s249 mulhi - r5 = r5 + r2 + ((((sel >> 11u) & 1u) != 0u) ? 0xa7e71c2fu : 0x042cd6e3u); // s250 add - r6 = r6 ^ __shfl_xor_sync(0xffffffffu, r1, 1); // s251 shfl - r6 = r6 ^ r1; // s252 xor - r2 = r2 * r5; // s253 mul - r0 = r0 + r6 + ((((sel >> 16u) & 1u) != 0u) ? 0xb83d78deu : 0x784a302bu); // s254 add - r2 = r2 - r4; // s255 sub - } - } - uint32_t lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint32_t hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((uint64_t)hi << 32) | (uint64_t)lo; -} - -// Host-side launch wrappers. Declared in program.h, called from host.cu. -cudaError_t igneum_launch_cache_fill(uint32_t* cache, uint32_t nSegments) { - if (nSegments == 0u) return cudaErrorInvalidValue; - uint32_t block = 256u; - uint32_t grid = (nSegments + block - 1u) / block; - igneum_cache_fill<<>>(cache, nSegments); - return cudaGetLastError(); -} - -cudaError_t igneum_launch_build(uint32_t* ds, const uint32_t* cache, uint32_t nItems) { - if (nItems == 0u) return cudaErrorInvalidValue; - uint32_t block = 256u; - uint32_t grid = (nItems + block - 1u) / block; - igneum_build<<>>(ds, cache, nItems); - return cudaGetLastError(); -} - -cudaError_t igneum_launch_hash(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask, - uint32_t nonces, uint32_t blockWarps) { - if (blockWarps == 0u || blockWarps > 32u) return cudaErrorInvalidValue; - uint32_t block = 32u * blockWarps; - if (nonces == 0u || (nonces % block) != 0u) return cudaErrorInvalidValue; - igneum_hash<<>>(ds, out, baseNonce, mask); - return cudaGetLastError(); -} - -cudaError_t igneum_hash_info(int* numRegs, int* blocksPerSM, uint32_t blockWarps) { - cudaFuncAttributes attr; - cudaError_t e = cudaFuncGetAttributes(&attr, igneum_hash); - if (e != cudaSuccess) return e; - *numRegs = attr.numRegs; - return cudaOccupancyMaxActiveBlocksPerMultiprocessor(blocksPerSM, igneum_hash, (int)(32u * blockWarps), 0); -} diff --git a/proto-cuda/packs-ca4/mx8_sh256x27/kernel_bound.cl b/proto-cuda/packs-ca4/mx8_sh256x27/kernel_bound.cl deleted file mode 100644 index 02546cd07..000000000 --- a/proto-cuda/packs-ca4/mx8_sh256x27/kernel_bound.cl +++ /dev/null @@ -1,891 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// OpenCL C twin of the Metal kernel for the same seed (see proto-opencl/README.md, WAVEFRONT.md and program.metal). -// Built from source at runtime by proto-opencl/host.c, which passes these defines: -// IGNEUM_GROUP work-group size of igneum_hash, a multiple of 32 (default 32: one work-group = one 32-lane unit) -// IGNEUM_EXCHANGE 0 = local-memory exchange with a barrier (any device, any wave width; the default) -// 1 = sub_group_shuffle_xor (cl_khr_subgroup_shuffle), only with IGNEUM_GROUP 32 and a sub-group size of exactly 32 -// 2 = intel_sub_group_shuffle_xor (cl_intel_subgroups), same condition -// The verification unit is always 32 lanes. A 64-wide hardware wave (AMD GCN/CDNA, RDNA in wave64) runs two units; -// the exchange masks are 1, 2, 4, 8, 16, so every partner lane lies inside the lane's own aligned run of 32. -#ifndef IGNEUM_GROUP -#define IGNEUM_GROUP 32 -#endif -#ifndef IGNEUM_EXCHANGE -#define IGNEUM_EXCHANGE 0 -#endif -#ifdef __OPENCL_VERSION__ -#define IGNEUM_KERNEL_HASH __kernel __attribute__((reqd_work_group_size(IGNEUM_GROUP, 1, 1))) -#define IGNEUM_LOCAL_WORDS(name, n) __local uint name[n] -#if IGNEUM_EXCHANGE == 1 -#ifdef cl_khr_subgroups -#pragma OPENCL EXTENSION cl_khr_subgroups : enable -#endif -#ifdef cl_khr_subgroup_shuffle -#pragma OPENCL EXTENSION cl_khr_subgroup_shuffle : enable -#endif -#elif IGNEUM_EXCHANGE == 2 -#pragma OPENCL EXTENSION cl_intel_subgroups : enable -#endif -#else -// Not an OpenCL compiler: proto-opencl/emu compiles this file as C++ and supplies the built-ins and these two macros. -#include "emu_opencl.h" -#endif - -#if IGNEUM_EXCHANGE == 1 -#define IGNEUM_SHFL_XOR(dst, a, m) dst = sub_group_shuffle_xor((a), (uint)(m)) -#define IGNEUM_BCAST0(dst, a) dst = sub_group_broadcast((a), 0u) -#elif IGNEUM_EXCHANGE == 2 -#define IGNEUM_SHFL_XOR(dst, a, m) dst = intel_sub_group_shuffle_xor((a), (uint)(m)) -#define IGNEUM_BCAST0(dst, a) dst = sub_group_broadcast((a), 0u) -#else -// Local-memory exchange. Two buffers of IGNEUM_GROUP words alternate (xk counts exchanges), so one barrier per -// exchange is enough: a lane can only overwrite buffer b at exchange k+2 after passing barrier k+1, and every lane -// reaches barrier k+1 only after its read of buffer b at exchange k. The partner lid ^ m stays inside the lane's -// aligned run of 32 because m < 32. Control flow is uniform, so every work-item reaches every barrier. -#define IGNEUM_SHFL_XOR(dst, a, m) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid ^ (uint)(m))]; xk += 1u; } -#define IGNEUM_BCAST0(dst, a) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid & ~31u)]; xk += 1u; } -#endif - -static inline uint splitmix32(uint x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -// n is a literal in 1..31 at every call site. OpenCL rotate() rotates left by n modulo 32. -static inline uint rotl_imm(uint x, uint n) { return rotate(x, n); } -// Right rotation by n modulo 32 as a left rotation by (32 - n) modulo 32; n == 0 gives x. -static inline uint rotr_var(uint x, uint n) { return rotate(x, (0u - n) & 31u); } -static inline uint ds_elem(uint i, uint d0, uint d1) { - uint x = i ^ d0; - x *= 0x9E3779B1u; x ^= x >> 15; - x += d1; - x *= 0x85EBCA77u; x ^= x >> 13; - x *= 0xC2B2AE3Du; x ^= x >> 16; - return x; -} - -// Memory-hard dataset core (MEMHARD.md). Cache: 2^26 words in 2^16 segments of 64 chained ChaCha12 lines. -// Item: 8 rounds of 8 x seed-parameterised mixer + one 64-byte cache read, then 8 x final mixer (class v3, mixer multiplier 8, -// docs/plans/mixer-x4.md: the round key of application j of round r is 0x9E3779B9 * (r * m + j + 1)). All parameters are literals. -#define MH_CACHE_LINE_MASK 0x003fffffu -#define MH_SEGMENT_LINES 64u -#define MH_QR(a, b, c, d, r1, r2, r3, r4) { a += b; d ^= a; d = mh_rotl(d, r1); c += d; b ^= c; b = mh_rotl(b, r2); a += b; d ^= a; d = mh_rotl(d, r3); c += d; b ^= c; b = mh_rotl(b, r4); } -static inline uint mh_rotl(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 at every call site - -// y = ChaCha12 core(x) + x -static inline void mh_chacha_block(const uint* x, uint* y) { - for (uint i = 0u; i < 16u; ++i) y[i] = x[i]; - for (uint r = 0u; r < 6u; ++r) { - MH_QR(y[0], y[4], y[8], y[12], 16u, 12u, 8u, 7u) MH_QR(y[1], y[5], y[9], y[13], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[6], y[10], y[14], 16u, 12u, 8u, 7u) MH_QR(y[3], y[7], y[11], y[15], 16u, 12u, 8u, 7u) - MH_QR(y[0], y[5], y[10], y[15], 16u, 12u, 8u, 7u) MH_QR(y[1], y[6], y[11], y[12], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[7], y[8], y[13], 16u, 12u, 8u, 7u) MH_QR(y[3], y[4], y[9], y[14], 16u, 12u, 8u, 7u) - } - for (uint i = 0u; i < 16u; ++i) y[i] += x[i]; -} - -// One cache segment: 64 chained lines written at cache[seg * 1024]. in_j = prev ^ (sigma || K || seg || j || tag), prev_0 = 0. -static inline void mh_cache_segment(__global uint* cache, uint seg) { - uint prev[16]; uint x[16]; uint y[16]; - for (uint i = 0u; i < 16u; ++i) prev[i] = 0u; - for (uint j = 0u; j < MH_SEGMENT_LINES; ++j) { - x[0] = 0x61707865u ^ prev[0]; x[1] = 0x3320646eu ^ prev[1]; x[2] = 0x79622d32u ^ prev[2]; x[3] = 0x6b206574u ^ prev[3]; - x[4] = 0x3067619fu ^ prev[4]; - x[5] = 0x3c269176u ^ prev[5]; - x[6] = 0x84a03b03u ^ prev[6]; - x[7] = 0xf8c63294u ^ prev[7]; - x[8] = 0xff977c5bu ^ prev[8]; - x[9] = 0xe60def3eu ^ prev[9]; - x[10] = 0x63630141u ^ prev[10]; - x[11] = 0xb8fbcb58u ^ prev[11]; - x[12] = seg ^ prev[12]; x[13] = j ^ prev[13]; x[14] = 0x49676e65u ^ prev[14]; x[15] = 0x756d4d48u ^ prev[15]; - mh_chacha_block(x, y); - __global uint* line = cache + ((seg * MH_SEGMENT_LINES + j) * 16u); - for (uint i = 0u; i < 16u; ++i) { line[i] = y[i]; prev[i] = y[i]; } - } -} - -// M_r: per word (s ^ (RC + rk)) * MUL, then a column round and a diagonal round with the seed-drawn rotations. -static inline void mh_mixer(uint* s, uint rk) { - s[0] = (s[0] ^ (0xbab68293u + rk)) * 0x42146205u; - s[1] = (s[1] ^ (0xcc162340u + rk)) * 0x52cbe0fbu; - s[2] = (s[2] ^ (0x6ce151ccu + rk)) * 0x7ecf4a03u; - s[3] = (s[3] ^ (0xe62b8997u + rk)) * 0x6728907fu; - s[4] = (s[4] ^ (0xc9c80297u + rk)) * 0xd81d9751u; - s[5] = (s[5] ^ (0xf74a1654u + rk)) * 0x132952c3u; - s[6] = (s[6] ^ (0x3d704af5u + rk)) * 0xf60de277u; - s[7] = (s[7] ^ (0x3cf522b7u + rk)) * 0x05358035u; - s[8] = (s[8] ^ (0x2b9cac04u + rk)) * 0xbaf6499du; - s[9] = (s[9] ^ (0xa880ac10u + rk)) * 0xe4db9667u; - s[10] = (s[10] ^ (0x13e5dd1du + rk)) * 0x3e98f45du; - s[11] = (s[11] ^ (0x6fc3e233u + rk)) * 0xd0004eddu; - s[12] = (s[12] ^ (0x2d83eeacu + rk)) * 0x2691630du; - s[13] = (s[13] ^ (0x9006e8bfu + rk)) * 0x9beb3bcfu; - s[14] = (s[14] ^ (0x2c4b5362u + rk)) * 0xab310379u; - s[15] = (s[15] ^ (0x31b49ee2u + rk)) * 0x99cfb423u; - MH_QR(s[0], s[4], s[8], s[12], 20u, 20u, 19u, 4u) MH_QR(s[1], s[5], s[9], s[13], 20u, 20u, 19u, 4u) - MH_QR(s[2], s[6], s[10], s[14], 20u, 20u, 19u, 4u) MH_QR(s[3], s[7], s[11], s[15], 20u, 20u, 19u, 4u) - MH_QR(s[0], s[5], s[10], s[15], 26u, 3u, 3u, 27u) MH_QR(s[1], s[6], s[11], s[12], 26u, 3u, 3u, 27u) - MH_QR(s[2], s[7], s[8], s[13], 26u, 3u, 3u, 27u) MH_QR(s[3], s[4], s[9], s[14], 26u, 3u, 3u, 27u) -} - -// Item t: 16 words. s = (K, t * MUL[i] + RC[i]); 8 rounds of 8 x mixer + cache line s[0] & mask; 8 x final mixer. -static inline void mh_item(__global const uint* cache, uint t, uint* s) { - s[0] = 0x3067619fu; - s[1] = 0x3c269176u; - s[2] = 0x84a03b03u; - s[3] = 0xf8c63294u; - s[4] = 0xff977c5bu; - s[5] = 0xe60def3eu; - s[6] = 0x63630141u; - s[7] = 0xb8fbcb58u; - s[8] = t * 0x42146205u + 0xbab68293u; - s[9] = t * 0x52cbe0fbu + 0xcc162340u; - s[10] = t * 0x7ecf4a03u + 0x6ce151ccu; - s[11] = t * 0x6728907fu + 0xe62b8997u; - s[12] = t * 0xd81d9751u + 0xc9c80297u; - s[13] = t * 0x132952c3u + 0xf74a1654u; - s[14] = t * 0xf60de277u + 0x3d704af5u; - s[15] = t * 0x05358035u + 0x3cf522b7u; - for (uint r = 0u; r < 8u; ++r) { - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (r * 8u + j + 1u)); - __global const uint* line = cache + ((s[0] & MH_CACHE_LINE_MASK) * 16u); - for (uint i = 0u; i < 16u; ++i) s[i] ^= line[i]; - } - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (64u + j + 1u)); -} -// dataset[w] without the dataset: derive item w >> 4 and take word w & 15. -static inline uint mh_word(__global const uint* cache, uint w) { uint s[16]; mh_item(cache, w >> 4u, s); return s[w & 15u]; } - -// Memory-hard dataset (MEMHARD.md). One work-item per cache segment; one work-item per 64-byte dataset item. -// The same constants as memhard.h in this pack (one emitter, three dialects). -__kernel void igneum_cache_fill(__global uint* cache, uint nSegments) { - uint seg = (uint)get_global_id(0); - if (seg < nSegments) mh_cache_segment(cache, seg); -} -__kernel void igneum_build(__global uint* ds, __global const uint* cache, uint nItems) { - uint t = (uint)get_global_id(0); - if (t < nItems) { - uint s[16]; - mh_item(cache, t, s); - __global uint* d = ds + ((ulong)t * 16u); - for (uint i = 0u; i < 16u; ++i) d[i] = s[i]; - } -} - -// One hash per work-item. IGNEUM_GROUP is a multiple of 32; lane = lid & 31 and every exchange stays inside the -// lane's own aligned run of 32 work-items, exactly like simd_shuffle_xor inside a 32-wide Metal SIMD group and -// __shfl_xor_sync inside a CUDA warp. Control flow is uniform (no branches at all). -IGNEUM_KERNEL_HASH void igneum_hash(__global const uint* ds, __global ulong* out, uint baseNonce, uint mask) { - uint gid = (uint)get_global_id(0); - uint lid = (uint)get_local_id(0); - uint nonce = baseNonce + gid; - uint r0, r1, r2, r3, r4, r5, r6, r7; -#if IGNEUM_EXCHANGE == 0 - IGNEUM_LOCAL_WORDS(xch, 2 * IGNEUM_GROUP); - uint xk = 0u; -#else - (void)lid; -#endif - { uint x = nonce ^ 0x67a9a7beu; x += 0x9e3779b9u; x = splitmix32(x); r0 = x ^ 0x1a155b25u; } // SEEDW[0], 0x9e3779b9u * 1u, SEEDW[1] - { uint x = nonce ^ 0x1a155b25u; x += 0x3c6ef372u; x = splitmix32(x); r1 = x ^ 0xfddfb732u; } // SEEDW[1], 0x9e3779b9u * 2u, SEEDW[2] - { uint x = nonce ^ 0xfddfb732u; x += 0xdaa66d2bu; x = splitmix32(x); r2 = x ^ 0x4b5af2e8u; } // SEEDW[2], 0x9e3779b9u * 3u, SEEDW[3] - { uint x = nonce ^ 0x4b5af2e8u; x += 0x78dde6e4u; x = splitmix32(x); r3 = x ^ 0xc55caf33u; } // SEEDW[3], 0x9e3779b9u * 4u, SEEDW[4] - { uint x = nonce ^ 0xc55caf33u; x += 0x1715609du; x = splitmix32(x); r4 = x ^ 0xa27c13b7u; } // SEEDW[4], 0x9e3779b9u * 5u, SEEDW[5] - { uint x = nonce ^ 0xa27c13b7u; x += 0xb54cda56u; x = splitmix32(x); r5 = x ^ 0x06628a48u; } // SEEDW[5], 0x9e3779b9u * 6u, SEEDW[6] - { uint x = nonce ^ 0x06628a48u; x += 0x5384540fu; x = splitmix32(x); r6 = x ^ 0x03852469u; } // SEEDW[6], 0x9e3779b9u * 7u, SEEDW[7] - { uint x = nonce ^ 0x03852469u; x += 0xf1bbcdc8u; x = splitmix32(x); r7 = x ^ 0x67a9a7beu; } // SEEDW[7], 0x9e3779b9u * 8u, SEEDW[0] - - for (uint it = 0u; it < 8u; ++it) { - uint sel = r0; - r2 = r3 * r4 + r2; // 0 mad - r2 = r1 * r1 + r2; // 1 mad - r2 = r3 * r2 + r2; // 2 mad - r3 = r3 ^ r5; // 3 xor - r7 = r7 ^ ds[r2 & mask]; // 4 load - r5 = r5 ^ ds[r7 & mask]; // 5 load - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 8u); r1 = r1 ^ t_; } // 6 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 8u); r7 = r7 ^ t_; } // 7 shfl - r1 = mul_hi(r1, r5); // 8 mulhi - r6 = rotr_var(r6, r3); // 9 rotr - r3 = r3 | r4; // 10 or - r4 = r4 ^ ds[r3 & mask]; // 11 load - r0 = mul_hi(r0, r4); // 12 mulhi - r5 = r5 + r1 + ((((sel >> 30u) & 1u) != 0u) ? 0xd3177981u : 0xc7934706u); // 13 add - r0 = r0 ^ ds[r4 & mask]; // 14 load - r2 = r2 - r4; // 15 sub - r2 = r2 ^ ds[r0 & mask]; // 16 load - r7 = r7 ^ ds[r2 & mask]; // 17 load - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 4u); r7 = r7 ^ t_; } // 18 shfl - r5 = r5 * r0; // 19 mul - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 2u); r3 = r3 ^ t_; } // 20 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 16u); r2 = r2 ^ t_; } // 21 shfl - r6 = mul_hi(r6, r2); // 22 mulhi - r6 = r6 ^ ds[r1 & mask]; // 23 load - r5 = r5 * r0; // 24 mul - r5 = rotl_imm(r5, 19u); // 25 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 2u); r7 = r7 ^ t_; } // 26 shfl - r0 = r0 ^ r5; // 27 xor - r0 = r0 ^ r4; // 28 xor - r3 = r3 - r0; // 29 sub - r5 = r5 * r1; // 30 mul - r7 = r7 ^ ds[r2 & mask]; // 31 load - r1 = r1 ^ ds[r0 & mask]; // 32 load - r5 = r5 ^ r6; // 33 xor - r5 = r5 ^ ds[r1 & mask]; // 34 load - r0 = mul_hi(r0, r5); // 35 mulhi - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 4u); r5 = r5 ^ t_; } // 36 shfl - r7 = r7 ^ ds[r0 & mask]; // 37 load - r3 = r3 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0x230c005cu : 0x75ba2fadu); // 38 add - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 4u); r1 = r1 ^ t_; } // 39 shfl - r2 = r2 ^ r5; // 40 xor - r3 = r6 * r3 + r3; // 41 mad - r6 = r6 - r7; // 42 sub - r7 = r7 ^ r0; // 43 xor - r1 = r1 ^ ds[r7 & mask]; // 44 load - r2 = r2 * r3; // 45 mul - r1 = mul_hi(r1, r5); // 46 mulhi - r4 = r4 - r3; // 47 sub - r2 = rotr_var(r2, r6); // 48 rotr - r3 = r3 ^ ds[r5 & mask]; // 49 load - r1 = r1 + r5 + ((((sel >> 7u) & 1u) != 0u) ? 0x1907970cu : 0x81b8bc2cu); // 50 add - r0 = r0 * r2; // 51 mul - r0 = r0 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x699fd448u : 0x4f92b968u); // 52 add - r1 = r1 + r0 + ((((sel >> 12u) & 1u) != 0u) ? 0x77b1520du : 0x2bb965afu); // 53 add - r7 = rotl_imm(r7, 14u); // 54 rotl - r3 = r3 + r7 + ((((sel >> 1u) & 1u) != 0u) ? 0xa54c55a0u : 0x7b0fe07au); // 55 add - r6 = r6 ^ ds[r7 & mask]; // 56 load - r1 = rotr_var(r1, r5); // 57 rotr - r5 = r5 ^ ds[r4 & mask]; // 58 load - r6 = r6 ^ ds[r2 & mask]; // 59 load - r3 = r5 * r0 + r3; // 60 mad - r5 = r5 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0xad7493e7u : 0xaf9dd72du); // 61 add - r4 = r4 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x1e07c3d9u : 0x89841d87u); // 62 add - r5 = rotl_imm(r5, 19u); // 63 rotl - // latency-shadow block (Counter ASIC 3.0 item 8): 256 ALU instructions x 27 passes after instruction 63, no load - for (uint sh = 0u; sh < 27u; ++sh) { - r5 = r5 + r2 + ((((sel >> 18u) & 1u) != 0u) ? 0x8bc12da9u : 0x92199f99u); // s0 add - r0 = r0 + r7 + ((((sel >> 26u) & 1u) != 0u) ? 0x63079e5au : 0x8d72d3adu); // s1 add - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 2u); r6 = r6 ^ t_; } // s2 shfl - r4 = r4 - r2; // s3 sub - r7 = r7 + r0 + ((((sel >> 31u) & 1u) != 0u) ? 0x5d4c7a60u : 0xb21b4babu); // s4 add - r0 = rotl_imm(r0, 11u); // s5 rotl - r0 = r0 + r1 + ((((sel >> 16u) & 1u) != 0u) ? 0xc88e2942u : 0x2fe0e98bu); // s6 add - r7 = r7 ^ r1; // s7 xor - r1 = r6 * r5 + r1; // s8 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 4u); r6 = r6 ^ t_; } // s9 shfl - r1 = r2 * r2 + r1; // s10 mad - r5 = r0 * r3 + r5; // s11 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 4u); r2 = r2 ^ t_; } // s12 shfl - r1 = r1 + r5 + ((((sel >> 25u) & 1u) != 0u) ? 0xbc48c63eu : 0xbdf8f9a5u); // s13 add - r1 = rotl_imm(r1, 29u); // s14 rotl - r1 = r1 - r4; // s15 sub - r7 = r7 | r1; // s16 or - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 8u); r2 = r2 ^ t_; } // s17 shfl - r7 = r7 ^ r4; // s18 xor - r6 = r6 * r1; // s19 mul - r5 = r6 * r0 + r5; // s20 mad - r3 = r3 - r1; // s21 sub - r6 = r6 * r0; // s22 mul - r2 = r2 + r0 + ((((sel >> 1u) & 1u) != 0u) ? 0x96e8f127u : 0x45c37cecu); // s23 add - r6 = r6 - r4; // s24 sub - r7 = r3 * r4 + r7; // s25 mad - r3 = rotl_imm(r3, 9u); // s26 rotl - r2 = r2 - r1; // s27 sub - r6 = mul_hi(r6, r0); // s28 mulhi - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 2u); r2 = r2 ^ t_; } // s29 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 1u); r1 = r1 ^ t_; } // s30 shfl - r1 = r1 + r6 + ((((sel >> 1u) & 1u) != 0u) ? 0xb1cdb2abu : 0x37985632u); // s31 add - r0 = rotr_var(r0, r1); // s32 rotr - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 2u); r3 = r3 ^ t_; } // s33 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 4u); r0 = r0 ^ t_; } // s34 shfl - r7 = r7 * r0; // s35 mul - r3 = r3 + r1 + ((((sel >> 0u) & 1u) != 0u) ? 0x856e0180u : 0x804e777eu); // s36 add - r1 = r1 ^ r6; // s37 xor - r2 = r2 * r7; // s38 mul - r6 = r6 + r5 + ((((sel >> 2u) & 1u) != 0u) ? 0xe9bd0cc4u : 0x0b06c8a7u); // s39 add - r5 = r5 * r7; // s40 mul - r2 = mul_hi(r2, r3); // s41 mulhi - r2 = r2 ^ r0; // s42 xor - r0 = rotl_imm(r0, 4u); // s43 rotl - r4 = mul_hi(r4, r3); // s44 mulhi - r6 = r6 + r7 + ((((sel >> 12u) & 1u) != 0u) ? 0xcdcb63f6u : 0xee822e17u); // s45 add - r3 = r2 * r0 + r3; // s46 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 8u); r4 = r4 ^ t_; } // s47 shfl - r6 = mul_hi(r6, r5); // s48 mulhi - r0 = r0 - r2; // s49 sub - r3 = r3 - r5; // s50 sub - r1 = rotr_var(r1, r4); // s51 rotr - r6 = r7 * r7 + r6; // s52 mad - r5 = r5 ^ r3; // s53 xor - r1 = r1 - r0; // s54 sub - r5 = r5 - r6; // s55 sub - r3 = r3 + r2 + ((((sel >> 10u) & 1u) != 0u) ? 0xd4758987u : 0x3dfad1b6u); // s56 add - r4 = r4 + r1 + ((((sel >> 0u) & 1u) != 0u) ? 0x0602d6beu : 0xfca75bc2u); // s57 add - r5 = mul_hi(r5, r6); // s58 mulhi - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 2u); r2 = r2 ^ t_; } // s59 shfl - r2 = rotl_imm(r2, 9u); // s60 rotl - r4 = r4 | r6; // s61 or - r6 = rotr_var(r6, r4); // s62 rotr - r2 = r2 * r4; // s63 mul - r0 = r0 ^ r5; // s64 xor - r2 = r2 ^ r7; // s65 xor - r2 = r2 + r1 + ((((sel >> 6u) & 1u) != 0u) ? 0x8596687au : 0xd71c02ffu); // s66 add - r1 = rotl_imm(r1, 21u); // s67 rotl - r3 = r3 * r2; // s68 mul - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 2u); r7 = r7 ^ t_; } // s69 shfl - r3 = r3 * r2; // s70 mul - r0 = r2 * r5 + r0; // s71 mad - r6 = r6 + r7 + ((((sel >> 0u) & 1u) != 0u) ? 0x08ac5733u : 0x3c6fe15du); // s72 add - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 8u); r3 = r3 ^ t_; } // s73 shfl - r3 = r3 * r6; // s74 mul - r6 = r6 ^ r7; // s75 xor - r3 = r3 | r0; // s76 or - r2 = r2 + r4 + ((((sel >> 1u) & 1u) != 0u) ? 0xc100b495u : 0x295d5faeu); // s77 add - r3 = mul_hi(r3, r7); // s78 mulhi - r4 = r4 | r1; // s79 or - r4 = rotr_var(r4, r3); // s80 rotr - r4 = r4 + r3 + ((((sel >> 15u) & 1u) != 0u) ? 0x5cc59530u : 0x274a9221u); // s81 add - r7 = r7 + r3 + ((((sel >> 5u) & 1u) != 0u) ? 0x141479ecu : 0xc8651f8eu); // s82 add - r3 = rotr_var(r3, r6); // s83 rotr - r2 = r4 * r7 + r2; // s84 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 16u); r2 = r2 ^ t_; } // s85 shfl - r3 = r3 ^ r2; // s86 xor - { uint t_; IGNEUM_SHFL_XOR(t_, r7, 2u); r5 = r5 ^ t_; } // s87 shfl - r0 = r0 ^ r4; // s88 xor - r3 = r3 + r2 + ((((sel >> 18u) & 1u) != 0u) ? 0x883c0c92u : 0x53f915c2u); // s89 add - r7 = rotr_var(r7, r5); // s90 rotr - r3 = r3 + r1 + ((((sel >> 31u) & 1u) != 0u) ? 0x3cc5bd20u : 0xe2be00a5u); // s91 add - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 1u); r7 = r7 ^ t_; } // s92 shfl - r6 = rotr_var(r6, r2); // s93 rotr - r7 = rotl_imm(r7, 14u); // s94 rotl - r4 = r5 * r5 + r4; // s95 mad - r2 = r4 * r5 + r2; // s96 mad - r1 = r1 ^ r0; // s97 xor - r5 = r5 * r4; // s98 mul - r2 = r2 - r0; // s99 sub - r7 = rotl_imm(r7, 30u); // s100 rotl - r5 = r5 + r6 + ((((sel >> 17u) & 1u) != 0u) ? 0xbfd646cbu : 0x423fd9c9u); // s101 add - r7 = r7 + r6 + ((((sel >> 11u) & 1u) != 0u) ? 0x19b74a43u : 0x8d3c011du); // s102 add - r5 = rotl_imm(r5, 6u); // s103 rotl - r0 = r0 * r4; // s104 mul - r0 = r0 | r5; // s105 or - r0 = r0 + r1 + ((((sel >> 15u) & 1u) != 0u) ? 0x7dcb7e18u : 0x8ce14721u); // s106 add - r0 = rotl_imm(r0, 15u); // s107 rotl - r4 = r4 - r2; // s108 sub - r2 = mul_hi(r2, r0); // s109 mulhi - r1 = mul_hi(r1, r0); // s110 mulhi - r0 = r0 + r2 + ((((sel >> 28u) & 1u) != 0u) ? 0x3c179ad8u : 0x2d9fc6b9u); // s111 add - r2 = mul_hi(r2, r0); // s112 mulhi - r6 = r6 - r3; // s113 sub - r7 = r6 * r3 + r7; // s114 mad - r5 = rotr_var(r5, r1); // s115 rotr - r6 = rotr_var(r6, r4); // s116 rotr - r2 = r2 + r1 + ((((sel >> 22u) & 1u) != 0u) ? 0x47359729u : 0x502138e2u); // s117 add - r3 = r2 * r0 + r3; // s118 mad - r1 = r1 * r3; // s119 mul - r2 = r0 * r4 + r2; // s120 mad - r0 = r0 * r3; // s121 mul - r2 = mul_hi(r2, r0); // s122 mulhi - r5 = r5 * r6; // s123 mul - r4 = r2 * r4 + r4; // s124 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 2u); r4 = r4 ^ t_; } // s125 shfl - r2 = r2 ^ r0; // s126 xor - r1 = rotl_imm(r1, 7u); // s127 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 4u); r7 = r7 ^ t_; } // s128 shfl - r6 = rotl_imm(r6, 17u); // s129 rotl - r2 = r2 + r5 + ((((sel >> 15u) & 1u) != 0u) ? 0x6c57e4e7u : 0x33dff776u); // s130 add - r0 = r0 | r3; // s131 or - r5 = rotr_var(r5, r0); // s132 rotr - r2 = r2 + r3 + ((((sel >> 30u) & 1u) != 0u) ? 0xe8212c0cu : 0x5db25b34u); // s133 add - r4 = r4 ^ r3; // s134 xor - r6 = r6 ^ r1; // s135 xor - r1 = r1 - r7; // s136 sub - r2 = r6 * r2 + r2; // s137 mad - r0 = mul_hi(r0, r5); // s138 mulhi - r2 = r2 + r7 + ((((sel >> 10u) & 1u) != 0u) ? 0xd44ff710u : 0x218d4090u); // s139 add - r1 = rotr_var(r1, r4); // s140 rotr - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 1u); r3 = r3 ^ t_; } // s141 shfl - r7 = r6 * r2 + r7; // s142 mad - r2 = r2 * r3; // s143 mul - r7 = r7 + r4 + ((((sel >> 29u) & 1u) != 0u) ? 0xb98942fau : 0xf4b1a8deu); // s144 add - r7 = rotl_imm(r7, 15u); // s145 rotl - r7 = r7 ^ r5; // s146 xor - r4 = r7 * r4 + r4; // s147 mad - r6 = rotr_var(r6, r5); // s148 rotr - r1 = r2 * r4 + r1; // s149 mad - r1 = r1 + r7 + ((((sel >> 17u) & 1u) != 0u) ? 0x0d48ba42u : 0x2bef10f2u); // s150 add - r5 = r5 ^ r4; // s151 xor - r7 = r7 + r0 + ((((sel >> 30u) & 1u) != 0u) ? 0xc98dea9cu : 0xdc5cc080u); // s152 add - r4 = r4 * r6; // s153 mul - r0 = r0 * r2; // s154 mul - r6 = r6 ^ r5; // s155 xor - r4 = rotr_var(r4, r2); // s156 rotr - r1 = rotl_imm(r1, 11u); // s157 rotl - r5 = r5 + r0 + ((((sel >> 11u) & 1u) != 0u) ? 0x6c752dcbu : 0x18197438u); // s158 add - r4 = r1 * r5 + r4; // s159 mad - r4 = rotl_imm(r4, 26u); // s160 rotl - r3 = r0 * r7 + r3; // s161 mad - r3 = rotr_var(r3, r1); // s162 rotr - r4 = r4 + r0 + ((((sel >> 3u) & 1u) != 0u) ? 0x8e481727u : 0xf1c46574u); // s163 add - r0 = mul_hi(r0, r4); // s164 mulhi - r2 = r2 + r6 + ((((sel >> 20u) & 1u) != 0u) ? 0x550ab406u : 0xac578137u); // s165 add - r0 = rotl_imm(r0, 13u); // s166 rotl - r3 = r3 + r1 + ((((sel >> 14u) & 1u) != 0u) ? 0xaebb5966u : 0xa33e6706u); // s167 add - r3 = r3 | r5; // s168 or - r6 = rotr_var(r6, r2); // s169 rotr - r4 = r4 ^ r6; // s170 xor - r6 = r6 - r1; // s171 sub - r7 = rotl_imm(r7, 22u); // s172 rotl - r5 = rotl_imm(r5, 15u); // s173 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r0, 8u); r7 = r7 ^ t_; } // s174 shfl - r0 = r0 ^ r5; // s175 xor - r7 = rotl_imm(r7, 6u); // s176 rotl - r7 = r7 - r0; // s177 sub - r3 = rotl_imm(r3, 30u); // s178 rotl - r7 = r6 * r1 + r7; // s179 mad - r6 = rotl_imm(r6, 9u); // s180 rotl - r2 = r2 ^ r4; // s181 xor - r2 = r2 ^ r7; // s182 xor - r7 = r7 ^ r2; // s183 xor - r1 = r1 + r2 + ((((sel >> 21u) & 1u) != 0u) ? 0x5bb7550fu : 0xd94d55acu); // s184 add - r3 = r3 | r5; // s185 or - r6 = mul_hi(r6, r3); // s186 mulhi - r4 = r4 | r0; // s187 or - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 2u); r7 = r7 ^ t_; } // s188 shfl - r6 = r6 + r5 + ((((sel >> 6u) & 1u) != 0u) ? 0x2a354e2du : 0x89e747feu); // s189 add - r1 = r1 ^ r4; // s190 xor - r7 = mul_hi(r7, r4); // s191 mulhi - r2 = rotl_imm(r2, 26u); // s192 rotl - r5 = rotl_imm(r5, 8u); // s193 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 16u); r4 = r4 ^ t_; } // s194 shfl - r4 = r4 ^ r5; // s195 xor - r1 = r1 * r3; // s196 mul - r5 = r5 + r2 + ((((sel >> 7u) & 1u) != 0u) ? 0x10cfdc71u : 0xea03e8e7u); // s197 add - r7 = r7 + r5 + ((((sel >> 15u) & 1u) != 0u) ? 0x66e148d3u : 0xa7ee0102u); // s198 add - r5 = r5 - r2; // s199 sub - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 2u); r5 = r5 ^ t_; } // s200 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 8u); r7 = r7 ^ t_; } // s201 shfl - r4 = rotr_var(r4, r5); // s202 rotr - r7 = r7 ^ r5; // s203 xor - r7 = r7 ^ r0; // s204 xor - r6 = r6 + r2 + ((((sel >> 10u) & 1u) != 0u) ? 0x8558b619u : 0x8379a4deu); // s205 add - r4 = rotl_imm(r4, 13u); // s206 rotl - r1 = mul_hi(r1, r3); // s207 mulhi - r1 = r4 * r4 + r1; // s208 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r0, 4u); r2 = r2 ^ t_; } // s209 shfl - r7 = r7 + r0 + ((((sel >> 11u) & 1u) != 0u) ? 0xf4049c4cu : 0xaa8cb14eu); // s210 add - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 16u); r7 = r7 ^ t_; } // s211 shfl - r1 = r1 ^ r0; // s212 xor - r7 = rotl_imm(r7, 3u); // s213 rotl - r4 = rotr_var(r4, r7); // s214 rotr - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 16u); r3 = r3 ^ t_; } // s215 shfl - r5 = r5 | r1; // s216 or - r1 = r1 - r2; // s217 sub - r6 = r6 - r5; // s218 sub - r6 = rotl_imm(r6, 4u); // s219 rotl - r2 = mul_hi(r2, r0); // s220 mulhi - r2 = r2 | r0; // s221 or - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 8u); r5 = r5 ^ t_; } // s222 shfl - r5 = mul_hi(r5, r6); // s223 mulhi - r0 = r0 - r6; // s224 sub - r7 = rotl_imm(r7, 23u); // s225 rotl - r4 = r4 | r2; // s226 or - r2 = r2 * r4; // s227 mul - r3 = rotl_imm(r3, 12u); // s228 rotl - r0 = rotr_var(r0, r4); // s229 rotr - r0 = r0 + r6 + ((((sel >> 16u) & 1u) != 0u) ? 0x2a7fecb2u : 0x1d176220u); // s230 add - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 4u); r1 = r1 ^ t_; } // s231 shfl - r2 = r2 * r1; // s232 mul - r7 = r0 * r1 + r7; // s233 mad - r5 = rotl_imm(r5, 22u); // s234 rotl - r4 = rotr_var(r4, r6); // s235 rotr - r0 = r5 * r1 + r0; // s236 mad - r6 = r6 ^ r4; // s237 xor - r4 = r4 ^ r6; // s238 xor - r6 = rotl_imm(r6, 18u); // s239 rotl - r4 = r4 + r7 + ((((sel >> 25u) & 1u) != 0u) ? 0xadce39f3u : 0x17dafb4du); // s240 add - r0 = r0 - r7; // s241 sub - r1 = rotr_var(r1, r6); // s242 rotr - r3 = r3 + r0 + ((((sel >> 29u) & 1u) != 0u) ? 0x0e7033b6u : 0xf2f77d26u); // s243 add - r2 = r7 * r4 + r2; // s244 mad - r4 = r0 * r6 + r4; // s245 mad - r5 = r5 * r7; // s246 mul - r3 = r3 + r5 + ((((sel >> 31u) & 1u) != 0u) ? 0x7b2ff6b7u : 0xfed2da4eu); // s247 add - r0 = r0 + r7 + ((((sel >> 0u) & 1u) != 0u) ? 0xb5fad7b1u : 0x03b2891cu); // s248 add - r5 = mul_hi(r5, r4); // s249 mulhi - r5 = r5 + r2 + ((((sel >> 11u) & 1u) != 0u) ? 0xa7e71c2fu : 0x042cd6e3u); // s250 add - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 1u); r6 = r6 ^ t_; } // s251 shfl - r6 = r6 ^ r1; // s252 xor - r2 = r2 * r5; // s253 mul - r0 = r0 + r6 + ((((sel >> 16u) & 1u) != 0u) ? 0xb83d78deu : 0x784a302bu); // s254 add - r2 = r2 - r4; // s255 sub - } - } - uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((ulong)hi << 32) | (ulong)lo; -} - -#if IGNEUM_EXCHANGE != 0 -// Reports the sub-group size this device uses for a work-group of IGNEUM_GROUP items. host.c runs it only when the -// per-kernel query (clGetKernelSubGroupInfoKHR on igneum_hash) is unavailable; that query is preferred because a -// compiler may pick a different wave width per kernel (RDNA: wave32 or wave64). See WAVEFRONT.md. -IGNEUM_KERNEL_HASH void igneum_probe_subgroup(__global uint* out) { - if (get_local_id(0) == 0u) { out[0] = get_sub_group_size(); out[1] = get_num_sub_groups(); } -} -#endif - -// Header-bound variant (bind.rs): the init words come from initw, not SEEDW. Same body as igneum_hash. -IGNEUM_KERNEL_HASH void igneum_hash_bound(__global const uint* ds, __global ulong* out, uint baseNonce, uint mask, __global const uint* initw) { - uint gid = (uint)get_global_id(0); - uint lid = (uint)get_local_id(0); - uint nonce = baseNonce + gid; - uint r0, r1, r2, r3, r4, r5, r6, r7; - uint iw0 = initw[0], iw1 = initw[1], iw2 = initw[2], iw3 = initw[3], iw4 = initw[4], iw5 = initw[5], iw6 = initw[6], iw7 = initw[7]; -#if IGNEUM_EXCHANGE == 0 - IGNEUM_LOCAL_WORDS(xch, 2 * IGNEUM_GROUP); - uint xk = 0u; -#else - (void)lid; -#endif - { uint x = nonce ^ iw0; x += 0x9e3779b9u * 1u; x = splitmix32(x); r0 = x ^ iw1; } - { uint x = nonce ^ iw1; x += 0x9e3779b9u * 2u; x = splitmix32(x); r1 = x ^ iw2; } - { uint x = nonce ^ iw2; x += 0x9e3779b9u * 3u; x = splitmix32(x); r2 = x ^ iw3; } - { uint x = nonce ^ iw3; x += 0x9e3779b9u * 4u; x = splitmix32(x); r3 = x ^ iw4; } - { uint x = nonce ^ iw4; x += 0x9e3779b9u * 5u; x = splitmix32(x); r4 = x ^ iw5; } - { uint x = nonce ^ iw5; x += 0x9e3779b9u * 6u; x = splitmix32(x); r5 = x ^ iw6; } - { uint x = nonce ^ iw6; x += 0x9e3779b9u * 7u; x = splitmix32(x); r6 = x ^ iw7; } - { uint x = nonce ^ iw7; x += 0x9e3779b9u * 8u; x = splitmix32(x); r7 = x ^ iw0; } - - for (uint it = 0u; it < 8u; ++it) { - uint sel = r0; - r2 = r3 * r4 + r2; // 0 mad - r2 = r1 * r1 + r2; // 1 mad - r2 = r3 * r2 + r2; // 2 mad - r3 = r3 ^ r5; // 3 xor - r7 = r7 ^ ds[r2 & mask]; // 4 load - r5 = r5 ^ ds[r7 & mask]; // 5 load - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 8u); r1 = r1 ^ t_; } // 6 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 8u); r7 = r7 ^ t_; } // 7 shfl - r1 = mul_hi(r1, r5); // 8 mulhi - r6 = rotr_var(r6, r3); // 9 rotr - r3 = r3 | r4; // 10 or - r4 = r4 ^ ds[r3 & mask]; // 11 load - r0 = mul_hi(r0, r4); // 12 mulhi - r5 = r5 + r1 + ((((sel >> 30u) & 1u) != 0u) ? 0xd3177981u : 0xc7934706u); // 13 add - r0 = r0 ^ ds[r4 & mask]; // 14 load - r2 = r2 - r4; // 15 sub - r2 = r2 ^ ds[r0 & mask]; // 16 load - r7 = r7 ^ ds[r2 & mask]; // 17 load - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 4u); r7 = r7 ^ t_; } // 18 shfl - r5 = r5 * r0; // 19 mul - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 2u); r3 = r3 ^ t_; } // 20 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 16u); r2 = r2 ^ t_; } // 21 shfl - r6 = mul_hi(r6, r2); // 22 mulhi - r6 = r6 ^ ds[r1 & mask]; // 23 load - r5 = r5 * r0; // 24 mul - r5 = rotl_imm(r5, 19u); // 25 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 2u); r7 = r7 ^ t_; } // 26 shfl - r0 = r0 ^ r5; // 27 xor - r0 = r0 ^ r4; // 28 xor - r3 = r3 - r0; // 29 sub - r5 = r5 * r1; // 30 mul - r7 = r7 ^ ds[r2 & mask]; // 31 load - r1 = r1 ^ ds[r0 & mask]; // 32 load - r5 = r5 ^ r6; // 33 xor - r5 = r5 ^ ds[r1 & mask]; // 34 load - r0 = mul_hi(r0, r5); // 35 mulhi - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 4u); r5 = r5 ^ t_; } // 36 shfl - r7 = r7 ^ ds[r0 & mask]; // 37 load - r3 = r3 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0x230c005cu : 0x75ba2fadu); // 38 add - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 4u); r1 = r1 ^ t_; } // 39 shfl - r2 = r2 ^ r5; // 40 xor - r3 = r6 * r3 + r3; // 41 mad - r6 = r6 - r7; // 42 sub - r7 = r7 ^ r0; // 43 xor - r1 = r1 ^ ds[r7 & mask]; // 44 load - r2 = r2 * r3; // 45 mul - r1 = mul_hi(r1, r5); // 46 mulhi - r4 = r4 - r3; // 47 sub - r2 = rotr_var(r2, r6); // 48 rotr - r3 = r3 ^ ds[r5 & mask]; // 49 load - r1 = r1 + r5 + ((((sel >> 7u) & 1u) != 0u) ? 0x1907970cu : 0x81b8bc2cu); // 50 add - r0 = r0 * r2; // 51 mul - r0 = r0 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x699fd448u : 0x4f92b968u); // 52 add - r1 = r1 + r0 + ((((sel >> 12u) & 1u) != 0u) ? 0x77b1520du : 0x2bb965afu); // 53 add - r7 = rotl_imm(r7, 14u); // 54 rotl - r3 = r3 + r7 + ((((sel >> 1u) & 1u) != 0u) ? 0xa54c55a0u : 0x7b0fe07au); // 55 add - r6 = r6 ^ ds[r7 & mask]; // 56 load - r1 = rotr_var(r1, r5); // 57 rotr - r5 = r5 ^ ds[r4 & mask]; // 58 load - r6 = r6 ^ ds[r2 & mask]; // 59 load - r3 = r5 * r0 + r3; // 60 mad - r5 = r5 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0xad7493e7u : 0xaf9dd72du); // 61 add - r4 = r4 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x1e07c3d9u : 0x89841d87u); // 62 add - r5 = rotl_imm(r5, 19u); // 63 rotl - // latency-shadow block (Counter ASIC 3.0 item 8): 256 ALU instructions x 27 passes after instruction 63, no load - for (uint sh = 0u; sh < 27u; ++sh) { - r5 = r5 + r2 + ((((sel >> 18u) & 1u) != 0u) ? 0x8bc12da9u : 0x92199f99u); // s0 add - r0 = r0 + r7 + ((((sel >> 26u) & 1u) != 0u) ? 0x63079e5au : 0x8d72d3adu); // s1 add - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 2u); r6 = r6 ^ t_; } // s2 shfl - r4 = r4 - r2; // s3 sub - r7 = r7 + r0 + ((((sel >> 31u) & 1u) != 0u) ? 0x5d4c7a60u : 0xb21b4babu); // s4 add - r0 = rotl_imm(r0, 11u); // s5 rotl - r0 = r0 + r1 + ((((sel >> 16u) & 1u) != 0u) ? 0xc88e2942u : 0x2fe0e98bu); // s6 add - r7 = r7 ^ r1; // s7 xor - r1 = r6 * r5 + r1; // s8 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 4u); r6 = r6 ^ t_; } // s9 shfl - r1 = r2 * r2 + r1; // s10 mad - r5 = r0 * r3 + r5; // s11 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 4u); r2 = r2 ^ t_; } // s12 shfl - r1 = r1 + r5 + ((((sel >> 25u) & 1u) != 0u) ? 0xbc48c63eu : 0xbdf8f9a5u); // s13 add - r1 = rotl_imm(r1, 29u); // s14 rotl - r1 = r1 - r4; // s15 sub - r7 = r7 | r1; // s16 or - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 8u); r2 = r2 ^ t_; } // s17 shfl - r7 = r7 ^ r4; // s18 xor - r6 = r6 * r1; // s19 mul - r5 = r6 * r0 + r5; // s20 mad - r3 = r3 - r1; // s21 sub - r6 = r6 * r0; // s22 mul - r2 = r2 + r0 + ((((sel >> 1u) & 1u) != 0u) ? 0x96e8f127u : 0x45c37cecu); // s23 add - r6 = r6 - r4; // s24 sub - r7 = r3 * r4 + r7; // s25 mad - r3 = rotl_imm(r3, 9u); // s26 rotl - r2 = r2 - r1; // s27 sub - r6 = mul_hi(r6, r0); // s28 mulhi - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 2u); r2 = r2 ^ t_; } // s29 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 1u); r1 = r1 ^ t_; } // s30 shfl - r1 = r1 + r6 + ((((sel >> 1u) & 1u) != 0u) ? 0xb1cdb2abu : 0x37985632u); // s31 add - r0 = rotr_var(r0, r1); // s32 rotr - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 2u); r3 = r3 ^ t_; } // s33 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 4u); r0 = r0 ^ t_; } // s34 shfl - r7 = r7 * r0; // s35 mul - r3 = r3 + r1 + ((((sel >> 0u) & 1u) != 0u) ? 0x856e0180u : 0x804e777eu); // s36 add - r1 = r1 ^ r6; // s37 xor - r2 = r2 * r7; // s38 mul - r6 = r6 + r5 + ((((sel >> 2u) & 1u) != 0u) ? 0xe9bd0cc4u : 0x0b06c8a7u); // s39 add - r5 = r5 * r7; // s40 mul - r2 = mul_hi(r2, r3); // s41 mulhi - r2 = r2 ^ r0; // s42 xor - r0 = rotl_imm(r0, 4u); // s43 rotl - r4 = mul_hi(r4, r3); // s44 mulhi - r6 = r6 + r7 + ((((sel >> 12u) & 1u) != 0u) ? 0xcdcb63f6u : 0xee822e17u); // s45 add - r3 = r2 * r0 + r3; // s46 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 8u); r4 = r4 ^ t_; } // s47 shfl - r6 = mul_hi(r6, r5); // s48 mulhi - r0 = r0 - r2; // s49 sub - r3 = r3 - r5; // s50 sub - r1 = rotr_var(r1, r4); // s51 rotr - r6 = r7 * r7 + r6; // s52 mad - r5 = r5 ^ r3; // s53 xor - r1 = r1 - r0; // s54 sub - r5 = r5 - r6; // s55 sub - r3 = r3 + r2 + ((((sel >> 10u) & 1u) != 0u) ? 0xd4758987u : 0x3dfad1b6u); // s56 add - r4 = r4 + r1 + ((((sel >> 0u) & 1u) != 0u) ? 0x0602d6beu : 0xfca75bc2u); // s57 add - r5 = mul_hi(r5, r6); // s58 mulhi - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 2u); r2 = r2 ^ t_; } // s59 shfl - r2 = rotl_imm(r2, 9u); // s60 rotl - r4 = r4 | r6; // s61 or - r6 = rotr_var(r6, r4); // s62 rotr - r2 = r2 * r4; // s63 mul - r0 = r0 ^ r5; // s64 xor - r2 = r2 ^ r7; // s65 xor - r2 = r2 + r1 + ((((sel >> 6u) & 1u) != 0u) ? 0x8596687au : 0xd71c02ffu); // s66 add - r1 = rotl_imm(r1, 21u); // s67 rotl - r3 = r3 * r2; // s68 mul - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 2u); r7 = r7 ^ t_; } // s69 shfl - r3 = r3 * r2; // s70 mul - r0 = r2 * r5 + r0; // s71 mad - r6 = r6 + r7 + ((((sel >> 0u) & 1u) != 0u) ? 0x08ac5733u : 0x3c6fe15du); // s72 add - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 8u); r3 = r3 ^ t_; } // s73 shfl - r3 = r3 * r6; // s74 mul - r6 = r6 ^ r7; // s75 xor - r3 = r3 | r0; // s76 or - r2 = r2 + r4 + ((((sel >> 1u) & 1u) != 0u) ? 0xc100b495u : 0x295d5faeu); // s77 add - r3 = mul_hi(r3, r7); // s78 mulhi - r4 = r4 | r1; // s79 or - r4 = rotr_var(r4, r3); // s80 rotr - r4 = r4 + r3 + ((((sel >> 15u) & 1u) != 0u) ? 0x5cc59530u : 0x274a9221u); // s81 add - r7 = r7 + r3 + ((((sel >> 5u) & 1u) != 0u) ? 0x141479ecu : 0xc8651f8eu); // s82 add - r3 = rotr_var(r3, r6); // s83 rotr - r2 = r4 * r7 + r2; // s84 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 16u); r2 = r2 ^ t_; } // s85 shfl - r3 = r3 ^ r2; // s86 xor - { uint t_; IGNEUM_SHFL_XOR(t_, r7, 2u); r5 = r5 ^ t_; } // s87 shfl - r0 = r0 ^ r4; // s88 xor - r3 = r3 + r2 + ((((sel >> 18u) & 1u) != 0u) ? 0x883c0c92u : 0x53f915c2u); // s89 add - r7 = rotr_var(r7, r5); // s90 rotr - r3 = r3 + r1 + ((((sel >> 31u) & 1u) != 0u) ? 0x3cc5bd20u : 0xe2be00a5u); // s91 add - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 1u); r7 = r7 ^ t_; } // s92 shfl - r6 = rotr_var(r6, r2); // s93 rotr - r7 = rotl_imm(r7, 14u); // s94 rotl - r4 = r5 * r5 + r4; // s95 mad - r2 = r4 * r5 + r2; // s96 mad - r1 = r1 ^ r0; // s97 xor - r5 = r5 * r4; // s98 mul - r2 = r2 - r0; // s99 sub - r7 = rotl_imm(r7, 30u); // s100 rotl - r5 = r5 + r6 + ((((sel >> 17u) & 1u) != 0u) ? 0xbfd646cbu : 0x423fd9c9u); // s101 add - r7 = r7 + r6 + ((((sel >> 11u) & 1u) != 0u) ? 0x19b74a43u : 0x8d3c011du); // s102 add - r5 = rotl_imm(r5, 6u); // s103 rotl - r0 = r0 * r4; // s104 mul - r0 = r0 | r5; // s105 or - r0 = r0 + r1 + ((((sel >> 15u) & 1u) != 0u) ? 0x7dcb7e18u : 0x8ce14721u); // s106 add - r0 = rotl_imm(r0, 15u); // s107 rotl - r4 = r4 - r2; // s108 sub - r2 = mul_hi(r2, r0); // s109 mulhi - r1 = mul_hi(r1, r0); // s110 mulhi - r0 = r0 + r2 + ((((sel >> 28u) & 1u) != 0u) ? 0x3c179ad8u : 0x2d9fc6b9u); // s111 add - r2 = mul_hi(r2, r0); // s112 mulhi - r6 = r6 - r3; // s113 sub - r7 = r6 * r3 + r7; // s114 mad - r5 = rotr_var(r5, r1); // s115 rotr - r6 = rotr_var(r6, r4); // s116 rotr - r2 = r2 + r1 + ((((sel >> 22u) & 1u) != 0u) ? 0x47359729u : 0x502138e2u); // s117 add - r3 = r2 * r0 + r3; // s118 mad - r1 = r1 * r3; // s119 mul - r2 = r0 * r4 + r2; // s120 mad - r0 = r0 * r3; // s121 mul - r2 = mul_hi(r2, r0); // s122 mulhi - r5 = r5 * r6; // s123 mul - r4 = r2 * r4 + r4; // s124 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 2u); r4 = r4 ^ t_; } // s125 shfl - r2 = r2 ^ r0; // s126 xor - r1 = rotl_imm(r1, 7u); // s127 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 4u); r7 = r7 ^ t_; } // s128 shfl - r6 = rotl_imm(r6, 17u); // s129 rotl - r2 = r2 + r5 + ((((sel >> 15u) & 1u) != 0u) ? 0x6c57e4e7u : 0x33dff776u); // s130 add - r0 = r0 | r3; // s131 or - r5 = rotr_var(r5, r0); // s132 rotr - r2 = r2 + r3 + ((((sel >> 30u) & 1u) != 0u) ? 0xe8212c0cu : 0x5db25b34u); // s133 add - r4 = r4 ^ r3; // s134 xor - r6 = r6 ^ r1; // s135 xor - r1 = r1 - r7; // s136 sub - r2 = r6 * r2 + r2; // s137 mad - r0 = mul_hi(r0, r5); // s138 mulhi - r2 = r2 + r7 + ((((sel >> 10u) & 1u) != 0u) ? 0xd44ff710u : 0x218d4090u); // s139 add - r1 = rotr_var(r1, r4); // s140 rotr - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 1u); r3 = r3 ^ t_; } // s141 shfl - r7 = r6 * r2 + r7; // s142 mad - r2 = r2 * r3; // s143 mul - r7 = r7 + r4 + ((((sel >> 29u) & 1u) != 0u) ? 0xb98942fau : 0xf4b1a8deu); // s144 add - r7 = rotl_imm(r7, 15u); // s145 rotl - r7 = r7 ^ r5; // s146 xor - r4 = r7 * r4 + r4; // s147 mad - r6 = rotr_var(r6, r5); // s148 rotr - r1 = r2 * r4 + r1; // s149 mad - r1 = r1 + r7 + ((((sel >> 17u) & 1u) != 0u) ? 0x0d48ba42u : 0x2bef10f2u); // s150 add - r5 = r5 ^ r4; // s151 xor - r7 = r7 + r0 + ((((sel >> 30u) & 1u) != 0u) ? 0xc98dea9cu : 0xdc5cc080u); // s152 add - r4 = r4 * r6; // s153 mul - r0 = r0 * r2; // s154 mul - r6 = r6 ^ r5; // s155 xor - r4 = rotr_var(r4, r2); // s156 rotr - r1 = rotl_imm(r1, 11u); // s157 rotl - r5 = r5 + r0 + ((((sel >> 11u) & 1u) != 0u) ? 0x6c752dcbu : 0x18197438u); // s158 add - r4 = r1 * r5 + r4; // s159 mad - r4 = rotl_imm(r4, 26u); // s160 rotl - r3 = r0 * r7 + r3; // s161 mad - r3 = rotr_var(r3, r1); // s162 rotr - r4 = r4 + r0 + ((((sel >> 3u) & 1u) != 0u) ? 0x8e481727u : 0xf1c46574u); // s163 add - r0 = mul_hi(r0, r4); // s164 mulhi - r2 = r2 + r6 + ((((sel >> 20u) & 1u) != 0u) ? 0x550ab406u : 0xac578137u); // s165 add - r0 = rotl_imm(r0, 13u); // s166 rotl - r3 = r3 + r1 + ((((sel >> 14u) & 1u) != 0u) ? 0xaebb5966u : 0xa33e6706u); // s167 add - r3 = r3 | r5; // s168 or - r6 = rotr_var(r6, r2); // s169 rotr - r4 = r4 ^ r6; // s170 xor - r6 = r6 - r1; // s171 sub - r7 = rotl_imm(r7, 22u); // s172 rotl - r5 = rotl_imm(r5, 15u); // s173 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r0, 8u); r7 = r7 ^ t_; } // s174 shfl - r0 = r0 ^ r5; // s175 xor - r7 = rotl_imm(r7, 6u); // s176 rotl - r7 = r7 - r0; // s177 sub - r3 = rotl_imm(r3, 30u); // s178 rotl - r7 = r6 * r1 + r7; // s179 mad - r6 = rotl_imm(r6, 9u); // s180 rotl - r2 = r2 ^ r4; // s181 xor - r2 = r2 ^ r7; // s182 xor - r7 = r7 ^ r2; // s183 xor - r1 = r1 + r2 + ((((sel >> 21u) & 1u) != 0u) ? 0x5bb7550fu : 0xd94d55acu); // s184 add - r3 = r3 | r5; // s185 or - r6 = mul_hi(r6, r3); // s186 mulhi - r4 = r4 | r0; // s187 or - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 2u); r7 = r7 ^ t_; } // s188 shfl - r6 = r6 + r5 + ((((sel >> 6u) & 1u) != 0u) ? 0x2a354e2du : 0x89e747feu); // s189 add - r1 = r1 ^ r4; // s190 xor - r7 = mul_hi(r7, r4); // s191 mulhi - r2 = rotl_imm(r2, 26u); // s192 rotl - r5 = rotl_imm(r5, 8u); // s193 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 16u); r4 = r4 ^ t_; } // s194 shfl - r4 = r4 ^ r5; // s195 xor - r1 = r1 * r3; // s196 mul - r5 = r5 + r2 + ((((sel >> 7u) & 1u) != 0u) ? 0x10cfdc71u : 0xea03e8e7u); // s197 add - r7 = r7 + r5 + ((((sel >> 15u) & 1u) != 0u) ? 0x66e148d3u : 0xa7ee0102u); // s198 add - r5 = r5 - r2; // s199 sub - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 2u); r5 = r5 ^ t_; } // s200 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 8u); r7 = r7 ^ t_; } // s201 shfl - r4 = rotr_var(r4, r5); // s202 rotr - r7 = r7 ^ r5; // s203 xor - r7 = r7 ^ r0; // s204 xor - r6 = r6 + r2 + ((((sel >> 10u) & 1u) != 0u) ? 0x8558b619u : 0x8379a4deu); // s205 add - r4 = rotl_imm(r4, 13u); // s206 rotl - r1 = mul_hi(r1, r3); // s207 mulhi - r1 = r4 * r4 + r1; // s208 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r0, 4u); r2 = r2 ^ t_; } // s209 shfl - r7 = r7 + r0 + ((((sel >> 11u) & 1u) != 0u) ? 0xf4049c4cu : 0xaa8cb14eu); // s210 add - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 16u); r7 = r7 ^ t_; } // s211 shfl - r1 = r1 ^ r0; // s212 xor - r7 = rotl_imm(r7, 3u); // s213 rotl - r4 = rotr_var(r4, r7); // s214 rotr - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 16u); r3 = r3 ^ t_; } // s215 shfl - r5 = r5 | r1; // s216 or - r1 = r1 - r2; // s217 sub - r6 = r6 - r5; // s218 sub - r6 = rotl_imm(r6, 4u); // s219 rotl - r2 = mul_hi(r2, r0); // s220 mulhi - r2 = r2 | r0; // s221 or - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 8u); r5 = r5 ^ t_; } // s222 shfl - r5 = mul_hi(r5, r6); // s223 mulhi - r0 = r0 - r6; // s224 sub - r7 = rotl_imm(r7, 23u); // s225 rotl - r4 = r4 | r2; // s226 or - r2 = r2 * r4; // s227 mul - r3 = rotl_imm(r3, 12u); // s228 rotl - r0 = rotr_var(r0, r4); // s229 rotr - r0 = r0 + r6 + ((((sel >> 16u) & 1u) != 0u) ? 0x2a7fecb2u : 0x1d176220u); // s230 add - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 4u); r1 = r1 ^ t_; } // s231 shfl - r2 = r2 * r1; // s232 mul - r7 = r0 * r1 + r7; // s233 mad - r5 = rotl_imm(r5, 22u); // s234 rotl - r4 = rotr_var(r4, r6); // s235 rotr - r0 = r5 * r1 + r0; // s236 mad - r6 = r6 ^ r4; // s237 xor - r4 = r4 ^ r6; // s238 xor - r6 = rotl_imm(r6, 18u); // s239 rotl - r4 = r4 + r7 + ((((sel >> 25u) & 1u) != 0u) ? 0xadce39f3u : 0x17dafb4du); // s240 add - r0 = r0 - r7; // s241 sub - r1 = rotr_var(r1, r6); // s242 rotr - r3 = r3 + r0 + ((((sel >> 29u) & 1u) != 0u) ? 0x0e7033b6u : 0xf2f77d26u); // s243 add - r2 = r7 * r4 + r2; // s244 mad - r4 = r0 * r6 + r4; // s245 mad - r5 = r5 * r7; // s246 mul - r3 = r3 + r5 + ((((sel >> 31u) & 1u) != 0u) ? 0x7b2ff6b7u : 0xfed2da4eu); // s247 add - r0 = r0 + r7 + ((((sel >> 0u) & 1u) != 0u) ? 0xb5fad7b1u : 0x03b2891cu); // s248 add - r5 = mul_hi(r5, r4); // s249 mulhi - r5 = r5 + r2 + ((((sel >> 11u) & 1u) != 0u) ? 0xa7e71c2fu : 0x042cd6e3u); // s250 add - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 1u); r6 = r6 ^ t_; } // s251 shfl - r6 = r6 ^ r1; // s252 xor - r2 = r2 * r5; // s253 mul - r0 = r0 + r6 + ((((sel >> 16u) & 1u) != 0u) ? 0xb83d78deu : 0x784a302bu); // s254 add - r2 = r2 - r4; // s255 sub - } - } - uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((ulong)hi << 32) | (ulong)lo; -} diff --git a/proto-cuda/packs-ca4/mx8_sh256x27/kernel_bound.cu b/proto-cuda/packs-ca4/mx8_sh256x27/kernel_bound.cu deleted file mode 100644 index 4a823531b..000000000 --- a/proto-cuda/packs-ca4/mx8_sh256x27/kernel_bound.cu +++ /dev/null @@ -1,382 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// Header-bound twin of igneum_hash in kernel.cu: the init words come from a kernel argument, not SEEDW. -// Host declarations (also in program_bound.h if present): -// struct IgneumInitWords { uint32_t w[8]; }; -// cudaError_t igneum_launch_hash_bound(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask, -// IgneumInitWords iw, uint32_t nonces, uint32_t blockWarps); -// cudaError_t igneum_hash_bound_info(int* numRegs, int* blocksPerSM, uint32_t blockWarps); -#include -#include -#include "program.h" - -struct IgneumInitWords { uint32_t w[8]; }; - -__device__ __forceinline__ uint32_t splitmix32(uint32_t x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -__device__ __forceinline__ uint32_t rotl_imm(uint32_t x, uint32_t n) { return (x << n) | (x >> (32u - n)); } -__device__ __forceinline__ uint32_t rotr_var(uint32_t x, uint32_t n) { n &= 31u; return (x >> n) | (x << ((32u - n) & 31u)); } - -__global__ void igneum_hash_bound(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask, IgneumInitWords iw) { - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t nonce = baseNonce + gid; - uint32_t r0, r1, r2, r3, r4, r5, r6, r7; - { uint32_t x = nonce ^ iw.w[0]; x += 0x9e3779b9u * 1u; x = splitmix32(x); r0 = x ^ iw.w[1]; } - { uint32_t x = nonce ^ iw.w[1]; x += 0x9e3779b9u * 2u; x = splitmix32(x); r1 = x ^ iw.w[2]; } - { uint32_t x = nonce ^ iw.w[2]; x += 0x9e3779b9u * 3u; x = splitmix32(x); r2 = x ^ iw.w[3]; } - { uint32_t x = nonce ^ iw.w[3]; x += 0x9e3779b9u * 4u; x = splitmix32(x); r3 = x ^ iw.w[4]; } - { uint32_t x = nonce ^ iw.w[4]; x += 0x9e3779b9u * 5u; x = splitmix32(x); r4 = x ^ iw.w[5]; } - { uint32_t x = nonce ^ iw.w[5]; x += 0x9e3779b9u * 6u; x = splitmix32(x); r5 = x ^ iw.w[6]; } - { uint32_t x = nonce ^ iw.w[6]; x += 0x9e3779b9u * 7u; x = splitmix32(x); r6 = x ^ iw.w[7]; } - { uint32_t x = nonce ^ iw.w[7]; x += 0x9e3779b9u * 8u; x = splitmix32(x); r7 = x ^ iw.w[0]; } - - for (uint32_t it = 0u; it < 8u; ++it) { - uint32_t sel = r0; - r2 = r3 * r4 + r2; // 0 mad - r2 = r1 * r1 + r2; // 1 mad - r2 = r3 * r2 + r2; // 2 mad - r3 = r3 ^ r5; // 3 xor - r7 = r7 ^ ds[r2 & mask]; // 4 load - r5 = r5 ^ ds[r7 & mask]; // 5 load - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r4, 8); // 6 shfl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r3, 8); // 7 shfl - r1 = __umulhi(r1, r5); // 8 mulhi - r6 = rotr_var(r6, r3); // 9 rotr - r3 = r3 | r4; // 10 or - r4 = r4 ^ ds[r3 & mask]; // 11 load - r0 = __umulhi(r0, r4); // 12 mulhi - r5 = r5 + r1 + ((((sel >> 30u) & 1u) != 0u) ? 0xd3177981u : 0xc7934706u); // 13 add - r0 = r0 ^ ds[r4 & mask]; // 14 load - r2 = r2 - r4; // 15 sub - r2 = r2 ^ ds[r0 & mask]; // 16 load - r7 = r7 ^ ds[r2 & mask]; // 17 load - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r3, 4); // 18 shfl - r5 = r5 * r0; // 19 mul - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r4, 2); // 20 shfl - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r4, 16); // 21 shfl - r6 = __umulhi(r6, r2); // 22 mulhi - r6 = r6 ^ ds[r1 & mask]; // 23 load - r5 = r5 * r0; // 24 mul - r5 = rotl_imm(r5, 19u); // 25 rotl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r6, 2); // 26 shfl - r0 = r0 ^ r5; // 27 xor - r0 = r0 ^ r4; // 28 xor - r3 = r3 - r0; // 29 sub - r5 = r5 * r1; // 30 mul - r7 = r7 ^ ds[r2 & mask]; // 31 load - r1 = r1 ^ ds[r0 & mask]; // 32 load - r5 = r5 ^ r6; // 33 xor - r5 = r5 ^ ds[r1 & mask]; // 34 load - r0 = __umulhi(r0, r5); // 35 mulhi - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r2, 4); // 36 shfl - r7 = r7 ^ ds[r0 & mask]; // 37 load - r3 = r3 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0x230c005cu : 0x75ba2fadu); // 38 add - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r5, 4); // 39 shfl - r2 = r2 ^ r5; // 40 xor - r3 = r6 * r3 + r3; // 41 mad - r6 = r6 - r7; // 42 sub - r7 = r7 ^ r0; // 43 xor - r1 = r1 ^ ds[r7 & mask]; // 44 load - r2 = r2 * r3; // 45 mul - r1 = __umulhi(r1, r5); // 46 mulhi - r4 = r4 - r3; // 47 sub - r2 = rotr_var(r2, r6); // 48 rotr - r3 = r3 ^ ds[r5 & mask]; // 49 load - r1 = r1 + r5 + ((((sel >> 7u) & 1u) != 0u) ? 0x1907970cu : 0x81b8bc2cu); // 50 add - r0 = r0 * r2; // 51 mul - r0 = r0 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x699fd448u : 0x4f92b968u); // 52 add - r1 = r1 + r0 + ((((sel >> 12u) & 1u) != 0u) ? 0x77b1520du : 0x2bb965afu); // 53 add - r7 = rotl_imm(r7, 14u); // 54 rotl - r3 = r3 + r7 + ((((sel >> 1u) & 1u) != 0u) ? 0xa54c55a0u : 0x7b0fe07au); // 55 add - r6 = r6 ^ ds[r7 & mask]; // 56 load - r1 = rotr_var(r1, r5); // 57 rotr - r5 = r5 ^ ds[r4 & mask]; // 58 load - r6 = r6 ^ ds[r2 & mask]; // 59 load - r3 = r5 * r0 + r3; // 60 mad - r5 = r5 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0xad7493e7u : 0xaf9dd72du); // 61 add - r4 = r4 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x1e07c3d9u : 0x89841d87u); // 62 add - r5 = rotl_imm(r5, 19u); // 63 rotl - // latency-shadow block (Counter ASIC 3.0 item 8): 256 ALU instructions x 27 passes after instruction 63, no load - for (uint32_t sh = 0u; sh < 27u; ++sh) { - r5 = r5 + r2 + ((((sel >> 18u) & 1u) != 0u) ? 0x8bc12da9u : 0x92199f99u); // s0 add - r0 = r0 + r7 + ((((sel >> 26u) & 1u) != 0u) ? 0x63079e5au : 0x8d72d3adu); // s1 add - r6 = r6 ^ __shfl_xor_sync(0xffffffffu, r3, 2); // s2 shfl - r4 = r4 - r2; // s3 sub - r7 = r7 + r0 + ((((sel >> 31u) & 1u) != 0u) ? 0x5d4c7a60u : 0xb21b4babu); // s4 add - r0 = rotl_imm(r0, 11u); // s5 rotl - r0 = r0 + r1 + ((((sel >> 16u) & 1u) != 0u) ? 0xc88e2942u : 0x2fe0e98bu); // s6 add - r7 = r7 ^ r1; // s7 xor - r1 = r6 * r5 + r1; // s8 mad - r6 = r6 ^ __shfl_xor_sync(0xffffffffu, r1, 4); // s9 shfl - r1 = r2 * r2 + r1; // s10 mad - r5 = r0 * r3 + r5; // s11 mad - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r6, 4); // s12 shfl - r1 = r1 + r5 + ((((sel >> 25u) & 1u) != 0u) ? 0xbc48c63eu : 0xbdf8f9a5u); // s13 add - r1 = rotl_imm(r1, 29u); // s14 rotl - r1 = r1 - r4; // s15 sub - r7 = r7 | r1; // s16 or - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r4, 8); // s17 shfl - r7 = r7 ^ r4; // s18 xor - r6 = r6 * r1; // s19 mul - r5 = r6 * r0 + r5; // s20 mad - r3 = r3 - r1; // s21 sub - r6 = r6 * r0; // s22 mul - r2 = r2 + r0 + ((((sel >> 1u) & 1u) != 0u) ? 0x96e8f127u : 0x45c37cecu); // s23 add - r6 = r6 - r4; // s24 sub - r7 = r3 * r4 + r7; // s25 mad - r3 = rotl_imm(r3, 9u); // s26 rotl - r2 = r2 - r1; // s27 sub - r6 = __umulhi(r6, r0); // s28 mulhi - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r4, 2); // s29 shfl - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r6, 1); // s30 shfl - r1 = r1 + r6 + ((((sel >> 1u) & 1u) != 0u) ? 0xb1cdb2abu : 0x37985632u); // s31 add - r0 = rotr_var(r0, r1); // s32 rotr - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r4, 2); // s33 shfl - r0 = r0 ^ __shfl_xor_sync(0xffffffffu, r3, 4); // s34 shfl - r7 = r7 * r0; // s35 mul - r3 = r3 + r1 + ((((sel >> 0u) & 1u) != 0u) ? 0x856e0180u : 0x804e777eu); // s36 add - r1 = r1 ^ r6; // s37 xor - r2 = r2 * r7; // s38 mul - r6 = r6 + r5 + ((((sel >> 2u) & 1u) != 0u) ? 0xe9bd0cc4u : 0x0b06c8a7u); // s39 add - r5 = r5 * r7; // s40 mul - r2 = __umulhi(r2, r3); // s41 mulhi - r2 = r2 ^ r0; // s42 xor - r0 = rotl_imm(r0, 4u); // s43 rotl - r4 = __umulhi(r4, r3); // s44 mulhi - r6 = r6 + r7 + ((((sel >> 12u) & 1u) != 0u) ? 0xcdcb63f6u : 0xee822e17u); // s45 add - r3 = r2 * r0 + r3; // s46 mad - r4 = r4 ^ __shfl_xor_sync(0xffffffffu, r3, 8); // s47 shfl - r6 = __umulhi(r6, r5); // s48 mulhi - r0 = r0 - r2; // s49 sub - r3 = r3 - r5; // s50 sub - r1 = rotr_var(r1, r4); // s51 rotr - r6 = r7 * r7 + r6; // s52 mad - r5 = r5 ^ r3; // s53 xor - r1 = r1 - r0; // s54 sub - r5 = r5 - r6; // s55 sub - r3 = r3 + r2 + ((((sel >> 10u) & 1u) != 0u) ? 0xd4758987u : 0x3dfad1b6u); // s56 add - r4 = r4 + r1 + ((((sel >> 0u) & 1u) != 0u) ? 0x0602d6beu : 0xfca75bc2u); // s57 add - r5 = __umulhi(r5, r6); // s58 mulhi - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r1, 2); // s59 shfl - r2 = rotl_imm(r2, 9u); // s60 rotl - r4 = r4 | r6; // s61 or - r6 = rotr_var(r6, r4); // s62 rotr - r2 = r2 * r4; // s63 mul - r0 = r0 ^ r5; // s64 xor - r2 = r2 ^ r7; // s65 xor - r2 = r2 + r1 + ((((sel >> 6u) & 1u) != 0u) ? 0x8596687au : 0xd71c02ffu); // s66 add - r1 = rotl_imm(r1, 21u); // s67 rotl - r3 = r3 * r2; // s68 mul - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r5, 2); // s69 shfl - r3 = r3 * r2; // s70 mul - r0 = r2 * r5 + r0; // s71 mad - r6 = r6 + r7 + ((((sel >> 0u) & 1u) != 0u) ? 0x08ac5733u : 0x3c6fe15du); // s72 add - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r2, 8); // s73 shfl - r3 = r3 * r6; // s74 mul - r6 = r6 ^ r7; // s75 xor - r3 = r3 | r0; // s76 or - r2 = r2 + r4 + ((((sel >> 1u) & 1u) != 0u) ? 0xc100b495u : 0x295d5faeu); // s77 add - r3 = __umulhi(r3, r7); // s78 mulhi - r4 = r4 | r1; // s79 or - r4 = rotr_var(r4, r3); // s80 rotr - r4 = r4 + r3 + ((((sel >> 15u) & 1u) != 0u) ? 0x5cc59530u : 0x274a9221u); // s81 add - r7 = r7 + r3 + ((((sel >> 5u) & 1u) != 0u) ? 0x141479ecu : 0xc8651f8eu); // s82 add - r3 = rotr_var(r3, r6); // s83 rotr - r2 = r4 * r7 + r2; // s84 mad - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r5, 16); // s85 shfl - r3 = r3 ^ r2; // s86 xor - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r7, 2); // s87 shfl - r0 = r0 ^ r4; // s88 xor - r3 = r3 + r2 + ((((sel >> 18u) & 1u) != 0u) ? 0x883c0c92u : 0x53f915c2u); // s89 add - r7 = rotr_var(r7, r5); // s90 rotr - r3 = r3 + r1 + ((((sel >> 31u) & 1u) != 0u) ? 0x3cc5bd20u : 0xe2be00a5u); // s91 add - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r2, 1); // s92 shfl - r6 = rotr_var(r6, r2); // s93 rotr - r7 = rotl_imm(r7, 14u); // s94 rotl - r4 = r5 * r5 + r4; // s95 mad - r2 = r4 * r5 + r2; // s96 mad - r1 = r1 ^ r0; // s97 xor - r5 = r5 * r4; // s98 mul - r2 = r2 - r0; // s99 sub - r7 = rotl_imm(r7, 30u); // s100 rotl - r5 = r5 + r6 + ((((sel >> 17u) & 1u) != 0u) ? 0xbfd646cbu : 0x423fd9c9u); // s101 add - r7 = r7 + r6 + ((((sel >> 11u) & 1u) != 0u) ? 0x19b74a43u : 0x8d3c011du); // s102 add - r5 = rotl_imm(r5, 6u); // s103 rotl - r0 = r0 * r4; // s104 mul - r0 = r0 | r5; // s105 or - r0 = r0 + r1 + ((((sel >> 15u) & 1u) != 0u) ? 0x7dcb7e18u : 0x8ce14721u); // s106 add - r0 = rotl_imm(r0, 15u); // s107 rotl - r4 = r4 - r2; // s108 sub - r2 = __umulhi(r2, r0); // s109 mulhi - r1 = __umulhi(r1, r0); // s110 mulhi - r0 = r0 + r2 + ((((sel >> 28u) & 1u) != 0u) ? 0x3c179ad8u : 0x2d9fc6b9u); // s111 add - r2 = __umulhi(r2, r0); // s112 mulhi - r6 = r6 - r3; // s113 sub - r7 = r6 * r3 + r7; // s114 mad - r5 = rotr_var(r5, r1); // s115 rotr - r6 = rotr_var(r6, r4); // s116 rotr - r2 = r2 + r1 + ((((sel >> 22u) & 1u) != 0u) ? 0x47359729u : 0x502138e2u); // s117 add - r3 = r2 * r0 + r3; // s118 mad - r1 = r1 * r3; // s119 mul - r2 = r0 * r4 + r2; // s120 mad - r0 = r0 * r3; // s121 mul - r2 = __umulhi(r2, r0); // s122 mulhi - r5 = r5 * r6; // s123 mul - r4 = r2 * r4 + r4; // s124 mad - r4 = r4 ^ __shfl_xor_sync(0xffffffffu, r2, 2); // s125 shfl - r2 = r2 ^ r0; // s126 xor - r1 = rotl_imm(r1, 7u); // s127 rotl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r2, 4); // s128 shfl - r6 = rotl_imm(r6, 17u); // s129 rotl - r2 = r2 + r5 + ((((sel >> 15u) & 1u) != 0u) ? 0x6c57e4e7u : 0x33dff776u); // s130 add - r0 = r0 | r3; // s131 or - r5 = rotr_var(r5, r0); // s132 rotr - r2 = r2 + r3 + ((((sel >> 30u) & 1u) != 0u) ? 0xe8212c0cu : 0x5db25b34u); // s133 add - r4 = r4 ^ r3; // s134 xor - r6 = r6 ^ r1; // s135 xor - r1 = r1 - r7; // s136 sub - r2 = r6 * r2 + r2; // s137 mad - r0 = __umulhi(r0, r5); // s138 mulhi - r2 = r2 + r7 + ((((sel >> 10u) & 1u) != 0u) ? 0xd44ff710u : 0x218d4090u); // s139 add - r1 = rotr_var(r1, r4); // s140 rotr - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r6, 1); // s141 shfl - r7 = r6 * r2 + r7; // s142 mad - r2 = r2 * r3; // s143 mul - r7 = r7 + r4 + ((((sel >> 29u) & 1u) != 0u) ? 0xb98942fau : 0xf4b1a8deu); // s144 add - r7 = rotl_imm(r7, 15u); // s145 rotl - r7 = r7 ^ r5; // s146 xor - r4 = r7 * r4 + r4; // s147 mad - r6 = rotr_var(r6, r5); // s148 rotr - r1 = r2 * r4 + r1; // s149 mad - r1 = r1 + r7 + ((((sel >> 17u) & 1u) != 0u) ? 0x0d48ba42u : 0x2bef10f2u); // s150 add - r5 = r5 ^ r4; // s151 xor - r7 = r7 + r0 + ((((sel >> 30u) & 1u) != 0u) ? 0xc98dea9cu : 0xdc5cc080u); // s152 add - r4 = r4 * r6; // s153 mul - r0 = r0 * r2; // s154 mul - r6 = r6 ^ r5; // s155 xor - r4 = rotr_var(r4, r2); // s156 rotr - r1 = rotl_imm(r1, 11u); // s157 rotl - r5 = r5 + r0 + ((((sel >> 11u) & 1u) != 0u) ? 0x6c752dcbu : 0x18197438u); // s158 add - r4 = r1 * r5 + r4; // s159 mad - r4 = rotl_imm(r4, 26u); // s160 rotl - r3 = r0 * r7 + r3; // s161 mad - r3 = rotr_var(r3, r1); // s162 rotr - r4 = r4 + r0 + ((((sel >> 3u) & 1u) != 0u) ? 0x8e481727u : 0xf1c46574u); // s163 add - r0 = __umulhi(r0, r4); // s164 mulhi - r2 = r2 + r6 + ((((sel >> 20u) & 1u) != 0u) ? 0x550ab406u : 0xac578137u); // s165 add - r0 = rotl_imm(r0, 13u); // s166 rotl - r3 = r3 + r1 + ((((sel >> 14u) & 1u) != 0u) ? 0xaebb5966u : 0xa33e6706u); // s167 add - r3 = r3 | r5; // s168 or - r6 = rotr_var(r6, r2); // s169 rotr - r4 = r4 ^ r6; // s170 xor - r6 = r6 - r1; // s171 sub - r7 = rotl_imm(r7, 22u); // s172 rotl - r5 = rotl_imm(r5, 15u); // s173 rotl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r0, 8); // s174 shfl - r0 = r0 ^ r5; // s175 xor - r7 = rotl_imm(r7, 6u); // s176 rotl - r7 = r7 - r0; // s177 sub - r3 = rotl_imm(r3, 30u); // s178 rotl - r7 = r6 * r1 + r7; // s179 mad - r6 = rotl_imm(r6, 9u); // s180 rotl - r2 = r2 ^ r4; // s181 xor - r2 = r2 ^ r7; // s182 xor - r7 = r7 ^ r2; // s183 xor - r1 = r1 + r2 + ((((sel >> 21u) & 1u) != 0u) ? 0x5bb7550fu : 0xd94d55acu); // s184 add - r3 = r3 | r5; // s185 or - r6 = __umulhi(r6, r3); // s186 mulhi - r4 = r4 | r0; // s187 or - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r1, 2); // s188 shfl - r6 = r6 + r5 + ((((sel >> 6u) & 1u) != 0u) ? 0x2a354e2du : 0x89e747feu); // s189 add - r1 = r1 ^ r4; // s190 xor - r7 = __umulhi(r7, r4); // s191 mulhi - r2 = rotl_imm(r2, 26u); // s192 rotl - r5 = rotl_imm(r5, 8u); // s193 rotl - r4 = r4 ^ __shfl_xor_sync(0xffffffffu, r5, 16); // s194 shfl - r4 = r4 ^ r5; // s195 xor - r1 = r1 * r3; // s196 mul - r5 = r5 + r2 + ((((sel >> 7u) & 1u) != 0u) ? 0x10cfdc71u : 0xea03e8e7u); // s197 add - r7 = r7 + r5 + ((((sel >> 15u) & 1u) != 0u) ? 0x66e148d3u : 0xa7ee0102u); // s198 add - r5 = r5 - r2; // s199 sub - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r1, 2); // s200 shfl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r1, 8); // s201 shfl - r4 = rotr_var(r4, r5); // s202 rotr - r7 = r7 ^ r5; // s203 xor - r7 = r7 ^ r0; // s204 xor - r6 = r6 + r2 + ((((sel >> 10u) & 1u) != 0u) ? 0x8558b619u : 0x8379a4deu); // s205 add - r4 = rotl_imm(r4, 13u); // s206 rotl - r1 = __umulhi(r1, r3); // s207 mulhi - r1 = r4 * r4 + r1; // s208 mad - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r0, 4); // s209 shfl - r7 = r7 + r0 + ((((sel >> 11u) & 1u) != 0u) ? 0xf4049c4cu : 0xaa8cb14eu); // s210 add - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r1, 16); // s211 shfl - r1 = r1 ^ r0; // s212 xor - r7 = rotl_imm(r7, 3u); // s213 rotl - r4 = rotr_var(r4, r7); // s214 rotr - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r2, 16); // s215 shfl - r5 = r5 | r1; // s216 or - r1 = r1 - r2; // s217 sub - r6 = r6 - r5; // s218 sub - r6 = rotl_imm(r6, 4u); // s219 rotl - r2 = __umulhi(r2, r0); // s220 mulhi - r2 = r2 | r0; // s221 or - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r2, 8); // s222 shfl - r5 = __umulhi(r5, r6); // s223 mulhi - r0 = r0 - r6; // s224 sub - r7 = rotl_imm(r7, 23u); // s225 rotl - r4 = r4 | r2; // s226 or - r2 = r2 * r4; // s227 mul - r3 = rotl_imm(r3, 12u); // s228 rotl - r0 = rotr_var(r0, r4); // s229 rotr - r0 = r0 + r6 + ((((sel >> 16u) & 1u) != 0u) ? 0x2a7fecb2u : 0x1d176220u); // s230 add - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r2, 4); // s231 shfl - r2 = r2 * r1; // s232 mul - r7 = r0 * r1 + r7; // s233 mad - r5 = rotl_imm(r5, 22u); // s234 rotl - r4 = rotr_var(r4, r6); // s235 rotr - r0 = r5 * r1 + r0; // s236 mad - r6 = r6 ^ r4; // s237 xor - r4 = r4 ^ r6; // s238 xor - r6 = rotl_imm(r6, 18u); // s239 rotl - r4 = r4 + r7 + ((((sel >> 25u) & 1u) != 0u) ? 0xadce39f3u : 0x17dafb4du); // s240 add - r0 = r0 - r7; // s241 sub - r1 = rotr_var(r1, r6); // s242 rotr - r3 = r3 + r0 + ((((sel >> 29u) & 1u) != 0u) ? 0x0e7033b6u : 0xf2f77d26u); // s243 add - r2 = r7 * r4 + r2; // s244 mad - r4 = r0 * r6 + r4; // s245 mad - r5 = r5 * r7; // s246 mul - r3 = r3 + r5 + ((((sel >> 31u) & 1u) != 0u) ? 0x7b2ff6b7u : 0xfed2da4eu); // s247 add - r0 = r0 + r7 + ((((sel >> 0u) & 1u) != 0u) ? 0xb5fad7b1u : 0x03b2891cu); // s248 add - r5 = __umulhi(r5, r4); // s249 mulhi - r5 = r5 + r2 + ((((sel >> 11u) & 1u) != 0u) ? 0xa7e71c2fu : 0x042cd6e3u); // s250 add - r6 = r6 ^ __shfl_xor_sync(0xffffffffu, r1, 1); // s251 shfl - r6 = r6 ^ r1; // s252 xor - r2 = r2 * r5; // s253 mul - r0 = r0 + r6 + ((((sel >> 16u) & 1u) != 0u) ? 0xb83d78deu : 0x784a302bu); // s254 add - r2 = r2 - r4; // s255 sub - } - } - uint32_t lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint32_t hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((uint64_t)hi << 32) | (uint64_t)lo; -} - -cudaError_t igneum_launch_hash_bound(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask, - IgneumInitWords iw, uint32_t nonces, uint32_t blockWarps) { - if (blockWarps == 0u || blockWarps > 32u) return cudaErrorInvalidValue; - uint32_t block = 32u * blockWarps; - if (nonces == 0u || (nonces % block) != 0u) return cudaErrorInvalidValue; - igneum_hash_bound<<>>(ds, out, baseNonce, mask, iw); - return cudaGetLastError(); -} - -cudaError_t igneum_hash_bound_info(int* numRegs, int* blocksPerSM, uint32_t blockWarps) { - cudaFuncAttributes attr; - cudaError_t e = cudaFuncGetAttributes(&attr, igneum_hash_bound); - if (e != cudaSuccess) return e; - *numRegs = attr.numRegs; - return cudaOccupancyMaxActiveBlocksPerMultiprocessor(blocksPerSM, igneum_hash_bound, (int)(32u * blockWarps), 0); -} diff --git a/proto-cuda/packs-ca4/mx8_sh256x27/memhard.h b/proto-cuda/packs-ca4/mx8_sh256x27/memhard.h deleted file mode 100644 index f7f34c732..000000000 --- a/proto-cuda/packs-ca4/mx8_sh256x27/memhard.h +++ /dev/null @@ -1,109 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// Memory-hard dataset core, the same text that the Mac's Metal kernels and CPU verifier were checked against. -// Included by kernel.cu (device), host.cu (host reference) and proto-opencl/host.c (C99 host reference). -// See proto-metal/MEMHARD.md for the construction. kernel.cl carries the same text in OpenCL C. -#pragma once -#ifdef __cplusplus -#include -#else -#include -#endif -#if defined(__CUDACC__) -#define IGNEUM_HD __host__ __device__ __forceinline__ -#elif defined(_MSC_VER) && !defined(__cplusplus) -#define IGNEUM_HD static __inline -#else -#define IGNEUM_HD static inline -#endif -// Memory-hard dataset core (MEMHARD.md). Cache: 2^26 words in 2^16 segments of 64 chained ChaCha12 lines. -// Item: 8 rounds of 8 x seed-parameterised mixer + one 64-byte cache read, then 8 x final mixer (class v3, mixer multiplier 8, -// docs/plans/mixer-x4.md: the round key of application j of round r is 0x9E3779B9 * (r * m + j + 1)). All parameters are literals. -#define MH_CACHE_LINE_MASK 0x003fffffu -#define MH_SEGMENT_LINES 64u -#define MH_QR(a, b, c, d, r1, r2, r3, r4) { a += b; d ^= a; d = mh_rotl(d, r1); c += d; b ^= c; b = mh_rotl(b, r2); a += b; d ^= a; d = mh_rotl(d, r3); c += d; b ^= c; b = mh_rotl(b, r4); } -IGNEUM_HD uint32_t mh_rotl(uint32_t x, uint32_t n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 at every call site - -// y = ChaCha12 core(x) + x -IGNEUM_HD void mh_chacha_block(const uint32_t* x, uint32_t* y) { - for (uint32_t i = 0u; i < 16u; ++i) y[i] = x[i]; - for (uint32_t r = 0u; r < 6u; ++r) { - MH_QR(y[0], y[4], y[8], y[12], 16u, 12u, 8u, 7u) MH_QR(y[1], y[5], y[9], y[13], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[6], y[10], y[14], 16u, 12u, 8u, 7u) MH_QR(y[3], y[7], y[11], y[15], 16u, 12u, 8u, 7u) - MH_QR(y[0], y[5], y[10], y[15], 16u, 12u, 8u, 7u) MH_QR(y[1], y[6], y[11], y[12], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[7], y[8], y[13], 16u, 12u, 8u, 7u) MH_QR(y[3], y[4], y[9], y[14], 16u, 12u, 8u, 7u) - } - for (uint32_t i = 0u; i < 16u; ++i) y[i] += x[i]; -} - -// One cache segment: 64 chained lines written at cache[seg * 1024]. in_j = prev ^ (sigma || K || seg || j || tag), prev_0 = 0. -IGNEUM_HD void mh_cache_segment(uint32_t* cache, uint32_t seg) { - uint32_t prev[16]; uint32_t x[16]; uint32_t y[16]; - for (uint32_t i = 0u; i < 16u; ++i) prev[i] = 0u; - for (uint32_t j = 0u; j < MH_SEGMENT_LINES; ++j) { - x[0] = 0x61707865u ^ prev[0]; x[1] = 0x3320646eu ^ prev[1]; x[2] = 0x79622d32u ^ prev[2]; x[3] = 0x6b206574u ^ prev[3]; - x[4] = 0x3067619fu ^ prev[4]; - x[5] = 0x3c269176u ^ prev[5]; - x[6] = 0x84a03b03u ^ prev[6]; - x[7] = 0xf8c63294u ^ prev[7]; - x[8] = 0xff977c5bu ^ prev[8]; - x[9] = 0xe60def3eu ^ prev[9]; - x[10] = 0x63630141u ^ prev[10]; - x[11] = 0xb8fbcb58u ^ prev[11]; - x[12] = seg ^ prev[12]; x[13] = j ^ prev[13]; x[14] = 0x49676e65u ^ prev[14]; x[15] = 0x756d4d48u ^ prev[15]; - mh_chacha_block(x, y); - uint32_t* line = cache + ((seg * MH_SEGMENT_LINES + j) * 16u); - for (uint32_t i = 0u; i < 16u; ++i) { line[i] = y[i]; prev[i] = y[i]; } - } -} - -// M_r: per word (s ^ (RC + rk)) * MUL, then a column round and a diagonal round with the seed-drawn rotations. -IGNEUM_HD void mh_mixer(uint32_t* s, uint32_t rk) { - s[0] = (s[0] ^ (0xbab68293u + rk)) * 0x42146205u; - s[1] = (s[1] ^ (0xcc162340u + rk)) * 0x52cbe0fbu; - s[2] = (s[2] ^ (0x6ce151ccu + rk)) * 0x7ecf4a03u; - s[3] = (s[3] ^ (0xe62b8997u + rk)) * 0x6728907fu; - s[4] = (s[4] ^ (0xc9c80297u + rk)) * 0xd81d9751u; - s[5] = (s[5] ^ (0xf74a1654u + rk)) * 0x132952c3u; - s[6] = (s[6] ^ (0x3d704af5u + rk)) * 0xf60de277u; - s[7] = (s[7] ^ (0x3cf522b7u + rk)) * 0x05358035u; - s[8] = (s[8] ^ (0x2b9cac04u + rk)) * 0xbaf6499du; - s[9] = (s[9] ^ (0xa880ac10u + rk)) * 0xe4db9667u; - s[10] = (s[10] ^ (0x13e5dd1du + rk)) * 0x3e98f45du; - s[11] = (s[11] ^ (0x6fc3e233u + rk)) * 0xd0004eddu; - s[12] = (s[12] ^ (0x2d83eeacu + rk)) * 0x2691630du; - s[13] = (s[13] ^ (0x9006e8bfu + rk)) * 0x9beb3bcfu; - s[14] = (s[14] ^ (0x2c4b5362u + rk)) * 0xab310379u; - s[15] = (s[15] ^ (0x31b49ee2u + rk)) * 0x99cfb423u; - MH_QR(s[0], s[4], s[8], s[12], 20u, 20u, 19u, 4u) MH_QR(s[1], s[5], s[9], s[13], 20u, 20u, 19u, 4u) - MH_QR(s[2], s[6], s[10], s[14], 20u, 20u, 19u, 4u) MH_QR(s[3], s[7], s[11], s[15], 20u, 20u, 19u, 4u) - MH_QR(s[0], s[5], s[10], s[15], 26u, 3u, 3u, 27u) MH_QR(s[1], s[6], s[11], s[12], 26u, 3u, 3u, 27u) - MH_QR(s[2], s[7], s[8], s[13], 26u, 3u, 3u, 27u) MH_QR(s[3], s[4], s[9], s[14], 26u, 3u, 3u, 27u) -} - -// Item t: 16 words. s = (K, t * MUL[i] + RC[i]); 8 rounds of 8 x mixer + cache line s[0] & mask; 8 x final mixer. -IGNEUM_HD void mh_item(const uint32_t* cache, uint32_t t, uint32_t* s) { - s[0] = 0x3067619fu; - s[1] = 0x3c269176u; - s[2] = 0x84a03b03u; - s[3] = 0xf8c63294u; - s[4] = 0xff977c5bu; - s[5] = 0xe60def3eu; - s[6] = 0x63630141u; - s[7] = 0xb8fbcb58u; - s[8] = t * 0x42146205u + 0xbab68293u; - s[9] = t * 0x52cbe0fbu + 0xcc162340u; - s[10] = t * 0x7ecf4a03u + 0x6ce151ccu; - s[11] = t * 0x6728907fu + 0xe62b8997u; - s[12] = t * 0xd81d9751u + 0xc9c80297u; - s[13] = t * 0x132952c3u + 0xf74a1654u; - s[14] = t * 0xf60de277u + 0x3d704af5u; - s[15] = t * 0x05358035u + 0x3cf522b7u; - for (uint32_t r = 0u; r < 8u; ++r) { - for (uint32_t j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (r * 8u + j + 1u)); - const uint32_t* line = cache + ((s[0] & MH_CACHE_LINE_MASK) * 16u); - for (uint32_t i = 0u; i < 16u; ++i) s[i] ^= line[i]; - } - for (uint32_t j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (64u + j + 1u)); -} -// dataset[w] without the dataset: derive item w >> 4 and take word w & 15. -IGNEUM_HD uint32_t mh_word(const uint32_t* cache, uint32_t w) { uint32_t s[16]; mh_item(cache, w >> 4u, s); return s[w & 15u]; } diff --git a/proto-cuda/packs-ca4/mx8_sh256x27/memhard.metal b/proto-cuda/packs-ca4/mx8_sh256x27/memhard.metal deleted file mode 100644 index 01b263d6d..000000000 --- a/proto-cuda/packs-ca4/mx8_sh256x27/memhard.metal +++ /dev/null @@ -1,107 +0,0 @@ -#include -using namespace metal; -// Memory-hard dataset core (MEMHARD.md). Cache: 2^26 words in 2^16 segments of 64 chained ChaCha12 lines. -// Item: 8 rounds of 8 x seed-parameterised mixer + one 64-byte cache read, then 8 x final mixer (class v3, mixer multiplier 8, -// docs/plans/mixer-x4.md: the round key of application j of round r is 0x9E3779B9 * (r * m + j + 1)). All parameters are literals. -#define MH_CACHE_LINE_MASK 0x003fffffu -#define MH_SEGMENT_LINES 64u -#define MH_QR(a, b, c, d, r1, r2, r3, r4) { a += b; d ^= a; d = mh_rotl(d, r1); c += d; b ^= c; b = mh_rotl(b, r2); a += b; d ^= a; d = mh_rotl(d, r3); c += d; b ^= c; b = mh_rotl(b, r4); } -inline uint mh_rotl(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 at every call site - -// y = ChaCha12 core(x) + x -inline void mh_chacha_block(const thread uint* x, thread uint* y) { - for (uint i = 0u; i < 16u; ++i) y[i] = x[i]; - for (uint r = 0u; r < 6u; ++r) { - MH_QR(y[0], y[4], y[8], y[12], 16u, 12u, 8u, 7u) MH_QR(y[1], y[5], y[9], y[13], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[6], y[10], y[14], 16u, 12u, 8u, 7u) MH_QR(y[3], y[7], y[11], y[15], 16u, 12u, 8u, 7u) - MH_QR(y[0], y[5], y[10], y[15], 16u, 12u, 8u, 7u) MH_QR(y[1], y[6], y[11], y[12], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[7], y[8], y[13], 16u, 12u, 8u, 7u) MH_QR(y[3], y[4], y[9], y[14], 16u, 12u, 8u, 7u) - } - for (uint i = 0u; i < 16u; ++i) y[i] += x[i]; -} - -// One cache segment: 64 chained lines written at cache[seg * 1024]. in_j = prev ^ (sigma || K || seg || j || tag), prev_0 = 0. -inline void mh_cache_segment(device uint* cache, uint seg) { - uint prev[16]; uint x[16]; uint y[16]; - for (uint i = 0u; i < 16u; ++i) prev[i] = 0u; - for (uint j = 0u; j < MH_SEGMENT_LINES; ++j) { - x[0] = 0x61707865u ^ prev[0]; x[1] = 0x3320646eu ^ prev[1]; x[2] = 0x79622d32u ^ prev[2]; x[3] = 0x6b206574u ^ prev[3]; - x[4] = 0x3067619fu ^ prev[4]; - x[5] = 0x3c269176u ^ prev[5]; - x[6] = 0x84a03b03u ^ prev[6]; - x[7] = 0xf8c63294u ^ prev[7]; - x[8] = 0xff977c5bu ^ prev[8]; - x[9] = 0xe60def3eu ^ prev[9]; - x[10] = 0x63630141u ^ prev[10]; - x[11] = 0xb8fbcb58u ^ prev[11]; - x[12] = seg ^ prev[12]; x[13] = j ^ prev[13]; x[14] = 0x49676e65u ^ prev[14]; x[15] = 0x756d4d48u ^ prev[15]; - mh_chacha_block(x, y); - device uint* line = cache + ((seg * MH_SEGMENT_LINES + j) * 16u); - for (uint i = 0u; i < 16u; ++i) { line[i] = y[i]; prev[i] = y[i]; } - } -} - -// M_r: per word (s ^ (RC + rk)) * MUL, then a column round and a diagonal round with the seed-drawn rotations. -inline void mh_mixer(thread uint* s, uint rk) { - s[0] = (s[0] ^ (0xbab68293u + rk)) * 0x42146205u; - s[1] = (s[1] ^ (0xcc162340u + rk)) * 0x52cbe0fbu; - s[2] = (s[2] ^ (0x6ce151ccu + rk)) * 0x7ecf4a03u; - s[3] = (s[3] ^ (0xe62b8997u + rk)) * 0x6728907fu; - s[4] = (s[4] ^ (0xc9c80297u + rk)) * 0xd81d9751u; - s[5] = (s[5] ^ (0xf74a1654u + rk)) * 0x132952c3u; - s[6] = (s[6] ^ (0x3d704af5u + rk)) * 0xf60de277u; - s[7] = (s[7] ^ (0x3cf522b7u + rk)) * 0x05358035u; - s[8] = (s[8] ^ (0x2b9cac04u + rk)) * 0xbaf6499du; - s[9] = (s[9] ^ (0xa880ac10u + rk)) * 0xe4db9667u; - s[10] = (s[10] ^ (0x13e5dd1du + rk)) * 0x3e98f45du; - s[11] = (s[11] ^ (0x6fc3e233u + rk)) * 0xd0004eddu; - s[12] = (s[12] ^ (0x2d83eeacu + rk)) * 0x2691630du; - s[13] = (s[13] ^ (0x9006e8bfu + rk)) * 0x9beb3bcfu; - s[14] = (s[14] ^ (0x2c4b5362u + rk)) * 0xab310379u; - s[15] = (s[15] ^ (0x31b49ee2u + rk)) * 0x99cfb423u; - MH_QR(s[0], s[4], s[8], s[12], 20u, 20u, 19u, 4u) MH_QR(s[1], s[5], s[9], s[13], 20u, 20u, 19u, 4u) - MH_QR(s[2], s[6], s[10], s[14], 20u, 20u, 19u, 4u) MH_QR(s[3], s[7], s[11], s[15], 20u, 20u, 19u, 4u) - MH_QR(s[0], s[5], s[10], s[15], 26u, 3u, 3u, 27u) MH_QR(s[1], s[6], s[11], s[12], 26u, 3u, 3u, 27u) - MH_QR(s[2], s[7], s[8], s[13], 26u, 3u, 3u, 27u) MH_QR(s[3], s[4], s[9], s[14], 26u, 3u, 3u, 27u) -} - -// Item t: 16 words. s = (K, t * MUL[i] + RC[i]); 8 rounds of 8 x mixer + cache line s[0] & mask; 8 x final mixer. -inline void mh_item(device const uint* cache, uint t, thread uint* s) { - s[0] = 0x3067619fu; - s[1] = 0x3c269176u; - s[2] = 0x84a03b03u; - s[3] = 0xf8c63294u; - s[4] = 0xff977c5bu; - s[5] = 0xe60def3eu; - s[6] = 0x63630141u; - s[7] = 0xb8fbcb58u; - s[8] = t * 0x42146205u + 0xbab68293u; - s[9] = t * 0x52cbe0fbu + 0xcc162340u; - s[10] = t * 0x7ecf4a03u + 0x6ce151ccu; - s[11] = t * 0x6728907fu + 0xe62b8997u; - s[12] = t * 0xd81d9751u + 0xc9c80297u; - s[13] = t * 0x132952c3u + 0xf74a1654u; - s[14] = t * 0xf60de277u + 0x3d704af5u; - s[15] = t * 0x05358035u + 0x3cf522b7u; - for (uint r = 0u; r < 8u; ++r) { - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (r * 8u + j + 1u)); - device const uint* line = cache + ((s[0] & MH_CACHE_LINE_MASK) * 16u); - for (uint i = 0u; i < 16u; ++i) s[i] ^= line[i]; - } - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (64u + j + 1u)); -} -// dataset[w] without the dataset: derive item w >> 4 and take word w & 15. -inline uint mh_word(device const uint* cache, uint w) { uint s[16]; mh_item(cache, w >> 4u, s); return s[w & 15u]; } - -// One thread per segment (2^16 threads). -kernel void igneum_cache_fill(device uint* cache [[buffer(0)]], uint gid [[thread_position_in_grid]]) { - mh_cache_segment(cache, gid); -} -// One thread per 64-byte item (dataset words / 16 threads). -kernel void igneum_build(device const uint* cache [[buffer(0)]], device uint* dataset [[buffer(1)]], - uint gid [[thread_position_in_grid]]) { - uint s[16]; - mh_item(cache, gid, s); - device uint* d = dataset + gid * 16u; - for (uint i = 0u; i < 16u; ++i) d[i] = s[i]; -} diff --git a/proto-cuda/packs-ca4/mx8_sh256x27/program.h b/proto-cuda/packs-ca4/mx8_sh256x27/program.h deleted file mode 100644 index e1c873c47..000000000 --- a/proto-cuda/packs-ca4/mx8_sh256x27/program.h +++ /dev/null @@ -1,70 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// Program metadata for host.cu plus the launch wrappers defined in kernel.cu. -// Also included by proto-opencl/host.c (C99), which defines IGNEUM_NO_CUDA first and reads only the macros. -#pragma once -#ifdef __cplusplus -#include -#else -#include -#endif -#ifndef IGNEUM_NO_CUDA -#include -#endif - -#define IGNEUM_SEED_STRING "igneum-genesis" -#define IGNEUM_SEED_BYTES_HEX "69676e65756d2d67656e65736973" -#define IGNEUM_GENERATOR 2 -#define IGNEUM_PROGRAM_ATTEMPT 0 -#define IGNEUM_PROGRAM_ID 0x9a8b77853b298baaull -#define IGNEUM_DAY_STRING "2026-10-03" -#define IGNEUM_DAY_BYTES_HEX "6461792f323032362d31302d3033" -#define IGNEUM_DAY0 0x3067619fu -#define IGNEUM_DAY1 0x3c269176u -#define IGNEUM_DATASET_LOG2 28 -#define IGNEUM_MASK 0x0fffffffu -#define IGNEUM_LANES 32 -#define IGNEUM_ITERATIONS 8 -#define IGNEUM_INSTR_COUNT 64 -#define IGNEUM_LOADS_PER_HASH 128 -#define IGNEUM_WIDE_LOADS_PER_HASH 0 -#define IGNEUM_OP_MIX "load=16 add=8 shfl=8 xor=6 mad=5 mul=5 mulhi=5 sub=4 rotl=3 rotr=3 or=1" -// Class v3 construction (Counter ASIC 2.0, 5 October 2026, docs/plans/mixer-x4.md): version 2 loads; the dataset item -// derivation applies the mixer IGNEUM_MIXER_MULT times per round (memhard.h), and the cache follows the growth rule. -#define IGNEUM_LOAD_CLASS "mx8+sh256x27" -#define IGNEUM_CLASS_MIXER_MULT 8 -#define IGNEUM_CACHE_GROWTH 1 // 1: cache words = 2^(26 + doublings(day)), doublings = floor(log2(1 + day / 1460)) -#define IGNEUM_LOAD_SLOTS 16 -#define IGNEUM_LOAD_MIX { 100, 0, 0 } -#define IGNEUM_LOAD_WIDTH_COUNTS { 16, 0, 0 } // loads of 4, 16, 64 bytes per program -#define IGNEUM_BYTES_PER_HASH 512 -#define IGNEUM_FOLD_ROT 11 -#define IGNEUM_FOLD_MUL 0x9e3779b1u -// Latency-shadow block (Counter ASIC 3.0 item 8, docs/analysis/latency-shadow-2026-10-06.md): NOT the lottery hash. A block of -// IGNEUM_SHADOW_INSTRS ALU instructions (the ten non-load families) runs IGNEUM_SHADOW_REPS times at the end of every -// iteration; the 16 loads, the acceptance rule and the base program are the class's without the shadow. -#define IGNEUM_SHADOW_INSTRS 256 -#define IGNEUM_SHADOW_REPS 27 -#define IGNEUM_SHADOW_INSTRS_PER_HASH 55296 -#define IGNEUM_SHADOW_OP_MIX "add=47 rotl=30 xor=30 shfl=29 mad=27 mul=22 sub=21 rotr=20 mulhi=18 or=12" -// 0 = closed-form dataset (ds_elem), 1 = memory-hard cache construction (MEMHARD.md, memhard.h) -#define IGNEUM_DATASET_MODE 1 - -#define IGNEUM_SEEDW_INIT { 0x67a9a7beu, 0x1a155b25u, 0xfddfb732u, 0x4b5af2e8u, 0xc55caf33u, 0xa27c13b7u, 0x06628a48u, 0x03852469u } -#define IGNEUM_KEY_INIT { 0x3067619fu, 0x3c269176u, 0x84a03b03u, 0xf8c63294u, 0xff977c5bu, 0xe60def3eu, 0x63630141u, 0xb8fbcb58u } -#define IGNEUM_CACHE_LOG2_WORDS 26 -#define IGNEUM_CACHE_SEGMENT_LOG2_LINES 6 -#define IGNEUM_CACHE_SEGMENTS 65536u -#define IGNEUM_ITEM_ROUNDS 8 -#define IGNEUM_MIXER_MULT 8 // mixer applications per round and after the last read (class v3, docs/plans/mixer-x4.md) -#define IGNEUM_MIX_ROT_INIT { 20u, 20u, 19u, 4u, 26u, 3u, 3u, 27u } -#define IGNEUM_MIX_MUL_INIT { 0x42146205u, 0x52cbe0fbu, 0x7ecf4a03u, 0x6728907fu, 0xd81d9751u, 0x132952c3u, 0xf60de277u, 0x05358035u, 0xbaf6499du, 0xe4db9667u, 0x3e98f45du, 0xd0004eddu, 0x2691630du, 0x9beb3bcfu, 0xab310379u, 0x99cfb423u } -#define IGNEUM_MIX_RC_INIT { 0xbab68293u, 0xcc162340u, 0x6ce151ccu, 0xe62b8997u, 0xc9c80297u, 0xf74a1654u, 0x3d704af5u, 0x3cf522b7u, 0x2b9cac04u, 0xa880ac10u, 0x13e5dd1du, 0x6fc3e233u, 0x2d83eeacu, 0x9006e8bfu, 0x2c4b5362u, 0x31b49ee2u } - -#ifndef IGNEUM_NO_CUDA -// Defined in kernel.cu. All launch on the default stream and return cudaGetLastError(). -cudaError_t igneum_launch_cache_fill(uint32_t* cache, uint32_t nSegments); -cudaError_t igneum_launch_build(uint32_t* ds, const uint32_t* cache, uint32_t nItems); -cudaError_t igneum_launch_hash(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask, - uint32_t nonces, uint32_t blockWarps); -cudaError_t igneum_hash_info(int* numRegs, int* blocksPerSM, uint32_t blockWarps); -#endif diff --git a/proto-cuda/packs-ca4/mx8_sh256x27/program.json b/proto-cuda/packs-ca4/mx8_sh256x27/program.json deleted file mode 100644 index 8588e0fe9..000000000 --- a/proto-cuda/packs-ca4/mx8_sh256x27/program.json +++ /dev/null @@ -1,389 +0,0 @@ -{ - "format": "igneum-program-pack-3", - "generator": 2, - "attempt": 0, - "program_id": "0x9a8b77853b298baa", - "program_id_derivation": "FNV-1a 64 over 'igneum-program/' || generator_le32 || seed_words as little-endian bytes || attempt_le32", - "dataset_mode": "memory-hard", - "seed": "igneum-genesis", - "seed_bytes": "69676e65756d2d67656e65736973", - "seed_words": ["0x67a9a7be", "0x1a155b25", "0xfddfb732", "0x4b5af2e8", "0xc55caf33", "0xa27c13b7", "0x06628a48", "0x03852469"], - "seed_derivation": "seed_words = FNV-1a 64 over seed_bytes (attempt 0) or seed_bytes || attempt_le32 (attempt k >= 1), basis ^ (salt * 0x9E3779B97F4A7C15) for salt 0..3, then h ^= h>>33; h *= 0xff51afd7ed558ccd; h ^= h>>33; words[2*salt] = low 32, words[2*salt+1] = high 32", - "generator_rule": "version 2: exactly 16 load slots drawn first from instructions 1..63 (partial Fisher-Yates), the other 48 ops from the ten non-load weights (sum 75); a load's source is drawn from the registers other than dst written by an earlier instruction and not read by a load since; the candidate must pass the acceptance rule of spec 01 section 1.4.6 (static: no cyclically stale load source, every register has an injecting write; dynamic: 64 units on the seed-keyed closed-form dataset with no constant register bit, no lane-constant load site, under 164 saturated final values, every output bit within 136 of 1024, distinct addresses above 245760), else the next attempt of the seed is tried", - "lanes": 32, - "registers": 8, - "iterations": 8, - "instruction_count": 64, - "loads_per_hash": 128, - "load_class": "mx8+sh256x27", - "mixer_mult": 8, - "cache_growth": true, - "mixer": "class v3 (Counter ASIC 2.0, 5 October 2026, docs/plans/mixer-x4.md): every mixer application of the item derivation is 8 applications with round keys (r * 8 + j + 1) * 0x9E3779B9, the 8 dependent cache reads per item unchanged; cache growth rule option C: cache words = 2^(26 + doublings(day)), dataset words = 2^(genesis_log2 + doublings(day)), doublings(day) = floor(log2(1 + day / 1460)) for day = days since genesis", - "load_slots": 16, - "load_mix_percent_4_16_64": [100, 0, 0], - "load_width_counts_4_16_64": [16, 0, 0], - "bytes_per_hash": 512, - "wide_load": "read-width experiment (5 October 2026, docs/plans/read-width.md), NOT the lottery hash: a load of W words (width field, 4 or 16) reads dataset[b .. b + W) with b = (src & mask) & ~(W - 1) and folds every word into dst: x = dst ^ w[0]; for j in 1..W: x = (rotl(x, 11) * 0x9e3779b1) ^ w[j]; dst = x; width 1 is the plain load; the width is drawn per instruction from the class mix with one extra below(100) draw after the nine of version 2, and the program id is FNV-1a 64 over 'igneum-program-rw/' || generator_le32 || seed words || attempt_le32 || mix[3] || load_slots", - "op_mix": {"load": 16, "add": 8, "shfl": 8, "xor": 6, "mad": 5, "mul": 5, "mulhi": 5, "sub": 4, "rotl": 3, "rotr": 3, "or": 1}, - "register_init": "for i in 0..7: x = nonce ^ seed_words[i]; x += 0x9e3779b9 * (i+1) (mod 2^32); x = splitmix32(x); r[i] = x ^ seed_words[(i+1) & 7]", - "splitmix32": "x ^= x>>16; x *= 0x7feb352d; x ^= x>>15; x *= 0x846ca68b; x ^= x>>16", - "iteration": "sel = r0 sampled once at the top of each iteration, then all instructions in order", - "output": "lo = r0 ^ rotl(r1,7) ^ rotl(r2,14) ^ rotl(r3,21); hi = r4 ^ rotl(r5,9) ^ rotl(r6,18) ^ rotl(r7,27); out = (hi << 32) | lo", - "op_semantics": { - "add": "dst = dst + src + (bit `bit` of sel ? imm2 : imm)", - "sub": "dst = dst - src", - "mul": "dst = dst * src (low 32)", - "mulhi": "dst = high 32 bits of dst * src", - "xor": "dst = dst ^ src", - "or": "dst = dst | src", - "rotl": "dst = rotl(dst, rot), rot in 1..31", - "rotr": "dst = rotr(dst, src & 31)", - "mad": "dst = src * src2 + dst", - "shfl": "dst = dst ^ (src of lane (lane ^ mask)), mask in {1,2,4,8,16}, within the 32-lane warp", - "load": "dst = dst ^ dataset[src & dataset.mask]", - "wload": "base = (src of lane 0 & dataset.mask) & ~31; dst = dst ^ dataset[base + lane] (warp-coalesced 128-byte load, lever b, only when --wide-frac > 0)" - }, - "dataset": { - "log2_words": 28, - "bytes": 1073741824, - "mask": "0x0fffffff", - "day": "2026-10-03", - "day_bytes": "6461792f323032362d31302d3033", - "day_words_from": "seed_words_from_bytes(day_bytes)", - "d0": "0x3067619f", - "d1": "0x3c269176", - "mode": "memory-hard", - "spec": "proto-metal/MEMHARD.md", - "key": ["0x3067619f", "0x3c269176", "0x84a03b03", "0xf8c63294", "0xff977c5b", "0xe60def3e", "0x63630141", "0xb8fbcb58"], - "key_derivation": "the 8 words of seed_words_from_bytes(day_bytes); d0, d1 are key[0], key[1]", - "cache": {"log2_words": 26, "bytes": 268435456, "line_words": 16, "segment_lines": 64, "segments": 65536, "block": "ChaCha12 core + feed-forward, rotations 16 12 8 7", "sigma": ["0x61707865", "0x3320646e", "0x79622d32", "0x6b206574"], "tag": ["0x49676e65", "0x756d4d48"], "chain": "in_j = prev_line ^ (sigma[0..3] || key[0..7] || seg || j || tag[0..1]); line_j = block(in_j); prev_0 = 0"}, - "mixer": {"draw": "SplitMix64 seeded with key[0] | key[1] << 32: rot[0..7] = 1 + next() % 31, mul[0..15] = low32(next()) | 1, rc[0..15] = low32(next())", "rot": [20, 20, 19, 4, 26, 3, 3, 27], "mul": ["0x42146205", "0x52cbe0fb", "0x7ecf4a03", "0x6728907f", "0xd81d9751", "0x132952c3", "0xf60de277", "0x05358035", "0xbaf6499d", "0xe4db9667", "0x3e98f45d", "0xd0004edd", "0x2691630d", "0x9beb3bcf", "0xab310379", "0x99cfb423"], "rc": ["0xbab68293", "0xcc162340", "0x6ce151cc", "0xe62b8997", "0xc9c80297", "0xf74a1654", "0x3d704af5", "0x3cf522b7", "0x2b9cac04", "0xa880ac10", "0x13e5dd1d", "0x6fc3e233", "0x2d83eeac", "0x9006e8bf", "0x2c4b5362", "0x31b49ee2"], "round": "for i in 0..15: s[i] = (s[i] ^ (rc[i] + (r+1) * 0x9E3779B9)) * mul[i]; then quarter rounds on columns (0,4,8,12) (1,5,9,13) (2,6,10,14) (3,7,11,15) with rot[0..3] and diagonals (0,5,10,15) (1,6,11,12) (2,7,8,13) (3,4,9,14) with rot[4..7]", "quarter_round": "a += b; d ^= a; d = rotl(d, r1); c += d; b ^= c; b = rotl(b, r2); a += b; d ^= a; d = rotl(d, r3); c += d; b ^= c; b = rotl(b, r4)"}, - "mixer_mult": 8, - "item": "s[0..7] = key; s[8+i] = t * mul[i] + rc[i] for i in 0..7; for r in 0..7: for j in 0..7: s = M(s, rk = (r * 8 + j + 1) * 0x9E3779B9); line = s[0] & 0x003fffff; s[i] ^= cache[line * 16 + i]; then for j in 0..7: s = M(s, rk = (64 + j + 1) * 0x9E3779B9); item(t) = s", - "word": "dataset[w] = item(w >> 4)[w & 15]" - }, - "shadow": {"instrs": 256, "reps": 27, "instrs_per_hash": 55296, "op_mix": {"add": 47, "rotl": 30, "xor": 30, "shfl": 29, "mad": 27, "mul": 22, "sub": 21, "rotr": 20, "mulhi": 18, "or": 12}, "rule": "Counter ASIC 3.0 item 8 (docs/analysis/latency-shadow-2026-10-06.md): after the 64 base instructions the program stream draws instrs more ALU instructions (the non-load table, nine draws each, the source as on an ALU slot); the block runs reps times at the end of every iteration with the iteration's sel; the acceptance rule interprets the base program only", "program_id_suffix": "'shadow/' || instrs_le16 || reps_le16", "instructions": [ - {"i": 0, "op": "add", "dst": 5, "src": 2, "src2": 1, "imm": "0x92199f99", "imm2": "0x8bc12da9", "rot": 9, "bit": 18, "mask": 4}, - {"i": 1, "op": "add", "dst": 0, "src": 7, "src2": 1, "imm": "0x8d72d3ad", "imm2": "0x63079e5a", "rot": 20, "bit": 26, "mask": 4}, - {"i": 2, "op": "shfl", "dst": 6, "src": 3, "src2": 6, "imm": "0x9beaeddf", "imm2": "0x744ecb00", "rot": 10, "bit": 4, "mask": 2}, - {"i": 3, "op": "sub", "dst": 4, "src": 2, "src2": 0, "imm": "0x2a3ddc67", "imm2": "0x72c80241", "rot": 30, "bit": 1, "mask": 16}, - {"i": 4, "op": "add", "dst": 7, "src": 0, "src2": 5, "imm": "0xb21b4bab", "imm2": "0x5d4c7a60", "rot": 17, "bit": 31, "mask": 4}, - {"i": 5, "op": "rotl", "dst": 0, "src": 6, "src2": 3, "imm": "0xe69d7919", "imm2": "0xa048c61e", "rot": 11, "bit": 1, "mask": 8}, - {"i": 6, "op": "add", "dst": 0, "src": 1, "src2": 3, "imm": "0x2fe0e98b", "imm2": "0xc88e2942", "rot": 5, "bit": 16, "mask": 16}, - {"i": 7, "op": "xor", "dst": 7, "src": 1, "src2": 0, "imm": "0xc16efe98", "imm2": "0x01a638c8", "rot": 8, "bit": 17, "mask": 1}, - {"i": 8, "op": "mad", "dst": 1, "src": 6, "src2": 5, "imm": "0x76ec7b8b", "imm2": "0x25663feb", "rot": 29, "bit": 30, "mask": 2}, - {"i": 9, "op": "shfl", "dst": 6, "src": 1, "src2": 0, "imm": "0x6c8ee3cb", "imm2": "0xea93237e", "rot": 27, "bit": 1, "mask": 4}, - {"i": 10, "op": "mad", "dst": 1, "src": 2, "src2": 2, "imm": "0x6dc4ea18", "imm2": "0x6efde6f5", "rot": 3, "bit": 20, "mask": 2}, - {"i": 11, "op": "mad", "dst": 5, "src": 0, "src2": 3, "imm": "0x023613fc", "imm2": "0x18c51939", "rot": 19, "bit": 31, "mask": 1}, - {"i": 12, "op": "shfl", "dst": 2, "src": 6, "src2": 1, "imm": "0x74aec8d2", "imm2": "0x7f7ad29c", "rot": 7, "bit": 8, "mask": 4}, - {"i": 13, "op": "add", "dst": 1, "src": 5, "src2": 6, "imm": "0xbdf8f9a5", "imm2": "0xbc48c63e", "rot": 6, "bit": 25, "mask": 2}, - {"i": 14, "op": "rotl", "dst": 1, "src": 2, "src2": 6, "imm": "0x7ad8ca8b", "imm2": "0x14c712ad", "rot": 29, "bit": 25, "mask": 8}, - {"i": 15, "op": "sub", "dst": 1, "src": 4, "src2": 5, "imm": "0xd3349f69", "imm2": "0x70aac45a", "rot": 29, "bit": 9, "mask": 16}, - {"i": 16, "op": "or", "dst": 7, "src": 1, "src2": 2, "imm": "0x08168ed1", "imm2": "0x28964037", "rot": 26, "bit": 0, "mask": 2}, - {"i": 17, "op": "shfl", "dst": 2, "src": 4, "src2": 6, "imm": "0x98d85f72", "imm2": "0x3dc87042", "rot": 29, "bit": 22, "mask": 8}, - {"i": 18, "op": "xor", "dst": 7, "src": 4, "src2": 0, "imm": "0x651d4a0e", "imm2": "0x93c198bd", "rot": 26, "bit": 12, "mask": 8}, - {"i": 19, "op": "mul", "dst": 6, "src": 1, "src2": 6, "imm": "0xd38ce89d", "imm2": "0x3dfad388", "rot": 5, "bit": 28, "mask": 8}, - {"i": 20, "op": "mad", "dst": 5, "src": 6, "src2": 0, "imm": "0x59d78b36", "imm2": "0xc4db274e", "rot": 25, "bit": 24, "mask": 1}, - {"i": 21, "op": "sub", "dst": 3, "src": 1, "src2": 7, "imm": "0xf6c6a6c7", "imm2": "0x8536f4e6", "rot": 6, "bit": 12, "mask": 4}, - {"i": 22, "op": "mul", "dst": 6, "src": 0, "src2": 7, "imm": "0x599445b4", "imm2": "0x632f8c32", "rot": 19, "bit": 4, "mask": 8}, - {"i": 23, "op": "add", "dst": 2, "src": 0, "src2": 7, "imm": "0x45c37cec", "imm2": "0x96e8f127", "rot": 15, "bit": 1, "mask": 4}, - {"i": 24, "op": "sub", "dst": 6, "src": 4, "src2": 3, "imm": "0xc1c44491", "imm2": "0x92e3ce57", "rot": 20, "bit": 25, "mask": 4}, - {"i": 25, "op": "mad", "dst": 7, "src": 3, "src2": 4, "imm": "0x89bfb8d3", "imm2": "0x19b5455e", "rot": 22, "bit": 2, "mask": 16}, - {"i": 26, "op": "rotl", "dst": 3, "src": 5, "src2": 7, "imm": "0x2f47ce8d", "imm2": "0x8b458ec5", "rot": 9, "bit": 0, "mask": 4}, - {"i": 27, "op": "sub", "dst": 2, "src": 1, "src2": 2, "imm": "0x9f0dce23", "imm2": "0x3cdca814", "rot": 25, "bit": 1, "mask": 2}, - {"i": 28, "op": "mulhi", "dst": 6, "src": 0, "src2": 3, "imm": "0x61fc9eb8", "imm2": "0x23202e9d", "rot": 30, "bit": 16, "mask": 16}, - {"i": 29, "op": "shfl", "dst": 2, "src": 4, "src2": 3, "imm": "0x339dbd65", "imm2": "0xc175f639", "rot": 15, "bit": 22, "mask": 2}, - {"i": 30, "op": "shfl", "dst": 1, "src": 6, "src2": 1, "imm": "0x5bd14589", "imm2": "0xa68a2bed", "rot": 31, "bit": 31, "mask": 1}, - {"i": 31, "op": "add", "dst": 1, "src": 6, "src2": 1, "imm": "0x37985632", "imm2": "0xb1cdb2ab", "rot": 29, "bit": 1, "mask": 8}, - {"i": 32, "op": "rotr", "dst": 0, "src": 1, "src2": 6, "imm": "0x4b2058f4", "imm2": "0xf06d8ac9", "rot": 9, "bit": 15, "mask": 16}, - {"i": 33, "op": "shfl", "dst": 3, "src": 4, "src2": 4, "imm": "0x2081626c", "imm2": "0x08d1bb87", "rot": 24, "bit": 29, "mask": 2}, - {"i": 34, "op": "shfl", "dst": 0, "src": 3, "src2": 6, "imm": "0xe4fcfe03", "imm2": "0x78a46b13", "rot": 15, "bit": 29, "mask": 4}, - {"i": 35, "op": "mul", "dst": 7, "src": 0, "src2": 4, "imm": "0x33148d30", "imm2": "0x5780a3d6", "rot": 6, "bit": 11, "mask": 2}, - {"i": 36, "op": "add", "dst": 3, "src": 1, "src2": 1, "imm": "0x804e777e", "imm2": "0x856e0180", "rot": 22, "bit": 0, "mask": 16}, - {"i": 37, "op": "xor", "dst": 1, "src": 6, "src2": 0, "imm": "0xc69aa2f6", "imm2": "0xafebe3e8", "rot": 15, "bit": 9, "mask": 16}, - {"i": 38, "op": "mul", "dst": 2, "src": 7, "src2": 4, "imm": "0xeb4c4082", "imm2": "0x3f1e0da7", "rot": 25, "bit": 20, "mask": 16}, - {"i": 39, "op": "add", "dst": 6, "src": 5, "src2": 5, "imm": "0x0b06c8a7", "imm2": "0xe9bd0cc4", "rot": 6, "bit": 2, "mask": 4}, - {"i": 40, "op": "mul", "dst": 5, "src": 7, "src2": 7, "imm": "0x070af1b6", "imm2": "0x6b648e75", "rot": 10, "bit": 6, "mask": 16}, - {"i": 41, "op": "mulhi", "dst": 2, "src": 3, "src2": 2, "imm": "0x389753b2", "imm2": "0x9e657305", "rot": 30, "bit": 2, "mask": 4}, - {"i": 42, "op": "xor", "dst": 2, "src": 0, "src2": 4, "imm": "0x0960e5b9", "imm2": "0xa925a406", "rot": 4, "bit": 6, "mask": 16}, - {"i": 43, "op": "rotl", "dst": 0, "src": 1, "src2": 1, "imm": "0x8eabe09f", "imm2": "0x76ef71e2", "rot": 4, "bit": 19, "mask": 8}, - {"i": 44, "op": "mulhi", "dst": 4, "src": 3, "src2": 7, "imm": "0x6a34768a", "imm2": "0x2e427c64", "rot": 21, "bit": 5, "mask": 4}, - {"i": 45, "op": "add", "dst": 6, "src": 7, "src2": 5, "imm": "0xee822e17", "imm2": "0xcdcb63f6", "rot": 27, "bit": 12, "mask": 16}, - {"i": 46, "op": "mad", "dst": 3, "src": 2, "src2": 0, "imm": "0xc89ead43", "imm2": "0x4d3108b2", "rot": 26, "bit": 17, "mask": 1}, - {"i": 47, "op": "shfl", "dst": 4, "src": 3, "src2": 0, "imm": "0xdd62be9e", "imm2": "0xf243f6f2", "rot": 8, "bit": 4, "mask": 8}, - {"i": 48, "op": "mulhi", "dst": 6, "src": 5, "src2": 0, "imm": "0xd3f7fa72", "imm2": "0x0debc83f", "rot": 25, "bit": 5, "mask": 2}, - {"i": 49, "op": "sub", "dst": 0, "src": 2, "src2": 6, "imm": "0x7afadd15", "imm2": "0xc07fc39c", "rot": 30, "bit": 29, "mask": 8}, - {"i": 50, "op": "sub", "dst": 3, "src": 5, "src2": 3, "imm": "0x179b78ec", "imm2": "0xaeccde37", "rot": 30, "bit": 17, "mask": 1}, - {"i": 51, "op": "rotr", "dst": 1, "src": 4, "src2": 1, "imm": "0xa4bfcea6", "imm2": "0xbf63bb2f", "rot": 2, "bit": 21, "mask": 2}, - {"i": 52, "op": "mad", "dst": 6, "src": 7, "src2": 7, "imm": "0xabf5ed10", "imm2": "0x8a285f51", "rot": 3, "bit": 23, "mask": 2}, - {"i": 53, "op": "xor", "dst": 5, "src": 3, "src2": 5, "imm": "0x88b416eb", "imm2": "0x36271d87", "rot": 19, "bit": 10, "mask": 2}, - {"i": 54, "op": "sub", "dst": 1, "src": 0, "src2": 1, "imm": "0xa3417dd3", "imm2": "0xcd98f620", "rot": 28, "bit": 2, "mask": 1}, - {"i": 55, "op": "sub", "dst": 5, "src": 6, "src2": 2, "imm": "0x45996c6f", "imm2": "0xe3d41087", "rot": 4, "bit": 31, "mask": 1}, - {"i": 56, "op": "add", "dst": 3, "src": 2, "src2": 0, "imm": "0x3dfad1b6", "imm2": "0xd4758987", "rot": 27, "bit": 10, "mask": 2}, - {"i": 57, "op": "add", "dst": 4, "src": 1, "src2": 3, "imm": "0xfca75bc2", "imm2": "0x0602d6be", "rot": 19, "bit": 0, "mask": 16}, - {"i": 58, "op": "mulhi", "dst": 5, "src": 6, "src2": 5, "imm": "0x83250a7b", "imm2": "0x2f93d53b", "rot": 25, "bit": 7, "mask": 2}, - {"i": 59, "op": "shfl", "dst": 2, "src": 1, "src2": 0, "imm": "0x13f4a089", "imm2": "0x145ea125", "rot": 12, "bit": 3, "mask": 2}, - {"i": 60, "op": "rotl", "dst": 2, "src": 5, "src2": 6, "imm": "0xad3170e3", "imm2": "0x15db04d1", "rot": 9, "bit": 13, "mask": 2}, - {"i": 61, "op": "or", "dst": 4, "src": 6, "src2": 2, "imm": "0x4b6305b2", "imm2": "0x6e7b2e9c", "rot": 27, "bit": 16, "mask": 4}, - {"i": 62, "op": "rotr", "dst": 6, "src": 4, "src2": 6, "imm": "0xfcd4b1c9", "imm2": "0xaf5c733b", "rot": 6, "bit": 21, "mask": 16}, - {"i": 63, "op": "mul", "dst": 2, "src": 4, "src2": 6, "imm": "0x95cebb3e", "imm2": "0xbba9cdfc", "rot": 19, "bit": 23, "mask": 1}, - {"i": 64, "op": "xor", "dst": 0, "src": 5, "src2": 7, "imm": "0x5f7579ff", "imm2": "0xb104e01a", "rot": 25, "bit": 12, "mask": 8}, - {"i": 65, "op": "xor", "dst": 2, "src": 7, "src2": 3, "imm": "0x749c7611", "imm2": "0xf17df3bb", "rot": 27, "bit": 26, "mask": 2}, - {"i": 66, "op": "add", "dst": 2, "src": 1, "src2": 6, "imm": "0xd71c02ff", "imm2": "0x8596687a", "rot": 4, "bit": 6, "mask": 2}, - {"i": 67, "op": "rotl", "dst": 1, "src": 3, "src2": 2, "imm": "0xd2864071", "imm2": "0xaa644ef3", "rot": 21, "bit": 5, "mask": 1}, - {"i": 68, "op": "mul", "dst": 3, "src": 2, "src2": 1, "imm": "0xd0689a5f", "imm2": "0xa3a1f063", "rot": 13, "bit": 28, "mask": 2}, - {"i": 69, "op": "shfl", "dst": 7, "src": 5, "src2": 6, "imm": "0xd01476eb", "imm2": "0x76abb5c2", "rot": 7, "bit": 30, "mask": 2}, - {"i": 70, "op": "mul", "dst": 3, "src": 2, "src2": 5, "imm": "0x59a75c10", "imm2": "0x1e1fbb72", "rot": 20, "bit": 16, "mask": 1}, - {"i": 71, "op": "mad", "dst": 0, "src": 2, "src2": 5, "imm": "0x39cca1df", "imm2": "0x22c057ac", "rot": 5, "bit": 23, "mask": 16}, - {"i": 72, "op": "add", "dst": 6, "src": 7, "src2": 3, "imm": "0x3c6fe15d", "imm2": "0x08ac5733", "rot": 14, "bit": 0, "mask": 4}, - {"i": 73, "op": "shfl", "dst": 3, "src": 2, "src2": 6, "imm": "0x2098627d", "imm2": "0xf4fecc81", "rot": 20, "bit": 30, "mask": 8}, - {"i": 74, "op": "mul", "dst": 3, "src": 6, "src2": 5, "imm": "0xf82e9e23", "imm2": "0x3ad62132", "rot": 10, "bit": 14, "mask": 16}, - {"i": 75, "op": "xor", "dst": 6, "src": 7, "src2": 4, "imm": "0x70548a91", "imm2": "0xa9715d2e", "rot": 6, "bit": 24, "mask": 1}, - {"i": 76, "op": "or", "dst": 3, "src": 0, "src2": 7, "imm": "0xa164325f", "imm2": "0xf300838b", "rot": 23, "bit": 23, "mask": 16}, - {"i": 77, "op": "add", "dst": 2, "src": 4, "src2": 6, "imm": "0x295d5fae", "imm2": "0xc100b495", "rot": 7, "bit": 1, "mask": 2}, - {"i": 78, "op": "mulhi", "dst": 3, "src": 7, "src2": 4, "imm": "0xb62cca87", "imm2": "0x2ebde415", "rot": 14, "bit": 7, "mask": 2}, - {"i": 79, "op": "or", "dst": 4, "src": 1, "src2": 7, "imm": "0xfcbc482d", "imm2": "0x8876e6cd", "rot": 29, "bit": 5, "mask": 2}, - {"i": 80, "op": "rotr", "dst": 4, "src": 3, "src2": 0, "imm": "0x6d64013b", "imm2": "0x675f4a8d", "rot": 26, "bit": 18, "mask": 8}, - {"i": 81, "op": "add", "dst": 4, "src": 3, "src2": 3, "imm": "0x274a9221", "imm2": "0x5cc59530", "rot": 15, "bit": 15, "mask": 2}, - {"i": 82, "op": "add", "dst": 7, "src": 3, "src2": 0, "imm": "0xc8651f8e", "imm2": "0x141479ec", "rot": 18, "bit": 5, "mask": 1}, - {"i": 83, "op": "rotr", "dst": 3, "src": 6, "src2": 0, "imm": "0xcc8a7766", "imm2": "0xc2eb5161", "rot": 4, "bit": 29, "mask": 2}, - {"i": 84, "op": "mad", "dst": 2, "src": 4, "src2": 7, "imm": "0xbc507d51", "imm2": "0x0b1196fd", "rot": 9, "bit": 7, "mask": 8}, - {"i": 85, "op": "shfl", "dst": 2, "src": 5, "src2": 5, "imm": "0x5a156c90", "imm2": "0xa6b3fbfa", "rot": 11, "bit": 2, "mask": 16}, - {"i": 86, "op": "xor", "dst": 3, "src": 2, "src2": 1, "imm": "0x8b042658", "imm2": "0xacf37a8f", "rot": 12, "bit": 23, "mask": 16}, - {"i": 87, "op": "shfl", "dst": 5, "src": 7, "src2": 2, "imm": "0x1a214238", "imm2": "0x017fdf5d", "rot": 29, "bit": 14, "mask": 2}, - {"i": 88, "op": "xor", "dst": 0, "src": 4, "src2": 2, "imm": "0xd523e612", "imm2": "0x2158c2ed", "rot": 30, "bit": 14, "mask": 4}, - {"i": 89, "op": "add", "dst": 3, "src": 2, "src2": 7, "imm": "0x53f915c2", "imm2": "0x883c0c92", "rot": 9, "bit": 18, "mask": 8}, - {"i": 90, "op": "rotr", "dst": 7, "src": 5, "src2": 5, "imm": "0xbd633b21", "imm2": "0xcf8c356c", "rot": 25, "bit": 31, "mask": 4}, - {"i": 91, "op": "add", "dst": 3, "src": 1, "src2": 1, "imm": "0xe2be00a5", "imm2": "0x3cc5bd20", "rot": 10, "bit": 31, "mask": 1}, - {"i": 92, "op": "shfl", "dst": 7, "src": 2, "src2": 4, "imm": "0x3d3da600", "imm2": "0x1f22df89", "rot": 4, "bit": 24, "mask": 1}, - {"i": 93, "op": "rotr", "dst": 6, "src": 2, "src2": 7, "imm": "0xb0f57471", "imm2": "0x8f2a7eca", "rot": 16, "bit": 25, "mask": 1}, - {"i": 94, "op": "rotl", "dst": 7, "src": 0, "src2": 4, "imm": "0x00a21815", "imm2": "0xeb4d7218", "rot": 14, "bit": 18, "mask": 16}, - {"i": 95, "op": "mad", "dst": 4, "src": 5, "src2": 5, "imm": "0xc4c3828e", "imm2": "0xfb7bba17", "rot": 16, "bit": 10, "mask": 16}, - {"i": 96, "op": "mad", "dst": 2, "src": 4, "src2": 5, "imm": "0xfc5bbc75", "imm2": "0xfc43468f", "rot": 31, "bit": 20, "mask": 16}, - {"i": 97, "op": "xor", "dst": 1, "src": 0, "src2": 2, "imm": "0x1fe9c249", "imm2": "0x164bb16b", "rot": 15, "bit": 9, "mask": 4}, - {"i": 98, "op": "mul", "dst": 5, "src": 4, "src2": 1, "imm": "0xeda725fa", "imm2": "0x66f7e9a2", "rot": 21, "bit": 6, "mask": 2}, - {"i": 99, "op": "sub", "dst": 2, "src": 0, "src2": 7, "imm": "0x8fb29f7d", "imm2": "0xf530eda1", "rot": 27, "bit": 30, "mask": 4}, - {"i": 100, "op": "rotl", "dst": 7, "src": 2, "src2": 0, "imm": "0x9f4de742", "imm2": "0x9b5ff871", "rot": 30, "bit": 21, "mask": 16}, - {"i": 101, "op": "add", "dst": 5, "src": 6, "src2": 7, "imm": "0x423fd9c9", "imm2": "0xbfd646cb", "rot": 17, "bit": 17, "mask": 2}, - {"i": 102, "op": "add", "dst": 7, "src": 6, "src2": 3, "imm": "0x8d3c011d", "imm2": "0x19b74a43", "rot": 12, "bit": 11, "mask": 4}, - {"i": 103, "op": "rotl", "dst": 5, "src": 6, "src2": 0, "imm": "0xcfc70303", "imm2": "0xf2b3e8ef", "rot": 6, "bit": 24, "mask": 1}, - {"i": 104, "op": "mul", "dst": 0, "src": 4, "src2": 5, "imm": "0x650475eb", "imm2": "0x11dcbd94", "rot": 15, "bit": 10, "mask": 1}, - {"i": 105, "op": "or", "dst": 0, "src": 5, "src2": 3, "imm": "0xaacaa145", "imm2": "0x9139d3fe", "rot": 4, "bit": 18, "mask": 2}, - {"i": 106, "op": "add", "dst": 0, "src": 1, "src2": 7, "imm": "0x8ce14721", "imm2": "0x7dcb7e18", "rot": 24, "bit": 15, "mask": 8}, - {"i": 107, "op": "rotl", "dst": 0, "src": 3, "src2": 3, "imm": "0xb36d6d98", "imm2": "0x2c3390c8", "rot": 15, "bit": 10, "mask": 16}, - {"i": 108, "op": "sub", "dst": 4, "src": 2, "src2": 5, "imm": "0xf6bbdaef", "imm2": "0x9db6f65e", "rot": 25, "bit": 10, "mask": 1}, - {"i": 109, "op": "mulhi", "dst": 2, "src": 0, "src2": 0, "imm": "0xb1721fd6", "imm2": "0xd96d52c9", "rot": 1, "bit": 27, "mask": 8}, - {"i": 110, "op": "mulhi", "dst": 1, "src": 0, "src2": 5, "imm": "0xfb4ca37f", "imm2": "0xb6ec7dbe", "rot": 27, "bit": 12, "mask": 8}, - {"i": 111, "op": "add", "dst": 0, "src": 2, "src2": 0, "imm": "0x2d9fc6b9", "imm2": "0x3c179ad8", "rot": 9, "bit": 28, "mask": 16}, - {"i": 112, "op": "mulhi", "dst": 2, "src": 0, "src2": 6, "imm": "0x71a9f6bd", "imm2": "0xd3417bf5", "rot": 2, "bit": 14, "mask": 8}, - {"i": 113, "op": "sub", "dst": 6, "src": 3, "src2": 5, "imm": "0xa3d19400", "imm2": "0x15df7330", "rot": 5, "bit": 2, "mask": 8}, - {"i": 114, "op": "mad", "dst": 7, "src": 6, "src2": 3, "imm": "0x1400d92e", "imm2": "0xfa4a158e", "rot": 16, "bit": 9, "mask": 1}, - {"i": 115, "op": "rotr", "dst": 5, "src": 1, "src2": 3, "imm": "0xd01684c3", "imm2": "0x6367febb", "rot": 23, "bit": 19, "mask": 2}, - {"i": 116, "op": "rotr", "dst": 6, "src": 4, "src2": 2, "imm": "0x8cd9eb96", "imm2": "0x98f33b24", "rot": 25, "bit": 1, "mask": 2}, - {"i": 117, "op": "add", "dst": 2, "src": 1, "src2": 7, "imm": "0x502138e2", "imm2": "0x47359729", "rot": 5, "bit": 22, "mask": 4}, - {"i": 118, "op": "mad", "dst": 3, "src": 2, "src2": 0, "imm": "0xd94a35c7", "imm2": "0xb83416c3", "rot": 11, "bit": 3, "mask": 4}, - {"i": 119, "op": "mul", "dst": 1, "src": 3, "src2": 6, "imm": "0x09b30cf7", "imm2": "0xef3b98e7", "rot": 17, "bit": 23, "mask": 8}, - {"i": 120, "op": "mad", "dst": 2, "src": 0, "src2": 4, "imm": "0x56416fe0", "imm2": "0x5e1dc8f3", "rot": 17, "bit": 14, "mask": 2}, - {"i": 121, "op": "mul", "dst": 0, "src": 3, "src2": 2, "imm": "0x2892697c", "imm2": "0x9cb3b14e", "rot": 29, "bit": 25, "mask": 2}, - {"i": 122, "op": "mulhi", "dst": 2, "src": 0, "src2": 3, "imm": "0xc33089a1", "imm2": "0xd6c5b530", "rot": 27, "bit": 13, "mask": 8}, - {"i": 123, "op": "mul", "dst": 5, "src": 6, "src2": 4, "imm": "0xdfe04e4a", "imm2": "0x3cc40160", "rot": 3, "bit": 14, "mask": 2}, - {"i": 124, "op": "mad", "dst": 4, "src": 2, "src2": 4, "imm": "0xb33dc1b7", "imm2": "0xab97743a", "rot": 7, "bit": 21, "mask": 4}, - {"i": 125, "op": "shfl", "dst": 4, "src": 2, "src2": 0, "imm": "0xfd469909", "imm2": "0xfc85dc55", "rot": 28, "bit": 24, "mask": 2}, - {"i": 126, "op": "xor", "dst": 2, "src": 0, "src2": 5, "imm": "0xa0094578", "imm2": "0xf1bee474", "rot": 31, "bit": 7, "mask": 2}, - {"i": 127, "op": "rotl", "dst": 1, "src": 7, "src2": 2, "imm": "0xb7ae00a0", "imm2": "0x86cce297", "rot": 7, "bit": 31, "mask": 8}, - {"i": 128, "op": "shfl", "dst": 7, "src": 2, "src2": 7, "imm": "0x019d1940", "imm2": "0x3fe9e7dd", "rot": 14, "bit": 4, "mask": 4}, - {"i": 129, "op": "rotl", "dst": 6, "src": 7, "src2": 7, "imm": "0x40a74cc4", "imm2": "0x09eb4adf", "rot": 17, "bit": 30, "mask": 1}, - {"i": 130, "op": "add", "dst": 2, "src": 5, "src2": 5, "imm": "0x33dff776", "imm2": "0x6c57e4e7", "rot": 27, "bit": 15, "mask": 4}, - {"i": 131, "op": "or", "dst": 0, "src": 3, "src2": 1, "imm": "0xe0c53bb9", "imm2": "0x124404f6", "rot": 4, "bit": 21, "mask": 16}, - {"i": 132, "op": "rotr", "dst": 5, "src": 0, "src2": 1, "imm": "0xe4353fce", "imm2": "0x559d0118", "rot": 2, "bit": 29, "mask": 4}, - {"i": 133, "op": "add", "dst": 2, "src": 3, "src2": 6, "imm": "0x5db25b34", "imm2": "0xe8212c0c", "rot": 21, "bit": 30, "mask": 1}, - {"i": 134, "op": "xor", "dst": 4, "src": 3, "src2": 6, "imm": "0x7366e50e", "imm2": "0x7cc1ffbd", "rot": 19, "bit": 18, "mask": 16}, - {"i": 135, "op": "xor", "dst": 6, "src": 1, "src2": 2, "imm": "0xa36decb7", "imm2": "0x79291734", "rot": 26, "bit": 0, "mask": 8}, - {"i": 136, "op": "sub", "dst": 1, "src": 7, "src2": 0, "imm": "0x225b03e4", "imm2": "0x7183e193", "rot": 16, "bit": 6, "mask": 2}, - {"i": 137, "op": "mad", "dst": 2, "src": 6, "src2": 2, "imm": "0x6f620d51", "imm2": "0x4e814e19", "rot": 6, "bit": 6, "mask": 2}, - {"i": 138, "op": "mulhi", "dst": 0, "src": 5, "src2": 6, "imm": "0x919d6bf3", "imm2": "0xc6240638", "rot": 17, "bit": 11, "mask": 16}, - {"i": 139, "op": "add", "dst": 2, "src": 7, "src2": 7, "imm": "0x218d4090", "imm2": "0xd44ff710", "rot": 1, "bit": 10, "mask": 16}, - {"i": 140, "op": "rotr", "dst": 1, "src": 4, "src2": 4, "imm": "0x4cbfa722", "imm2": "0x114e9564", "rot": 28, "bit": 9, "mask": 16}, - {"i": 141, "op": "shfl", "dst": 3, "src": 6, "src2": 2, "imm": "0xf702c6a1", "imm2": "0xf8f3c5d9", "rot": 7, "bit": 24, "mask": 1}, - {"i": 142, "op": "mad", "dst": 7, "src": 6, "src2": 2, "imm": "0xa2d285be", "imm2": "0x9cc94532", "rot": 15, "bit": 20, "mask": 8}, - {"i": 143, "op": "mul", "dst": 2, "src": 3, "src2": 7, "imm": "0x08b7ed80", "imm2": "0xa8abced4", "rot": 18, "bit": 10, "mask": 4}, - {"i": 144, "op": "add", "dst": 7, "src": 4, "src2": 2, "imm": "0xf4b1a8de", "imm2": "0xb98942fa", "rot": 18, "bit": 29, "mask": 8}, - {"i": 145, "op": "rotl", "dst": 7, "src": 6, "src2": 1, "imm": "0x64b6ba2d", "imm2": "0xf9e86793", "rot": 15, "bit": 24, "mask": 8}, - {"i": 146, "op": "xor", "dst": 7, "src": 5, "src2": 3, "imm": "0x41c42b5f", "imm2": "0x7c7e0e39", "rot": 8, "bit": 23, "mask": 2}, - {"i": 147, "op": "mad", "dst": 4, "src": 7, "src2": 4, "imm": "0x3caa807a", "imm2": "0x553a0cec", "rot": 11, "bit": 22, "mask": 8}, - {"i": 148, "op": "rotr", "dst": 6, "src": 5, "src2": 3, "imm": "0x3cb1289a", "imm2": "0x6b36f78a", "rot": 1, "bit": 9, "mask": 16}, - {"i": 149, "op": "mad", "dst": 1, "src": 2, "src2": 4, "imm": "0x95310ea8", "imm2": "0xc533aa6a", "rot": 16, "bit": 17, "mask": 1}, - {"i": 150, "op": "add", "dst": 1, "src": 7, "src2": 7, "imm": "0x2bef10f2", "imm2": "0x0d48ba42", "rot": 8, "bit": 17, "mask": 4}, - {"i": 151, "op": "xor", "dst": 5, "src": 4, "src2": 7, "imm": "0xda997ab2", "imm2": "0xae31d69e", "rot": 11, "bit": 31, "mask": 2}, - {"i": 152, "op": "add", "dst": 7, "src": 0, "src2": 6, "imm": "0xdc5cc080", "imm2": "0xc98dea9c", "rot": 16, "bit": 30, "mask": 2}, - {"i": 153, "op": "mul", "dst": 4, "src": 6, "src2": 7, "imm": "0xbfbf5f6c", "imm2": "0x61f611bb", "rot": 14, "bit": 22, "mask": 2}, - {"i": 154, "op": "mul", "dst": 0, "src": 2, "src2": 3, "imm": "0xa78008c3", "imm2": "0xbad5eeb1", "rot": 10, "bit": 28, "mask": 8}, - {"i": 155, "op": "xor", "dst": 6, "src": 5, "src2": 4, "imm": "0x1a73b866", "imm2": "0x1f5a62c9", "rot": 9, "bit": 27, "mask": 8}, - {"i": 156, "op": "rotr", "dst": 4, "src": 2, "src2": 0, "imm": "0x9c88500c", "imm2": "0xe25dccf9", "rot": 11, "bit": 2, "mask": 16}, - {"i": 157, "op": "rotl", "dst": 1, "src": 4, "src2": 4, "imm": "0xb05e7669", "imm2": "0x9db704b1", "rot": 11, "bit": 28, "mask": 4}, - {"i": 158, "op": "add", "dst": 5, "src": 0, "src2": 6, "imm": "0x18197438", "imm2": "0x6c752dcb", "rot": 11, "bit": 11, "mask": 2}, - {"i": 159, "op": "mad", "dst": 4, "src": 1, "src2": 5, "imm": "0x4796a65e", "imm2": "0x00e08c7a", "rot": 8, "bit": 19, "mask": 4}, - {"i": 160, "op": "rotl", "dst": 4, "src": 7, "src2": 5, "imm": "0x08a03052", "imm2": "0x0204f0ba", "rot": 26, "bit": 5, "mask": 2}, - {"i": 161, "op": "mad", "dst": 3, "src": 0, "src2": 7, "imm": "0xa0603b0e", "imm2": "0x7eee83d5", "rot": 28, "bit": 22, "mask": 16}, - {"i": 162, "op": "rotr", "dst": 3, "src": 1, "src2": 6, "imm": "0x78a3c69d", "imm2": "0x684693a0", "rot": 21, "bit": 23, "mask": 4}, - {"i": 163, "op": "add", "dst": 4, "src": 0, "src2": 7, "imm": "0xf1c46574", "imm2": "0x8e481727", "rot": 9, "bit": 3, "mask": 4}, - {"i": 164, "op": "mulhi", "dst": 0, "src": 4, "src2": 3, "imm": "0x04644afa", "imm2": "0x64bda2b5", "rot": 26, "bit": 16, "mask": 16}, - {"i": 165, "op": "add", "dst": 2, "src": 6, "src2": 2, "imm": "0xac578137", "imm2": "0x550ab406", "rot": 13, "bit": 20, "mask": 16}, - {"i": 166, "op": "rotl", "dst": 0, "src": 4, "src2": 5, "imm": "0x7f564760", "imm2": "0xb9a8b4f8", "rot": 13, "bit": 29, "mask": 1}, - {"i": 167, "op": "add", "dst": 3, "src": 1, "src2": 0, "imm": "0xa33e6706", "imm2": "0xaebb5966", "rot": 12, "bit": 14, "mask": 16}, - {"i": 168, "op": "or", "dst": 3, "src": 5, "src2": 3, "imm": "0x65b2f3eb", "imm2": "0xb1d00d20", "rot": 12, "bit": 2, "mask": 8}, - {"i": 169, "op": "rotr", "dst": 6, "src": 2, "src2": 3, "imm": "0x7f21faf5", "imm2": "0xbbf0d3f9", "rot": 17, "bit": 13, "mask": 8}, - {"i": 170, "op": "xor", "dst": 4, "src": 6, "src2": 2, "imm": "0x162c7140", "imm2": "0x90d404ad", "rot": 26, "bit": 1, "mask": 4}, - {"i": 171, "op": "sub", "dst": 6, "src": 1, "src2": 7, "imm": "0x6ef9c76e", "imm2": "0xfb7ba272", "rot": 31, "bit": 25, "mask": 1}, - {"i": 172, "op": "rotl", "dst": 7, "src": 5, "src2": 4, "imm": "0xde04eb3b", "imm2": "0xd56caa00", "rot": 22, "bit": 21, "mask": 4}, - {"i": 173, "op": "rotl", "dst": 5, "src": 3, "src2": 5, "imm": "0xa9eac934", "imm2": "0x2c338e51", "rot": 15, "bit": 22, "mask": 2}, - {"i": 174, "op": "shfl", "dst": 7, "src": 0, "src2": 3, "imm": "0x700be4e3", "imm2": "0x4bcfc732", "rot": 19, "bit": 6, "mask": 8}, - {"i": 175, "op": "xor", "dst": 0, "src": 5, "src2": 1, "imm": "0x02ccdba9", "imm2": "0xd0915be0", "rot": 15, "bit": 2, "mask": 16}, - {"i": 176, "op": "rotl", "dst": 7, "src": 4, "src2": 1, "imm": "0x489c8165", "imm2": "0xf24b5a4f", "rot": 6, "bit": 22, "mask": 1}, - {"i": 177, "op": "sub", "dst": 7, "src": 0, "src2": 6, "imm": "0x23ad9693", "imm2": "0x9a8c2f7b", "rot": 24, "bit": 2, "mask": 2}, - {"i": 178, "op": "rotl", "dst": 3, "src": 0, "src2": 6, "imm": "0xafa72a42", "imm2": "0x371d74ee", "rot": 30, "bit": 16, "mask": 1}, - {"i": 179, "op": "mad", "dst": 7, "src": 6, "src2": 1, "imm": "0x18a2a3f3", "imm2": "0xb811b951", "rot": 2, "bit": 9, "mask": 2}, - {"i": 180, "op": "rotl", "dst": 6, "src": 4, "src2": 1, "imm": "0xe6c69c0e", "imm2": "0xfc46a951", "rot": 9, "bit": 31, "mask": 4}, - {"i": 181, "op": "xor", "dst": 2, "src": 4, "src2": 7, "imm": "0xbd1b89e4", "imm2": "0xdf4bce5c", "rot": 15, "bit": 19, "mask": 4}, - {"i": 182, "op": "xor", "dst": 2, "src": 7, "src2": 5, "imm": "0x3dd12aed", "imm2": "0xd0756a69", "rot": 13, "bit": 16, "mask": 4}, - {"i": 183, "op": "xor", "dst": 7, "src": 2, "src2": 3, "imm": "0x77ce69d6", "imm2": "0x2b39bdf2", "rot": 8, "bit": 22, "mask": 16}, - {"i": 184, "op": "add", "dst": 1, "src": 2, "src2": 4, "imm": "0xd94d55ac", "imm2": "0x5bb7550f", "rot": 31, "bit": 21, "mask": 1}, - {"i": 185, "op": "or", "dst": 3, "src": 5, "src2": 6, "imm": "0x7b1ce846", "imm2": "0xcc3b8509", "rot": 28, "bit": 9, "mask": 4}, - {"i": 186, "op": "mulhi", "dst": 6, "src": 3, "src2": 0, "imm": "0xa5e24690", "imm2": "0x2200ba81", "rot": 26, "bit": 10, "mask": 16}, - {"i": 187, "op": "or", "dst": 4, "src": 0, "src2": 3, "imm": "0xf6efe759", "imm2": "0xae1f7118", "rot": 19, "bit": 20, "mask": 4}, - {"i": 188, "op": "shfl", "dst": 7, "src": 1, "src2": 5, "imm": "0xdb318b45", "imm2": "0xcec459c9", "rot": 20, "bit": 11, "mask": 2}, - {"i": 189, "op": "add", "dst": 6, "src": 5, "src2": 1, "imm": "0x89e747fe", "imm2": "0x2a354e2d", "rot": 22, "bit": 6, "mask": 1}, - {"i": 190, "op": "xor", "dst": 1, "src": 4, "src2": 2, "imm": "0xc379e617", "imm2": "0x75e9d63b", "rot": 23, "bit": 3, "mask": 2}, - {"i": 191, "op": "mulhi", "dst": 7, "src": 4, "src2": 3, "imm": "0x5a710287", "imm2": "0x7fe4ead6", "rot": 10, "bit": 25, "mask": 4}, - {"i": 192, "op": "rotl", "dst": 2, "src": 1, "src2": 2, "imm": "0x4d5e59a3", "imm2": "0x1e4fef28", "rot": 26, "bit": 0, "mask": 1}, - {"i": 193, "op": "rotl", "dst": 5, "src": 3, "src2": 7, "imm": "0x7444c47d", "imm2": "0xdad5f8be", "rot": 8, "bit": 30, "mask": 2}, - {"i": 194, "op": "shfl", "dst": 4, "src": 5, "src2": 7, "imm": "0x6a225bbb", "imm2": "0xd6532cd7", "rot": 13, "bit": 17, "mask": 16}, - {"i": 195, "op": "xor", "dst": 4, "src": 5, "src2": 3, "imm": "0x68344b9a", "imm2": "0xcb46a38b", "rot": 1, "bit": 27, "mask": 16}, - {"i": 196, "op": "mul", "dst": 1, "src": 3, "src2": 4, "imm": "0xbebf7359", "imm2": "0x3f0890ba", "rot": 18, "bit": 12, "mask": 4}, - {"i": 197, "op": "add", "dst": 5, "src": 2, "src2": 3, "imm": "0xea03e8e7", "imm2": "0x10cfdc71", "rot": 31, "bit": 7, "mask": 8}, - {"i": 198, "op": "add", "dst": 7, "src": 5, "src2": 1, "imm": "0xa7ee0102", "imm2": "0x66e148d3", "rot": 12, "bit": 15, "mask": 1}, - {"i": 199, "op": "sub", "dst": 5, "src": 2, "src2": 4, "imm": "0xc215584e", "imm2": "0x55fce30f", "rot": 30, "bit": 9, "mask": 2}, - {"i": 200, "op": "shfl", "dst": 5, "src": 1, "src2": 1, "imm": "0x308f8358", "imm2": "0x489f1f93", "rot": 13, "bit": 13, "mask": 2}, - {"i": 201, "op": "shfl", "dst": 7, "src": 1, "src2": 5, "imm": "0x713f1957", "imm2": "0x2239f757", "rot": 15, "bit": 7, "mask": 8}, - {"i": 202, "op": "rotr", "dst": 4, "src": 5, "src2": 1, "imm": "0xb037b6e7", "imm2": "0x49a8b528", "rot": 27, "bit": 13, "mask": 16}, - {"i": 203, "op": "xor", "dst": 7, "src": 5, "src2": 1, "imm": "0x56110241", "imm2": "0xefc8386d", "rot": 22, "bit": 20, "mask": 1}, - {"i": 204, "op": "xor", "dst": 7, "src": 0, "src2": 0, "imm": "0x0827fcc7", "imm2": "0xf4f5dd07", "rot": 13, "bit": 7, "mask": 2}, - {"i": 205, "op": "add", "dst": 6, "src": 2, "src2": 0, "imm": "0x8379a4de", "imm2": "0x8558b619", "rot": 26, "bit": 10, "mask": 1}, - {"i": 206, "op": "rotl", "dst": 4, "src": 3, "src2": 3, "imm": "0x091ceef0", "imm2": "0x69b0f72f", "rot": 13, "bit": 7, "mask": 1}, - {"i": 207, "op": "mulhi", "dst": 1, "src": 3, "src2": 4, "imm": "0x758edac1", "imm2": "0x98dd2f21", "rot": 15, "bit": 9, "mask": 1}, - {"i": 208, "op": "mad", "dst": 1, "src": 4, "src2": 4, "imm": "0x78871a1a", "imm2": "0x5e14e1c8", "rot": 19, "bit": 6, "mask": 2}, - {"i": 209, "op": "shfl", "dst": 2, "src": 0, "src2": 2, "imm": "0xf7e0b9aa", "imm2": "0xaecfd347", "rot": 21, "bit": 9, "mask": 4}, - {"i": 210, "op": "add", "dst": 7, "src": 0, "src2": 7, "imm": "0xaa8cb14e", "imm2": "0xf4049c4c", "rot": 24, "bit": 11, "mask": 8}, - {"i": 211, "op": "shfl", "dst": 7, "src": 1, "src2": 6, "imm": "0xc230d919", "imm2": "0xf76d08fb", "rot": 21, "bit": 13, "mask": 16}, - {"i": 212, "op": "xor", "dst": 1, "src": 0, "src2": 2, "imm": "0x31a5af90", "imm2": "0x7eef58ee", "rot": 9, "bit": 12, "mask": 4}, - {"i": 213, "op": "rotl", "dst": 7, "src": 1, "src2": 6, "imm": "0x0db26138", "imm2": "0x8e3c31f9", "rot": 3, "bit": 26, "mask": 2}, - {"i": 214, "op": "rotr", "dst": 4, "src": 7, "src2": 3, "imm": "0x29558100", "imm2": "0xe4b13ad6", "rot": 29, "bit": 24, "mask": 8}, - {"i": 215, "op": "shfl", "dst": 3, "src": 2, "src2": 6, "imm": "0x1b48c3d0", "imm2": "0x5c674ff6", "rot": 5, "bit": 9, "mask": 16}, - {"i": 216, "op": "or", "dst": 5, "src": 1, "src2": 0, "imm": "0xef768632", "imm2": "0x6de9d10d", "rot": 10, "bit": 4, "mask": 4}, - {"i": 217, "op": "sub", "dst": 1, "src": 2, "src2": 5, "imm": "0x88ca7f5a", "imm2": "0x24718a36", "rot": 18, "bit": 31, "mask": 1}, - {"i": 218, "op": "sub", "dst": 6, "src": 5, "src2": 0, "imm": "0xc4a06728", "imm2": "0xdc2a4fd8", "rot": 9, "bit": 9, "mask": 2}, - {"i": 219, "op": "rotl", "dst": 6, "src": 5, "src2": 7, "imm": "0xb7489e47", "imm2": "0xf13795c5", "rot": 4, "bit": 3, "mask": 8}, - {"i": 220, "op": "mulhi", "dst": 2, "src": 0, "src2": 2, "imm": "0x873cd31b", "imm2": "0x3dfbc55d", "rot": 12, "bit": 23, "mask": 8}, - {"i": 221, "op": "or", "dst": 2, "src": 0, "src2": 3, "imm": "0xdc21f099", "imm2": "0xee06f01e", "rot": 2, "bit": 17, "mask": 8}, - {"i": 222, "op": "shfl", "dst": 5, "src": 2, "src2": 6, "imm": "0x36def499", "imm2": "0xa2849d59", "rot": 23, "bit": 4, "mask": 8}, - {"i": 223, "op": "mulhi", "dst": 5, "src": 6, "src2": 7, "imm": "0xfb95fbca", "imm2": "0xc1aac427", "rot": 14, "bit": 11, "mask": 2}, - {"i": 224, "op": "sub", "dst": 0, "src": 6, "src2": 7, "imm": "0x3d9d29c4", "imm2": "0x34d0dcc0", "rot": 17, "bit": 6, "mask": 4}, - {"i": 225, "op": "rotl", "dst": 7, "src": 0, "src2": 5, "imm": "0x93b01b8e", "imm2": "0xfe1d75ac", "rot": 23, "bit": 15, "mask": 1}, - {"i": 226, "op": "or", "dst": 4, "src": 2, "src2": 4, "imm": "0xfed76e8e", "imm2": "0x1c24ecd8", "rot": 2, "bit": 6, "mask": 16}, - {"i": 227, "op": "mul", "dst": 2, "src": 4, "src2": 1, "imm": "0x5795f5b0", "imm2": "0x0566ea2a", "rot": 6, "bit": 28, "mask": 4}, - {"i": 228, "op": "rotl", "dst": 3, "src": 0, "src2": 2, "imm": "0x8c8486de", "imm2": "0xd066aa8f", "rot": 12, "bit": 20, "mask": 1}, - {"i": 229, "op": "rotr", "dst": 0, "src": 4, "src2": 0, "imm": "0x23a3e882", "imm2": "0xaca23902", "rot": 5, "bit": 22, "mask": 16}, - {"i": 230, "op": "add", "dst": 0, "src": 6, "src2": 4, "imm": "0x1d176220", "imm2": "0x2a7fecb2", "rot": 20, "bit": 16, "mask": 2}, - {"i": 231, "op": "shfl", "dst": 1, "src": 2, "src2": 0, "imm": "0x91172787", "imm2": "0xc5d7af28", "rot": 3, "bit": 24, "mask": 4}, - {"i": 232, "op": "mul", "dst": 2, "src": 1, "src2": 2, "imm": "0x0be68835", "imm2": "0xde692bdb", "rot": 30, "bit": 17, "mask": 1}, - {"i": 233, "op": "mad", "dst": 7, "src": 0, "src2": 1, "imm": "0xf90d2db5", "imm2": "0x96c4c175", "rot": 28, "bit": 7, "mask": 4}, - {"i": 234, "op": "rotl", "dst": 5, "src": 2, "src2": 1, "imm": "0x16379736", "imm2": "0x6973b905", "rot": 22, "bit": 17, "mask": 4}, - {"i": 235, "op": "rotr", "dst": 4, "src": 6, "src2": 2, "imm": "0x84292a13", "imm2": "0x0f897740", "rot": 12, "bit": 6, "mask": 8}, - {"i": 236, "op": "mad", "dst": 0, "src": 5, "src2": 1, "imm": "0xeb7de837", "imm2": "0x64f0c302", "rot": 4, "bit": 19, "mask": 1}, - {"i": 237, "op": "xor", "dst": 6, "src": 4, "src2": 4, "imm": "0x6ab4b683", "imm2": "0x20f17adb", "rot": 1, "bit": 0, "mask": 16}, - {"i": 238, "op": "xor", "dst": 4, "src": 6, "src2": 1, "imm": "0x5836b35c", "imm2": "0x3293cc4a", "rot": 16, "bit": 22, "mask": 16}, - {"i": 239, "op": "rotl", "dst": 6, "src": 1, "src2": 6, "imm": "0x769d8bc0", "imm2": "0xdc86c9cc", "rot": 18, "bit": 27, "mask": 16}, - {"i": 240, "op": "add", "dst": 4, "src": 7, "src2": 1, "imm": "0x17dafb4d", "imm2": "0xadce39f3", "rot": 12, "bit": 25, "mask": 8}, - {"i": 241, "op": "sub", "dst": 0, "src": 7, "src2": 4, "imm": "0xd4690bda", "imm2": "0xdab9b27b", "rot": 30, "bit": 21, "mask": 1}, - {"i": 242, "op": "rotr", "dst": 1, "src": 6, "src2": 2, "imm": "0x670a2d0a", "imm2": "0x0612e33c", "rot": 31, "bit": 25, "mask": 2}, - {"i": 243, "op": "add", "dst": 3, "src": 0, "src2": 2, "imm": "0xf2f77d26", "imm2": "0x0e7033b6", "rot": 27, "bit": 29, "mask": 1}, - {"i": 244, "op": "mad", "dst": 2, "src": 7, "src2": 4, "imm": "0xa9eefc9d", "imm2": "0x16166c85", "rot": 18, "bit": 23, "mask": 16}, - {"i": 245, "op": "mad", "dst": 4, "src": 0, "src2": 6, "imm": "0xb26f6c0f", "imm2": "0x0630e821", "rot": 23, "bit": 30, "mask": 4}, - {"i": 246, "op": "mul", "dst": 5, "src": 7, "src2": 2, "imm": "0x269ce4bf", "imm2": "0x1ce28d32", "rot": 30, "bit": 15, "mask": 16}, - {"i": 247, "op": "add", "dst": 3, "src": 5, "src2": 0, "imm": "0xfed2da4e", "imm2": "0x7b2ff6b7", "rot": 25, "bit": 31, "mask": 2}, - {"i": 248, "op": "add", "dst": 0, "src": 7, "src2": 5, "imm": "0x03b2891c", "imm2": "0xb5fad7b1", "rot": 2, "bit": 0, "mask": 4}, - {"i": 249, "op": "mulhi", "dst": 5, "src": 4, "src2": 6, "imm": "0xa69a1e71", "imm2": "0x15f0c0ea", "rot": 4, "bit": 1, "mask": 4}, - {"i": 250, "op": "add", "dst": 5, "src": 2, "src2": 3, "imm": "0x042cd6e3", "imm2": "0xa7e71c2f", "rot": 24, "bit": 11, "mask": 8}, - {"i": 251, "op": "shfl", "dst": 6, "src": 1, "src2": 2, "imm": "0x89c6b683", "imm2": "0x10bbf661", "rot": 24, "bit": 5, "mask": 1}, - {"i": 252, "op": "xor", "dst": 6, "src": 1, "src2": 3, "imm": "0x265c66d6", "imm2": "0xd4a689ed", "rot": 6, "bit": 14, "mask": 8}, - {"i": 253, "op": "mul", "dst": 2, "src": 5, "src2": 5, "imm": "0x890b8201", "imm2": "0x97c36bf3", "rot": 17, "bit": 22, "mask": 4}, - {"i": 254, "op": "add", "dst": 0, "src": 6, "src2": 0, "imm": "0x784a302b", "imm2": "0xb83d78de", "rot": 27, "bit": 16, "mask": 4}, - {"i": 255, "op": "sub", "dst": 2, "src": 4, "src2": 0, "imm": "0x817adb38", "imm2": "0xf3a3534b", "rot": 7, "bit": 22, "mask": 4} - ]}, - "instructions": [ - {"i": 0, "op": "mad", "dst": 2, "src": 3, "src2": 4, "imm": "0xbf7b174d", "imm2": "0x337b762e", "rot": 17, "bit": 2, "mask": 2, "width": 1}, - {"i": 1, "op": "mad", "dst": 2, "src": 1, "src2": 1, "imm": "0xdd04a5da", "imm2": "0x42da7657", "rot": 15, "bit": 30, "mask": 16, "width": 1}, - {"i": 2, "op": "mad", "dst": 2, "src": 3, "src2": 2, "imm": "0x734003fa", "imm2": "0x5bb67700", "rot": 3, "bit": 20, "mask": 1, "width": 1}, - {"i": 3, "op": "xor", "dst": 3, "src": 5, "src2": 5, "imm": "0xc55a1b1c", "imm2": "0xa19720f3", "rot": 7, "bit": 8, "mask": 1, "width": 1}, - {"i": 4, "op": "load", "dst": 7, "src": 2, "src2": 5, "imm": "0xad572dd7", "imm2": "0x9ceb3ea7", "rot": 18, "bit": 30, "mask": 2, "width": 1}, - {"i": 5, "op": "load", "dst": 5, "src": 7, "src2": 3, "imm": "0x769a53be", "imm2": "0x80f9067e", "rot": 12, "bit": 22, "mask": 1, "width": 1}, - {"i": 6, "op": "shfl", "dst": 1, "src": 4, "src2": 0, "imm": "0xd3613d88", "imm2": "0x262fb219", "rot": 10, "bit": 30, "mask": 8, "width": 1}, - {"i": 7, "op": "shfl", "dst": 7, "src": 3, "src2": 4, "imm": "0xce38e42f", "imm2": "0xb868b818", "rot": 11, "bit": 8, "mask": 8, "width": 1}, - {"i": 8, "op": "mulhi", "dst": 1, "src": 5, "src2": 2, "imm": "0xa5eebca5", "imm2": "0x5703a72b", "rot": 13, "bit": 13, "mask": 16, "width": 1}, - {"i": 9, "op": "rotr", "dst": 6, "src": 3, "src2": 4, "imm": "0x17a5a9c7", "imm2": "0xdcfb93a1", "rot": 20, "bit": 27, "mask": 2, "width": 1}, - {"i": 10, "op": "or", "dst": 3, "src": 4, "src2": 1, "imm": "0xccb7d785", "imm2": "0xc335364c", "rot": 14, "bit": 12, "mask": 4, "width": 1}, - {"i": 11, "op": "load", "dst": 4, "src": 3, "src2": 2, "imm": "0x88cb9af3", "imm2": "0x4e7dc10d", "rot": 24, "bit": 17, "mask": 4, "width": 1}, - {"i": 12, "op": "mulhi", "dst": 0, "src": 4, "src2": 4, "imm": "0x45374321", "imm2": "0x3cd91989", "rot": 11, "bit": 4, "mask": 2, "width": 1}, - {"i": 13, "op": "add", "dst": 5, "src": 1, "src2": 2, "imm": "0xc7934706", "imm2": "0xd3177981", "rot": 16, "bit": 30, "mask": 2, "width": 1}, - {"i": 14, "op": "load", "dst": 0, "src": 4, "src2": 3, "imm": "0xfd7f56bb", "imm2": "0x65e14f52", "rot": 13, "bit": 22, "mask": 2, "width": 1}, - {"i": 15, "op": "sub", "dst": 2, "src": 4, "src2": 4, "imm": "0x35a80b49", "imm2": "0x060f2d13", "rot": 16, "bit": 20, "mask": 16, "width": 1}, - {"i": 16, "op": "load", "dst": 2, "src": 0, "src2": 2, "imm": "0xae0a32c2", "imm2": "0x4c2a4cfe", "rot": 8, "bit": 31, "mask": 16, "width": 1}, - {"i": 17, "op": "load", "dst": 7, "src": 2, "src2": 6, "imm": "0x82a84cc3", "imm2": "0x21a38d68", "rot": 15, "bit": 21, "mask": 2, "width": 1}, - {"i": 18, "op": "shfl", "dst": 7, "src": 3, "src2": 3, "imm": "0xa3818806", "imm2": "0x8f66b5c8", "rot": 14, "bit": 6, "mask": 4, "width": 1}, - {"i": 19, "op": "mul", "dst": 5, "src": 0, "src2": 1, "imm": "0xa00de107", "imm2": "0x77bfcaa5", "rot": 3, "bit": 10, "mask": 2, "width": 1}, - {"i": 20, "op": "shfl", "dst": 3, "src": 4, "src2": 7, "imm": "0x1d2b8cab", "imm2": "0x80b4f9a2", "rot": 14, "bit": 25, "mask": 2, "width": 1}, - {"i": 21, "op": "shfl", "dst": 2, "src": 4, "src2": 5, "imm": "0x3ac915d2", "imm2": "0x5fba7bc2", "rot": 16, "bit": 1, "mask": 16, "width": 1}, - {"i": 22, "op": "mulhi", "dst": 6, "src": 2, "src2": 0, "imm": "0xdc3ec8fd", "imm2": "0x599e2fa3", "rot": 22, "bit": 3, "mask": 2, "width": 1}, - {"i": 23, "op": "load", "dst": 6, "src": 1, "src2": 5, "imm": "0x2a6b16d5", "imm2": "0xd73e396f", "rot": 28, "bit": 29, "mask": 2, "width": 1}, - {"i": 24, "op": "mul", "dst": 5, "src": 0, "src2": 7, "imm": "0x376d0223", "imm2": "0xe1c2169a", "rot": 4, "bit": 16, "mask": 16, "width": 1}, - {"i": 25, "op": "rotl", "dst": 5, "src": 7, "src2": 3, "imm": "0x78ad8c60", "imm2": "0x6f5b77d5", "rot": 19, "bit": 11, "mask": 16, "width": 1}, - {"i": 26, "op": "shfl", "dst": 7, "src": 6, "src2": 5, "imm": "0x93915b9f", "imm2": "0x1e61fb6b", "rot": 28, "bit": 23, "mask": 2, "width": 1}, - {"i": 27, "op": "xor", "dst": 0, "src": 5, "src2": 7, "imm": "0x6378fe15", "imm2": "0x66c78f42", "rot": 12, "bit": 31, "mask": 8, "width": 1}, - {"i": 28, "op": "xor", "dst": 0, "src": 4, "src2": 7, "imm": "0x20a57fda", "imm2": "0x088c848e", "rot": 16, "bit": 13, "mask": 4, "width": 1}, - {"i": 29, "op": "sub", "dst": 3, "src": 0, "src2": 2, "imm": "0x49d95fd5", "imm2": "0x1a5f946a", "rot": 6, "bit": 12, "mask": 1, "width": 1}, - {"i": 30, "op": "mul", "dst": 5, "src": 1, "src2": 7, "imm": "0x0a816217", "imm2": "0x405c4f73", "rot": 13, "bit": 27, "mask": 4, "width": 1}, - {"i": 31, "op": "load", "dst": 7, "src": 2, "src2": 2, "imm": "0x09ed045e", "imm2": "0xd69c4715", "rot": 5, "bit": 9, "mask": 2, "width": 1}, - {"i": 32, "op": "load", "dst": 1, "src": 0, "src2": 6, "imm": "0xeb79ea49", "imm2": "0xcc587f5a", "rot": 6, "bit": 8, "mask": 16, "width": 1}, - {"i": 33, "op": "xor", "dst": 5, "src": 6, "src2": 1, "imm": "0x3027401e", "imm2": "0x5f20c27e", "rot": 18, "bit": 9, "mask": 2, "width": 1}, - {"i": 34, "op": "load", "dst": 5, "src": 1, "src2": 3, "imm": "0x0e1cab07", "imm2": "0x09356c5b", "rot": 19, "bit": 31, "mask": 1, "width": 1}, - {"i": 35, "op": "mulhi", "dst": 0, "src": 5, "src2": 2, "imm": "0x90e31357", "imm2": "0xabd32484", "rot": 26, "bit": 5, "mask": 8, "width": 1}, - {"i": 36, "op": "shfl", "dst": 5, "src": 2, "src2": 5, "imm": "0xee9a955f", "imm2": "0x31b3faed", "rot": 8, "bit": 24, "mask": 4, "width": 1}, - {"i": 37, "op": "load", "dst": 7, "src": 0, "src2": 7, "imm": "0x3ba2f832", "imm2": "0x1160dcd3", "rot": 4, "bit": 29, "mask": 1, "width": 1}, - {"i": 38, "op": "add", "dst": 3, "src": 1, "src2": 7, "imm": "0x75ba2fad", "imm2": "0x230c005c", "rot": 4, "bit": 27, "mask": 1, "width": 1}, - {"i": 39, "op": "shfl", "dst": 1, "src": 5, "src2": 3, "imm": "0xdbf37e75", "imm2": "0xb5ac1969", "rot": 30, "bit": 13, "mask": 4, "width": 1}, - {"i": 40, "op": "xor", "dst": 2, "src": 5, "src2": 5, "imm": "0x47f136c5", "imm2": "0x06ce9153", "rot": 19, "bit": 10, "mask": 2, "width": 1}, - {"i": 41, "op": "mad", "dst": 3, "src": 6, "src2": 3, "imm": "0xce13eff8", "imm2": "0x04cc1d55", "rot": 3, "bit": 1, "mask": 4, "width": 1}, - {"i": 42, "op": "sub", "dst": 6, "src": 7, "src2": 1, "imm": "0x6a65ab71", "imm2": "0x8fbc1bcd", "rot": 4, "bit": 1, "mask": 8, "width": 1}, - {"i": 43, "op": "xor", "dst": 7, "src": 0, "src2": 7, "imm": "0xdaeb4928", "imm2": "0xc0423027", "rot": 24, "bit": 11, "mask": 8, "width": 1}, - {"i": 44, "op": "load", "dst": 1, "src": 7, "src2": 7, "imm": "0x778f01c9", "imm2": "0x28cedcea", "rot": 12, "bit": 4, "mask": 16, "width": 1}, - {"i": 45, "op": "mul", "dst": 2, "src": 3, "src2": 2, "imm": "0xf4264f1b", "imm2": "0x0f627d56", "rot": 5, "bit": 28, "mask": 8, "width": 1}, - {"i": 46, "op": "mulhi", "dst": 1, "src": 5, "src2": 1, "imm": "0xffb2147a", "imm2": "0xccde9b05", "rot": 13, "bit": 9, "mask": 2, "width": 1}, - {"i": 47, "op": "sub", "dst": 4, "src": 3, "src2": 5, "imm": "0x73b36234", "imm2": "0x3f5d5997", "rot": 7, "bit": 18, "mask": 2, "width": 1}, - {"i": 48, "op": "rotr", "dst": 2, "src": 6, "src2": 3, "imm": "0x3a4d9aa9", "imm2": "0x212bec7b", "rot": 4, "bit": 29, "mask": 16, "width": 1}, - {"i": 49, "op": "load", "dst": 3, "src": 5, "src2": 2, "imm": "0x626f11df", "imm2": "0x56cd5bfd", "rot": 7, "bit": 1, "mask": 1, "width": 1}, - {"i": 50, "op": "add", "dst": 1, "src": 5, "src2": 6, "imm": "0x81b8bc2c", "imm2": "0x1907970c", "rot": 28, "bit": 7, "mask": 4, "width": 1}, - {"i": 51, "op": "mul", "dst": 0, "src": 2, "src2": 2, "imm": "0xa8848b30", "imm2": "0xef6ac348", "rot": 9, "bit": 15, "mask": 8, "width": 1}, - {"i": 52, "op": "add", "dst": 0, "src": 2, "src2": 0, "imm": "0x4f92b968", "imm2": "0x699fd448", "rot": 22, "bit": 6, "mask": 4, "width": 1}, - {"i": 53, "op": "add", "dst": 1, "src": 0, "src2": 2, "imm": "0x2bb965af", "imm2": "0x77b1520d", "rot": 2, "bit": 12, "mask": 8, "width": 1}, - {"i": 54, "op": "rotl", "dst": 7, "src": 1, "src2": 0, "imm": "0x553e678b", "imm2": "0x3cc8eae0", "rot": 14, "bit": 20, "mask": 2, "width": 1}, - {"i": 55, "op": "add", "dst": 3, "src": 7, "src2": 2, "imm": "0x7b0fe07a", "imm2": "0xa54c55a0", "rot": 10, "bit": 1, "mask": 1, "width": 1}, - {"i": 56, "op": "load", "dst": 6, "src": 7, "src2": 4, "imm": "0x01eba9aa", "imm2": "0x2758c0f7", "rot": 14, "bit": 15, "mask": 4, "width": 1}, - {"i": 57, "op": "rotr", "dst": 1, "src": 5, "src2": 2, "imm": "0x1f5267b3", "imm2": "0x236f5a27", "rot": 2, "bit": 31, "mask": 16, "width": 1}, - {"i": 58, "op": "load", "dst": 5, "src": 4, "src2": 3, "imm": "0xa9954a9b", "imm2": "0x6a54d4e8", "rot": 11, "bit": 10, "mask": 16, "width": 1}, - {"i": 59, "op": "load", "dst": 6, "src": 2, "src2": 4, "imm": "0x9923ff88", "imm2": "0x9357254e", "rot": 16, "bit": 1, "mask": 16, "width": 1}, - {"i": 60, "op": "mad", "dst": 3, "src": 5, "src2": 0, "imm": "0xc0cc51a6", "imm2": "0x3fd7701b", "rot": 20, "bit": 1, "mask": 4, "width": 1}, - {"i": 61, "op": "add", "dst": 5, "src": 7, "src2": 7, "imm": "0xaf9dd72d", "imm2": "0xad7493e7", "rot": 7, "bit": 31, "mask": 16, "width": 1}, - {"i": 62, "op": "add", "dst": 4, "src": 6, "src2": 2, "imm": "0x89841d87", "imm2": "0x1e07c3d9", "rot": 6, "bit": 27, "mask": 1, "width": 1}, - {"i": 63, "op": "rotl", "dst": 5, "src": 4, "src2": 2, "imm": "0xae210f8d", "imm2": "0x8e499ba4", "rot": 19, "bit": 9, "mask": 1, "width": 1} - ] -} diff --git a/proto-cuda/packs-ca4/mx8_sh256x27/program.metal b/proto-cuda/packs-ca4/mx8_sh256x27/program.metal deleted file mode 100644 index 2830702a7..000000000 --- a/proto-cuda/packs-ca4/mx8_sh256x27/program.metal +++ /dev/null @@ -1,368 +0,0 @@ -#include -using namespace metal; - -#define MASK 0x0fffffffu -constant uint SEEDW[8] = { 0x67a9a7beu, 0x1a155b25u, 0xfddfb732u, 0x4b5af2e8u, 0xc55caf33u, 0xa27c13b7u, 0x06628a48u, 0x03852469u }; - -inline uint splitmix32(uint x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -inline uint rotl_imm(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 -inline uint rotr_var(uint x, uint n) { n &= 31u; return (x >> n) | (x << ((32u - n) & 31u)); } -inline uint ds_elem(uint i, uint d0, uint d1) { - uint x = i ^ d0; - x *= 0x9E3779B1u; x ^= x >> 15; - x += d1; - x *= 0x85EBCA77u; x ^= x >> 13; - x *= 0xC2B2AE3Du; x ^= x >> 16; - return x; -} - -kernel void igneum_hash(device const uint* dataset [[buffer(0)]], - device ulong* out [[buffer(1)]], - constant uint& baseNonce [[buffer(2)]], - uint gid [[thread_position_in_grid]]) { - uint nonce = baseNonce + gid; - uint r0, r1, r2, r3, r4, r5, r6, r7; - { uint x = nonce ^ SEEDW[0]; x += 0x9e3779b9u * 1u; x = splitmix32(x); r0 = x ^ SEEDW[1]; } - { uint x = nonce ^ SEEDW[1]; x += 0x9e3779b9u * 2u; x = splitmix32(x); r1 = x ^ SEEDW[2]; } - { uint x = nonce ^ SEEDW[2]; x += 0x9e3779b9u * 3u; x = splitmix32(x); r2 = x ^ SEEDW[3]; } - { uint x = nonce ^ SEEDW[3]; x += 0x9e3779b9u * 4u; x = splitmix32(x); r3 = x ^ SEEDW[4]; } - { uint x = nonce ^ SEEDW[4]; x += 0x9e3779b9u * 5u; x = splitmix32(x); r4 = x ^ SEEDW[5]; } - { uint x = nonce ^ SEEDW[5]; x += 0x9e3779b9u * 6u; x = splitmix32(x); r5 = x ^ SEEDW[6]; } - { uint x = nonce ^ SEEDW[6]; x += 0x9e3779b9u * 7u; x = splitmix32(x); r6 = x ^ SEEDW[7]; } - { uint x = nonce ^ SEEDW[7]; x += 0x9e3779b9u * 8u; x = splitmix32(x); r7 = x ^ SEEDW[0]; } - - for (uint it = 0u; it < 8u; ++it) { - uint sel = r0; - r2 = r3 * r4 + r2; // 0 - r2 = r1 * r1 + r2; // 1 - r2 = r3 * r2 + r2; // 2 - r3 = r3 ^ r5; // 3 - r7 = r7 ^ dataset[r2 & MASK]; // 4 - r5 = r5 ^ dataset[r7 & MASK]; // 5 - r1 = r1 ^ simd_shuffle_xor(r4, (ushort)8); // 6 - r7 = r7 ^ simd_shuffle_xor(r3, (ushort)8); // 7 - r1 = mulhi(r1, r5); // 8 - r6 = rotr_var(r6, r3); // 9 - r3 = r3 | r4; // 10 - r4 = r4 ^ dataset[r3 & MASK]; // 11 - r0 = mulhi(r0, r4); // 12 - r5 = r5 + r1 + select(0xc7934706u, 0xd3177981u, ((sel >> 30u) & 1u) != 0u); // 13 - r0 = r0 ^ dataset[r4 & MASK]; // 14 - r2 = r2 - r4; // 15 - r2 = r2 ^ dataset[r0 & MASK]; // 16 - r7 = r7 ^ dataset[r2 & MASK]; // 17 - r7 = r7 ^ simd_shuffle_xor(r3, (ushort)4); // 18 - r5 = r5 * r0; // 19 - r3 = r3 ^ simd_shuffle_xor(r4, (ushort)2); // 20 - r2 = r2 ^ simd_shuffle_xor(r4, (ushort)16); // 21 - r6 = mulhi(r6, r2); // 22 - r6 = r6 ^ dataset[r1 & MASK]; // 23 - r5 = r5 * r0; // 24 - r5 = rotl_imm(r5, 19u); // 25 - r7 = r7 ^ simd_shuffle_xor(r6, (ushort)2); // 26 - r0 = r0 ^ r5; // 27 - r0 = r0 ^ r4; // 28 - r3 = r3 - r0; // 29 - r5 = r5 * r1; // 30 - r7 = r7 ^ dataset[r2 & MASK]; // 31 - r1 = r1 ^ dataset[r0 & MASK]; // 32 - r5 = r5 ^ r6; // 33 - r5 = r5 ^ dataset[r1 & MASK]; // 34 - r0 = mulhi(r0, r5); // 35 - r5 = r5 ^ simd_shuffle_xor(r2, (ushort)4); // 36 - r7 = r7 ^ dataset[r0 & MASK]; // 37 - r3 = r3 + r1 + select(0x75ba2fadu, 0x230c005cu, ((sel >> 27u) & 1u) != 0u); // 38 - r1 = r1 ^ simd_shuffle_xor(r5, (ushort)4); // 39 - r2 = r2 ^ r5; // 40 - r3 = r6 * r3 + r3; // 41 - r6 = r6 - r7; // 42 - r7 = r7 ^ r0; // 43 - r1 = r1 ^ dataset[r7 & MASK]; // 44 - r2 = r2 * r3; // 45 - r1 = mulhi(r1, r5); // 46 - r4 = r4 - r3; // 47 - r2 = rotr_var(r2, r6); // 48 - r3 = r3 ^ dataset[r5 & MASK]; // 49 - r1 = r1 + r5 + select(0x81b8bc2cu, 0x1907970cu, ((sel >> 7u) & 1u) != 0u); // 50 - r0 = r0 * r2; // 51 - r0 = r0 + r2 + select(0x4f92b968u, 0x699fd448u, ((sel >> 6u) & 1u) != 0u); // 52 - r1 = r1 + r0 + select(0x2bb965afu, 0x77b1520du, ((sel >> 12u) & 1u) != 0u); // 53 - r7 = rotl_imm(r7, 14u); // 54 - r3 = r3 + r7 + select(0x7b0fe07au, 0xa54c55a0u, ((sel >> 1u) & 1u) != 0u); // 55 - r6 = r6 ^ dataset[r7 & MASK]; // 56 - r1 = rotr_var(r1, r5); // 57 - r5 = r5 ^ dataset[r4 & MASK]; // 58 - r6 = r6 ^ dataset[r2 & MASK]; // 59 - r3 = r5 * r0 + r3; // 60 - r5 = r5 + r7 + select(0xaf9dd72du, 0xad7493e7u, ((sel >> 31u) & 1u) != 0u); // 61 - r4 = r4 + r6 + select(0x89841d87u, 0x1e07c3d9u, ((sel >> 27u) & 1u) != 0u); // 62 - r5 = rotl_imm(r5, 19u); // 63 - // latency-shadow block (Counter ASIC 3.0 item 8): 256 ALU instructions x 27 passes after instruction 63, no load - for (uint sh = 0u; sh < 27u; ++sh) { - r5 = r5 + r2 + select(0x92199f99u, 0x8bc12da9u, ((sel >> 18u) & 1u) != 0u); // s0 add - r0 = r0 + r7 + select(0x8d72d3adu, 0x63079e5au, ((sel >> 26u) & 1u) != 0u); // s1 add - r6 = r6 ^ simd_shuffle_xor(r3, (ushort)2); // s2 shfl - r4 = r4 - r2; // s3 sub - r7 = r7 + r0 + select(0xb21b4babu, 0x5d4c7a60u, ((sel >> 31u) & 1u) != 0u); // s4 add - r0 = rotl_imm(r0, 11u); // s5 rotl - r0 = r0 + r1 + select(0x2fe0e98bu, 0xc88e2942u, ((sel >> 16u) & 1u) != 0u); // s6 add - r7 = r7 ^ r1; // s7 xor - r1 = r6 * r5 + r1; // s8 mad - r6 = r6 ^ simd_shuffle_xor(r1, (ushort)4); // s9 shfl - r1 = r2 * r2 + r1; // s10 mad - r5 = r0 * r3 + r5; // s11 mad - r2 = r2 ^ simd_shuffle_xor(r6, (ushort)4); // s12 shfl - r1 = r1 + r5 + select(0xbdf8f9a5u, 0xbc48c63eu, ((sel >> 25u) & 1u) != 0u); // s13 add - r1 = rotl_imm(r1, 29u); // s14 rotl - r1 = r1 - r4; // s15 sub - r7 = r7 | r1; // s16 or - r2 = r2 ^ simd_shuffle_xor(r4, (ushort)8); // s17 shfl - r7 = r7 ^ r4; // s18 xor - r6 = r6 * r1; // s19 mul - r5 = r6 * r0 + r5; // s20 mad - r3 = r3 - r1; // s21 sub - r6 = r6 * r0; // s22 mul - r2 = r2 + r0 + select(0x45c37cecu, 0x96e8f127u, ((sel >> 1u) & 1u) != 0u); // s23 add - r6 = r6 - r4; // s24 sub - r7 = r3 * r4 + r7; // s25 mad - r3 = rotl_imm(r3, 9u); // s26 rotl - r2 = r2 - r1; // s27 sub - r6 = mulhi(r6, r0); // s28 mulhi - r2 = r2 ^ simd_shuffle_xor(r4, (ushort)2); // s29 shfl - r1 = r1 ^ simd_shuffle_xor(r6, (ushort)1); // s30 shfl - r1 = r1 + r6 + select(0x37985632u, 0xb1cdb2abu, ((sel >> 1u) & 1u) != 0u); // s31 add - r0 = rotr_var(r0, r1); // s32 rotr - r3 = r3 ^ simd_shuffle_xor(r4, (ushort)2); // s33 shfl - r0 = r0 ^ simd_shuffle_xor(r3, (ushort)4); // s34 shfl - r7 = r7 * r0; // s35 mul - r3 = r3 + r1 + select(0x804e777eu, 0x856e0180u, ((sel >> 0u) & 1u) != 0u); // s36 add - r1 = r1 ^ r6; // s37 xor - r2 = r2 * r7; // s38 mul - r6 = r6 + r5 + select(0x0b06c8a7u, 0xe9bd0cc4u, ((sel >> 2u) & 1u) != 0u); // s39 add - r5 = r5 * r7; // s40 mul - r2 = mulhi(r2, r3); // s41 mulhi - r2 = r2 ^ r0; // s42 xor - r0 = rotl_imm(r0, 4u); // s43 rotl - r4 = mulhi(r4, r3); // s44 mulhi - r6 = r6 + r7 + select(0xee822e17u, 0xcdcb63f6u, ((sel >> 12u) & 1u) != 0u); // s45 add - r3 = r2 * r0 + r3; // s46 mad - r4 = r4 ^ simd_shuffle_xor(r3, (ushort)8); // s47 shfl - r6 = mulhi(r6, r5); // s48 mulhi - r0 = r0 - r2; // s49 sub - r3 = r3 - r5; // s50 sub - r1 = rotr_var(r1, r4); // s51 rotr - r6 = r7 * r7 + r6; // s52 mad - r5 = r5 ^ r3; // s53 xor - r1 = r1 - r0; // s54 sub - r5 = r5 - r6; // s55 sub - r3 = r3 + r2 + select(0x3dfad1b6u, 0xd4758987u, ((sel >> 10u) & 1u) != 0u); // s56 add - r4 = r4 + r1 + select(0xfca75bc2u, 0x0602d6beu, ((sel >> 0u) & 1u) != 0u); // s57 add - r5 = mulhi(r5, r6); // s58 mulhi - r2 = r2 ^ simd_shuffle_xor(r1, (ushort)2); // s59 shfl - r2 = rotl_imm(r2, 9u); // s60 rotl - r4 = r4 | r6; // s61 or - r6 = rotr_var(r6, r4); // s62 rotr - r2 = r2 * r4; // s63 mul - r0 = r0 ^ r5; // s64 xor - r2 = r2 ^ r7; // s65 xor - r2 = r2 + r1 + select(0xd71c02ffu, 0x8596687au, ((sel >> 6u) & 1u) != 0u); // s66 add - r1 = rotl_imm(r1, 21u); // s67 rotl - r3 = r3 * r2; // s68 mul - r7 = r7 ^ simd_shuffle_xor(r5, (ushort)2); // s69 shfl - r3 = r3 * r2; // s70 mul - r0 = r2 * r5 + r0; // s71 mad - r6 = r6 + r7 + select(0x3c6fe15du, 0x08ac5733u, ((sel >> 0u) & 1u) != 0u); // s72 add - r3 = r3 ^ simd_shuffle_xor(r2, (ushort)8); // s73 shfl - r3 = r3 * r6; // s74 mul - r6 = r6 ^ r7; // s75 xor - r3 = r3 | r0; // s76 or - r2 = r2 + r4 + select(0x295d5faeu, 0xc100b495u, ((sel >> 1u) & 1u) != 0u); // s77 add - r3 = mulhi(r3, r7); // s78 mulhi - r4 = r4 | r1; // s79 or - r4 = rotr_var(r4, r3); // s80 rotr - r4 = r4 + r3 + select(0x274a9221u, 0x5cc59530u, ((sel >> 15u) & 1u) != 0u); // s81 add - r7 = r7 + r3 + select(0xc8651f8eu, 0x141479ecu, ((sel >> 5u) & 1u) != 0u); // s82 add - r3 = rotr_var(r3, r6); // s83 rotr - r2 = r4 * r7 + r2; // s84 mad - r2 = r2 ^ simd_shuffle_xor(r5, (ushort)16); // s85 shfl - r3 = r3 ^ r2; // s86 xor - r5 = r5 ^ simd_shuffle_xor(r7, (ushort)2); // s87 shfl - r0 = r0 ^ r4; // s88 xor - r3 = r3 + r2 + select(0x53f915c2u, 0x883c0c92u, ((sel >> 18u) & 1u) != 0u); // s89 add - r7 = rotr_var(r7, r5); // s90 rotr - r3 = r3 + r1 + select(0xe2be00a5u, 0x3cc5bd20u, ((sel >> 31u) & 1u) != 0u); // s91 add - r7 = r7 ^ simd_shuffle_xor(r2, (ushort)1); // s92 shfl - r6 = rotr_var(r6, r2); // s93 rotr - r7 = rotl_imm(r7, 14u); // s94 rotl - r4 = r5 * r5 + r4; // s95 mad - r2 = r4 * r5 + r2; // s96 mad - r1 = r1 ^ r0; // s97 xor - r5 = r5 * r4; // s98 mul - r2 = r2 - r0; // s99 sub - r7 = rotl_imm(r7, 30u); // s100 rotl - r5 = r5 + r6 + select(0x423fd9c9u, 0xbfd646cbu, ((sel >> 17u) & 1u) != 0u); // s101 add - r7 = r7 + r6 + select(0x8d3c011du, 0x19b74a43u, ((sel >> 11u) & 1u) != 0u); // s102 add - r5 = rotl_imm(r5, 6u); // s103 rotl - r0 = r0 * r4; // s104 mul - r0 = r0 | r5; // s105 or - r0 = r0 + r1 + select(0x8ce14721u, 0x7dcb7e18u, ((sel >> 15u) & 1u) != 0u); // s106 add - r0 = rotl_imm(r0, 15u); // s107 rotl - r4 = r4 - r2; // s108 sub - r2 = mulhi(r2, r0); // s109 mulhi - r1 = mulhi(r1, r0); // s110 mulhi - r0 = r0 + r2 + select(0x2d9fc6b9u, 0x3c179ad8u, ((sel >> 28u) & 1u) != 0u); // s111 add - r2 = mulhi(r2, r0); // s112 mulhi - r6 = r6 - r3; // s113 sub - r7 = r6 * r3 + r7; // s114 mad - r5 = rotr_var(r5, r1); // s115 rotr - r6 = rotr_var(r6, r4); // s116 rotr - r2 = r2 + r1 + select(0x502138e2u, 0x47359729u, ((sel >> 22u) & 1u) != 0u); // s117 add - r3 = r2 * r0 + r3; // s118 mad - r1 = r1 * r3; // s119 mul - r2 = r0 * r4 + r2; // s120 mad - r0 = r0 * r3; // s121 mul - r2 = mulhi(r2, r0); // s122 mulhi - r5 = r5 * r6; // s123 mul - r4 = r2 * r4 + r4; // s124 mad - r4 = r4 ^ simd_shuffle_xor(r2, (ushort)2); // s125 shfl - r2 = r2 ^ r0; // s126 xor - r1 = rotl_imm(r1, 7u); // s127 rotl - r7 = r7 ^ simd_shuffle_xor(r2, (ushort)4); // s128 shfl - r6 = rotl_imm(r6, 17u); // s129 rotl - r2 = r2 + r5 + select(0x33dff776u, 0x6c57e4e7u, ((sel >> 15u) & 1u) != 0u); // s130 add - r0 = r0 | r3; // s131 or - r5 = rotr_var(r5, r0); // s132 rotr - r2 = r2 + r3 + select(0x5db25b34u, 0xe8212c0cu, ((sel >> 30u) & 1u) != 0u); // s133 add - r4 = r4 ^ r3; // s134 xor - r6 = r6 ^ r1; // s135 xor - r1 = r1 - r7; // s136 sub - r2 = r6 * r2 + r2; // s137 mad - r0 = mulhi(r0, r5); // s138 mulhi - r2 = r2 + r7 + select(0x218d4090u, 0xd44ff710u, ((sel >> 10u) & 1u) != 0u); // s139 add - r1 = rotr_var(r1, r4); // s140 rotr - r3 = r3 ^ simd_shuffle_xor(r6, (ushort)1); // s141 shfl - r7 = r6 * r2 + r7; // s142 mad - r2 = r2 * r3; // s143 mul - r7 = r7 + r4 + select(0xf4b1a8deu, 0xb98942fau, ((sel >> 29u) & 1u) != 0u); // s144 add - r7 = rotl_imm(r7, 15u); // s145 rotl - r7 = r7 ^ r5; // s146 xor - r4 = r7 * r4 + r4; // s147 mad - r6 = rotr_var(r6, r5); // s148 rotr - r1 = r2 * r4 + r1; // s149 mad - r1 = r1 + r7 + select(0x2bef10f2u, 0x0d48ba42u, ((sel >> 17u) & 1u) != 0u); // s150 add - r5 = r5 ^ r4; // s151 xor - r7 = r7 + r0 + select(0xdc5cc080u, 0xc98dea9cu, ((sel >> 30u) & 1u) != 0u); // s152 add - r4 = r4 * r6; // s153 mul - r0 = r0 * r2; // s154 mul - r6 = r6 ^ r5; // s155 xor - r4 = rotr_var(r4, r2); // s156 rotr - r1 = rotl_imm(r1, 11u); // s157 rotl - r5 = r5 + r0 + select(0x18197438u, 0x6c752dcbu, ((sel >> 11u) & 1u) != 0u); // s158 add - r4 = r1 * r5 + r4; // s159 mad - r4 = rotl_imm(r4, 26u); // s160 rotl - r3 = r0 * r7 + r3; // s161 mad - r3 = rotr_var(r3, r1); // s162 rotr - r4 = r4 + r0 + select(0xf1c46574u, 0x8e481727u, ((sel >> 3u) & 1u) != 0u); // s163 add - r0 = mulhi(r0, r4); // s164 mulhi - r2 = r2 + r6 + select(0xac578137u, 0x550ab406u, ((sel >> 20u) & 1u) != 0u); // s165 add - r0 = rotl_imm(r0, 13u); // s166 rotl - r3 = r3 + r1 + select(0xa33e6706u, 0xaebb5966u, ((sel >> 14u) & 1u) != 0u); // s167 add - r3 = r3 | r5; // s168 or - r6 = rotr_var(r6, r2); // s169 rotr - r4 = r4 ^ r6; // s170 xor - r6 = r6 - r1; // s171 sub - r7 = rotl_imm(r7, 22u); // s172 rotl - r5 = rotl_imm(r5, 15u); // s173 rotl - r7 = r7 ^ simd_shuffle_xor(r0, (ushort)8); // s174 shfl - r0 = r0 ^ r5; // s175 xor - r7 = rotl_imm(r7, 6u); // s176 rotl - r7 = r7 - r0; // s177 sub - r3 = rotl_imm(r3, 30u); // s178 rotl - r7 = r6 * r1 + r7; // s179 mad - r6 = rotl_imm(r6, 9u); // s180 rotl - r2 = r2 ^ r4; // s181 xor - r2 = r2 ^ r7; // s182 xor - r7 = r7 ^ r2; // s183 xor - r1 = r1 + r2 + select(0xd94d55acu, 0x5bb7550fu, ((sel >> 21u) & 1u) != 0u); // s184 add - r3 = r3 | r5; // s185 or - r6 = mulhi(r6, r3); // s186 mulhi - r4 = r4 | r0; // s187 or - r7 = r7 ^ simd_shuffle_xor(r1, (ushort)2); // s188 shfl - r6 = r6 + r5 + select(0x89e747feu, 0x2a354e2du, ((sel >> 6u) & 1u) != 0u); // s189 add - r1 = r1 ^ r4; // s190 xor - r7 = mulhi(r7, r4); // s191 mulhi - r2 = rotl_imm(r2, 26u); // s192 rotl - r5 = rotl_imm(r5, 8u); // s193 rotl - r4 = r4 ^ simd_shuffle_xor(r5, (ushort)16); // s194 shfl - r4 = r4 ^ r5; // s195 xor - r1 = r1 * r3; // s196 mul - r5 = r5 + r2 + select(0xea03e8e7u, 0x10cfdc71u, ((sel >> 7u) & 1u) != 0u); // s197 add - r7 = r7 + r5 + select(0xa7ee0102u, 0x66e148d3u, ((sel >> 15u) & 1u) != 0u); // s198 add - r5 = r5 - r2; // s199 sub - r5 = r5 ^ simd_shuffle_xor(r1, (ushort)2); // s200 shfl - r7 = r7 ^ simd_shuffle_xor(r1, (ushort)8); // s201 shfl - r4 = rotr_var(r4, r5); // s202 rotr - r7 = r7 ^ r5; // s203 xor - r7 = r7 ^ r0; // s204 xor - r6 = r6 + r2 + select(0x8379a4deu, 0x8558b619u, ((sel >> 10u) & 1u) != 0u); // s205 add - r4 = rotl_imm(r4, 13u); // s206 rotl - r1 = mulhi(r1, r3); // s207 mulhi - r1 = r4 * r4 + r1; // s208 mad - r2 = r2 ^ simd_shuffle_xor(r0, (ushort)4); // s209 shfl - r7 = r7 + r0 + select(0xaa8cb14eu, 0xf4049c4cu, ((sel >> 11u) & 1u) != 0u); // s210 add - r7 = r7 ^ simd_shuffle_xor(r1, (ushort)16); // s211 shfl - r1 = r1 ^ r0; // s212 xor - r7 = rotl_imm(r7, 3u); // s213 rotl - r4 = rotr_var(r4, r7); // s214 rotr - r3 = r3 ^ simd_shuffle_xor(r2, (ushort)16); // s215 shfl - r5 = r5 | r1; // s216 or - r1 = r1 - r2; // s217 sub - r6 = r6 - r5; // s218 sub - r6 = rotl_imm(r6, 4u); // s219 rotl - r2 = mulhi(r2, r0); // s220 mulhi - r2 = r2 | r0; // s221 or - r5 = r5 ^ simd_shuffle_xor(r2, (ushort)8); // s222 shfl - r5 = mulhi(r5, r6); // s223 mulhi - r0 = r0 - r6; // s224 sub - r7 = rotl_imm(r7, 23u); // s225 rotl - r4 = r4 | r2; // s226 or - r2 = r2 * r4; // s227 mul - r3 = rotl_imm(r3, 12u); // s228 rotl - r0 = rotr_var(r0, r4); // s229 rotr - r0 = r0 + r6 + select(0x1d176220u, 0x2a7fecb2u, ((sel >> 16u) & 1u) != 0u); // s230 add - r1 = r1 ^ simd_shuffle_xor(r2, (ushort)4); // s231 shfl - r2 = r2 * r1; // s232 mul - r7 = r0 * r1 + r7; // s233 mad - r5 = rotl_imm(r5, 22u); // s234 rotl - r4 = rotr_var(r4, r6); // s235 rotr - r0 = r5 * r1 + r0; // s236 mad - r6 = r6 ^ r4; // s237 xor - r4 = r4 ^ r6; // s238 xor - r6 = rotl_imm(r6, 18u); // s239 rotl - r4 = r4 + r7 + select(0x17dafb4du, 0xadce39f3u, ((sel >> 25u) & 1u) != 0u); // s240 add - r0 = r0 - r7; // s241 sub - r1 = rotr_var(r1, r6); // s242 rotr - r3 = r3 + r0 + select(0xf2f77d26u, 0x0e7033b6u, ((sel >> 29u) & 1u) != 0u); // s243 add - r2 = r7 * r4 + r2; // s244 mad - r4 = r0 * r6 + r4; // s245 mad - r5 = r5 * r7; // s246 mul - r3 = r3 + r5 + select(0xfed2da4eu, 0x7b2ff6b7u, ((sel >> 31u) & 1u) != 0u); // s247 add - r0 = r0 + r7 + select(0x03b2891cu, 0xb5fad7b1u, ((sel >> 0u) & 1u) != 0u); // s248 add - r5 = mulhi(r5, r4); // s249 mulhi - r5 = r5 + r2 + select(0x042cd6e3u, 0xa7e71c2fu, ((sel >> 11u) & 1u) != 0u); // s250 add - r6 = r6 ^ simd_shuffle_xor(r1, (ushort)1); // s251 shfl - r6 = r6 ^ r1; // s252 xor - r2 = r2 * r5; // s253 mul - r0 = r0 + r6 + select(0x784a302bu, 0xb83d78deu, ((sel >> 16u) & 1u) != 0u); // s254 add - r2 = r2 - r4; // s255 sub - } - } - uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((ulong)hi << 32) | (ulong)lo; -} diff --git a/proto-cuda/packs-ca4/mx8_sh256x27/program_bound.metal b/proto-cuda/packs-ca4/mx8_sh256x27/program_bound.metal deleted file mode 100644 index 7a8da2d3e..000000000 --- a/proto-cuda/packs-ca4/mx8_sh256x27/program_bound.metal +++ /dev/null @@ -1,370 +0,0 @@ -#include -using namespace metal; - -#define MASK 0x0fffffffu -constant uint SEEDW[8] = { 0x67a9a7beu, 0x1a155b25u, 0xfddfb732u, 0x4b5af2e8u, 0xc55caf33u, 0xa27c13b7u, 0x06628a48u, 0x03852469u }; - -inline uint splitmix32(uint x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -inline uint rotl_imm(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 -inline uint rotr_var(uint x, uint n) { n &= 31u; return (x >> n) | (x << ((32u - n) & 31u)); } -inline uint ds_elem(uint i, uint d0, uint d1) { - uint x = i ^ d0; - x *= 0x9E3779B1u; x ^= x >> 15; - x += d1; - x *= 0x85EBCA77u; x ^= x >> 13; - x *= 0xC2B2AE3Du; x ^= x >> 16; - return x; -} - -// Header-bound variant: the init words come from buffer 3 (bind.rs), not from SEEDW. -kernel void igneum_hash_bound(device const uint* dataset [[buffer(0)]], - device ulong* out [[buffer(1)]], - constant uint& baseNonce [[buffer(2)]], - constant uint* initw [[buffer(3)]], - uint gid [[thread_position_in_grid]]) { - uint nonce = baseNonce + gid; - uint r0, r1, r2, r3, r4, r5, r6, r7; - { uint x = nonce ^ initw[0]; x += 0x9e3779b9u * 1u; x = splitmix32(x); r0 = x ^ initw[1]; } - { uint x = nonce ^ initw[1]; x += 0x9e3779b9u * 2u; x = splitmix32(x); r1 = x ^ initw[2]; } - { uint x = nonce ^ initw[2]; x += 0x9e3779b9u * 3u; x = splitmix32(x); r2 = x ^ initw[3]; } - { uint x = nonce ^ initw[3]; x += 0x9e3779b9u * 4u; x = splitmix32(x); r3 = x ^ initw[4]; } - { uint x = nonce ^ initw[4]; x += 0x9e3779b9u * 5u; x = splitmix32(x); r4 = x ^ initw[5]; } - { uint x = nonce ^ initw[5]; x += 0x9e3779b9u * 6u; x = splitmix32(x); r5 = x ^ initw[6]; } - { uint x = nonce ^ initw[6]; x += 0x9e3779b9u * 7u; x = splitmix32(x); r6 = x ^ initw[7]; } - { uint x = nonce ^ initw[7]; x += 0x9e3779b9u * 8u; x = splitmix32(x); r7 = x ^ initw[0]; } - - for (uint it = 0u; it < 8u; ++it) { - uint sel = r0; - r2 = r3 * r4 + r2; // 0 - r2 = r1 * r1 + r2; // 1 - r2 = r3 * r2 + r2; // 2 - r3 = r3 ^ r5; // 3 - r7 = r7 ^ dataset[r2 & MASK]; // 4 - r5 = r5 ^ dataset[r7 & MASK]; // 5 - r1 = r1 ^ simd_shuffle_xor(r4, (ushort)8); // 6 - r7 = r7 ^ simd_shuffle_xor(r3, (ushort)8); // 7 - r1 = mulhi(r1, r5); // 8 - r6 = rotr_var(r6, r3); // 9 - r3 = r3 | r4; // 10 - r4 = r4 ^ dataset[r3 & MASK]; // 11 - r0 = mulhi(r0, r4); // 12 - r5 = r5 + r1 + select(0xc7934706u, 0xd3177981u, ((sel >> 30u) & 1u) != 0u); // 13 - r0 = r0 ^ dataset[r4 & MASK]; // 14 - r2 = r2 - r4; // 15 - r2 = r2 ^ dataset[r0 & MASK]; // 16 - r7 = r7 ^ dataset[r2 & MASK]; // 17 - r7 = r7 ^ simd_shuffle_xor(r3, (ushort)4); // 18 - r5 = r5 * r0; // 19 - r3 = r3 ^ simd_shuffle_xor(r4, (ushort)2); // 20 - r2 = r2 ^ simd_shuffle_xor(r4, (ushort)16); // 21 - r6 = mulhi(r6, r2); // 22 - r6 = r6 ^ dataset[r1 & MASK]; // 23 - r5 = r5 * r0; // 24 - r5 = rotl_imm(r5, 19u); // 25 - r7 = r7 ^ simd_shuffle_xor(r6, (ushort)2); // 26 - r0 = r0 ^ r5; // 27 - r0 = r0 ^ r4; // 28 - r3 = r3 - r0; // 29 - r5 = r5 * r1; // 30 - r7 = r7 ^ dataset[r2 & MASK]; // 31 - r1 = r1 ^ dataset[r0 & MASK]; // 32 - r5 = r5 ^ r6; // 33 - r5 = r5 ^ dataset[r1 & MASK]; // 34 - r0 = mulhi(r0, r5); // 35 - r5 = r5 ^ simd_shuffle_xor(r2, (ushort)4); // 36 - r7 = r7 ^ dataset[r0 & MASK]; // 37 - r3 = r3 + r1 + select(0x75ba2fadu, 0x230c005cu, ((sel >> 27u) & 1u) != 0u); // 38 - r1 = r1 ^ simd_shuffle_xor(r5, (ushort)4); // 39 - r2 = r2 ^ r5; // 40 - r3 = r6 * r3 + r3; // 41 - r6 = r6 - r7; // 42 - r7 = r7 ^ r0; // 43 - r1 = r1 ^ dataset[r7 & MASK]; // 44 - r2 = r2 * r3; // 45 - r1 = mulhi(r1, r5); // 46 - r4 = r4 - r3; // 47 - r2 = rotr_var(r2, r6); // 48 - r3 = r3 ^ dataset[r5 & MASK]; // 49 - r1 = r1 + r5 + select(0x81b8bc2cu, 0x1907970cu, ((sel >> 7u) & 1u) != 0u); // 50 - r0 = r0 * r2; // 51 - r0 = r0 + r2 + select(0x4f92b968u, 0x699fd448u, ((sel >> 6u) & 1u) != 0u); // 52 - r1 = r1 + r0 + select(0x2bb965afu, 0x77b1520du, ((sel >> 12u) & 1u) != 0u); // 53 - r7 = rotl_imm(r7, 14u); // 54 - r3 = r3 + r7 + select(0x7b0fe07au, 0xa54c55a0u, ((sel >> 1u) & 1u) != 0u); // 55 - r6 = r6 ^ dataset[r7 & MASK]; // 56 - r1 = rotr_var(r1, r5); // 57 - r5 = r5 ^ dataset[r4 & MASK]; // 58 - r6 = r6 ^ dataset[r2 & MASK]; // 59 - r3 = r5 * r0 + r3; // 60 - r5 = r5 + r7 + select(0xaf9dd72du, 0xad7493e7u, ((sel >> 31u) & 1u) != 0u); // 61 - r4 = r4 + r6 + select(0x89841d87u, 0x1e07c3d9u, ((sel >> 27u) & 1u) != 0u); // 62 - r5 = rotl_imm(r5, 19u); // 63 - // latency-shadow block (Counter ASIC 3.0 item 8): 256 ALU instructions x 27 passes after instruction 63, no load - for (uint sh = 0u; sh < 27u; ++sh) { - r5 = r5 + r2 + select(0x92199f99u, 0x8bc12da9u, ((sel >> 18u) & 1u) != 0u); // s0 add - r0 = r0 + r7 + select(0x8d72d3adu, 0x63079e5au, ((sel >> 26u) & 1u) != 0u); // s1 add - r6 = r6 ^ simd_shuffle_xor(r3, (ushort)2); // s2 shfl - r4 = r4 - r2; // s3 sub - r7 = r7 + r0 + select(0xb21b4babu, 0x5d4c7a60u, ((sel >> 31u) & 1u) != 0u); // s4 add - r0 = rotl_imm(r0, 11u); // s5 rotl - r0 = r0 + r1 + select(0x2fe0e98bu, 0xc88e2942u, ((sel >> 16u) & 1u) != 0u); // s6 add - r7 = r7 ^ r1; // s7 xor - r1 = r6 * r5 + r1; // s8 mad - r6 = r6 ^ simd_shuffle_xor(r1, (ushort)4); // s9 shfl - r1 = r2 * r2 + r1; // s10 mad - r5 = r0 * r3 + r5; // s11 mad - r2 = r2 ^ simd_shuffle_xor(r6, (ushort)4); // s12 shfl - r1 = r1 + r5 + select(0xbdf8f9a5u, 0xbc48c63eu, ((sel >> 25u) & 1u) != 0u); // s13 add - r1 = rotl_imm(r1, 29u); // s14 rotl - r1 = r1 - r4; // s15 sub - r7 = r7 | r1; // s16 or - r2 = r2 ^ simd_shuffle_xor(r4, (ushort)8); // s17 shfl - r7 = r7 ^ r4; // s18 xor - r6 = r6 * r1; // s19 mul - r5 = r6 * r0 + r5; // s20 mad - r3 = r3 - r1; // s21 sub - r6 = r6 * r0; // s22 mul - r2 = r2 + r0 + select(0x45c37cecu, 0x96e8f127u, ((sel >> 1u) & 1u) != 0u); // s23 add - r6 = r6 - r4; // s24 sub - r7 = r3 * r4 + r7; // s25 mad - r3 = rotl_imm(r3, 9u); // s26 rotl - r2 = r2 - r1; // s27 sub - r6 = mulhi(r6, r0); // s28 mulhi - r2 = r2 ^ simd_shuffle_xor(r4, (ushort)2); // s29 shfl - r1 = r1 ^ simd_shuffle_xor(r6, (ushort)1); // s30 shfl - r1 = r1 + r6 + select(0x37985632u, 0xb1cdb2abu, ((sel >> 1u) & 1u) != 0u); // s31 add - r0 = rotr_var(r0, r1); // s32 rotr - r3 = r3 ^ simd_shuffle_xor(r4, (ushort)2); // s33 shfl - r0 = r0 ^ simd_shuffle_xor(r3, (ushort)4); // s34 shfl - r7 = r7 * r0; // s35 mul - r3 = r3 + r1 + select(0x804e777eu, 0x856e0180u, ((sel >> 0u) & 1u) != 0u); // s36 add - r1 = r1 ^ r6; // s37 xor - r2 = r2 * r7; // s38 mul - r6 = r6 + r5 + select(0x0b06c8a7u, 0xe9bd0cc4u, ((sel >> 2u) & 1u) != 0u); // s39 add - r5 = r5 * r7; // s40 mul - r2 = mulhi(r2, r3); // s41 mulhi - r2 = r2 ^ r0; // s42 xor - r0 = rotl_imm(r0, 4u); // s43 rotl - r4 = mulhi(r4, r3); // s44 mulhi - r6 = r6 + r7 + select(0xee822e17u, 0xcdcb63f6u, ((sel >> 12u) & 1u) != 0u); // s45 add - r3 = r2 * r0 + r3; // s46 mad - r4 = r4 ^ simd_shuffle_xor(r3, (ushort)8); // s47 shfl - r6 = mulhi(r6, r5); // s48 mulhi - r0 = r0 - r2; // s49 sub - r3 = r3 - r5; // s50 sub - r1 = rotr_var(r1, r4); // s51 rotr - r6 = r7 * r7 + r6; // s52 mad - r5 = r5 ^ r3; // s53 xor - r1 = r1 - r0; // s54 sub - r5 = r5 - r6; // s55 sub - r3 = r3 + r2 + select(0x3dfad1b6u, 0xd4758987u, ((sel >> 10u) & 1u) != 0u); // s56 add - r4 = r4 + r1 + select(0xfca75bc2u, 0x0602d6beu, ((sel >> 0u) & 1u) != 0u); // s57 add - r5 = mulhi(r5, r6); // s58 mulhi - r2 = r2 ^ simd_shuffle_xor(r1, (ushort)2); // s59 shfl - r2 = rotl_imm(r2, 9u); // s60 rotl - r4 = r4 | r6; // s61 or - r6 = rotr_var(r6, r4); // s62 rotr - r2 = r2 * r4; // s63 mul - r0 = r0 ^ r5; // s64 xor - r2 = r2 ^ r7; // s65 xor - r2 = r2 + r1 + select(0xd71c02ffu, 0x8596687au, ((sel >> 6u) & 1u) != 0u); // s66 add - r1 = rotl_imm(r1, 21u); // s67 rotl - r3 = r3 * r2; // s68 mul - r7 = r7 ^ simd_shuffle_xor(r5, (ushort)2); // s69 shfl - r3 = r3 * r2; // s70 mul - r0 = r2 * r5 + r0; // s71 mad - r6 = r6 + r7 + select(0x3c6fe15du, 0x08ac5733u, ((sel >> 0u) & 1u) != 0u); // s72 add - r3 = r3 ^ simd_shuffle_xor(r2, (ushort)8); // s73 shfl - r3 = r3 * r6; // s74 mul - r6 = r6 ^ r7; // s75 xor - r3 = r3 | r0; // s76 or - r2 = r2 + r4 + select(0x295d5faeu, 0xc100b495u, ((sel >> 1u) & 1u) != 0u); // s77 add - r3 = mulhi(r3, r7); // s78 mulhi - r4 = r4 | r1; // s79 or - r4 = rotr_var(r4, r3); // s80 rotr - r4 = r4 + r3 + select(0x274a9221u, 0x5cc59530u, ((sel >> 15u) & 1u) != 0u); // s81 add - r7 = r7 + r3 + select(0xc8651f8eu, 0x141479ecu, ((sel >> 5u) & 1u) != 0u); // s82 add - r3 = rotr_var(r3, r6); // s83 rotr - r2 = r4 * r7 + r2; // s84 mad - r2 = r2 ^ simd_shuffle_xor(r5, (ushort)16); // s85 shfl - r3 = r3 ^ r2; // s86 xor - r5 = r5 ^ simd_shuffle_xor(r7, (ushort)2); // s87 shfl - r0 = r0 ^ r4; // s88 xor - r3 = r3 + r2 + select(0x53f915c2u, 0x883c0c92u, ((sel >> 18u) & 1u) != 0u); // s89 add - r7 = rotr_var(r7, r5); // s90 rotr - r3 = r3 + r1 + select(0xe2be00a5u, 0x3cc5bd20u, ((sel >> 31u) & 1u) != 0u); // s91 add - r7 = r7 ^ simd_shuffle_xor(r2, (ushort)1); // s92 shfl - r6 = rotr_var(r6, r2); // s93 rotr - r7 = rotl_imm(r7, 14u); // s94 rotl - r4 = r5 * r5 + r4; // s95 mad - r2 = r4 * r5 + r2; // s96 mad - r1 = r1 ^ r0; // s97 xor - r5 = r5 * r4; // s98 mul - r2 = r2 - r0; // s99 sub - r7 = rotl_imm(r7, 30u); // s100 rotl - r5 = r5 + r6 + select(0x423fd9c9u, 0xbfd646cbu, ((sel >> 17u) & 1u) != 0u); // s101 add - r7 = r7 + r6 + select(0x8d3c011du, 0x19b74a43u, ((sel >> 11u) & 1u) != 0u); // s102 add - r5 = rotl_imm(r5, 6u); // s103 rotl - r0 = r0 * r4; // s104 mul - r0 = r0 | r5; // s105 or - r0 = r0 + r1 + select(0x8ce14721u, 0x7dcb7e18u, ((sel >> 15u) & 1u) != 0u); // s106 add - r0 = rotl_imm(r0, 15u); // s107 rotl - r4 = r4 - r2; // s108 sub - r2 = mulhi(r2, r0); // s109 mulhi - r1 = mulhi(r1, r0); // s110 mulhi - r0 = r0 + r2 + select(0x2d9fc6b9u, 0x3c179ad8u, ((sel >> 28u) & 1u) != 0u); // s111 add - r2 = mulhi(r2, r0); // s112 mulhi - r6 = r6 - r3; // s113 sub - r7 = r6 * r3 + r7; // s114 mad - r5 = rotr_var(r5, r1); // s115 rotr - r6 = rotr_var(r6, r4); // s116 rotr - r2 = r2 + r1 + select(0x502138e2u, 0x47359729u, ((sel >> 22u) & 1u) != 0u); // s117 add - r3 = r2 * r0 + r3; // s118 mad - r1 = r1 * r3; // s119 mul - r2 = r0 * r4 + r2; // s120 mad - r0 = r0 * r3; // s121 mul - r2 = mulhi(r2, r0); // s122 mulhi - r5 = r5 * r6; // s123 mul - r4 = r2 * r4 + r4; // s124 mad - r4 = r4 ^ simd_shuffle_xor(r2, (ushort)2); // s125 shfl - r2 = r2 ^ r0; // s126 xor - r1 = rotl_imm(r1, 7u); // s127 rotl - r7 = r7 ^ simd_shuffle_xor(r2, (ushort)4); // s128 shfl - r6 = rotl_imm(r6, 17u); // s129 rotl - r2 = r2 + r5 + select(0x33dff776u, 0x6c57e4e7u, ((sel >> 15u) & 1u) != 0u); // s130 add - r0 = r0 | r3; // s131 or - r5 = rotr_var(r5, r0); // s132 rotr - r2 = r2 + r3 + select(0x5db25b34u, 0xe8212c0cu, ((sel >> 30u) & 1u) != 0u); // s133 add - r4 = r4 ^ r3; // s134 xor - r6 = r6 ^ r1; // s135 xor - r1 = r1 - r7; // s136 sub - r2 = r6 * r2 + r2; // s137 mad - r0 = mulhi(r0, r5); // s138 mulhi - r2 = r2 + r7 + select(0x218d4090u, 0xd44ff710u, ((sel >> 10u) & 1u) != 0u); // s139 add - r1 = rotr_var(r1, r4); // s140 rotr - r3 = r3 ^ simd_shuffle_xor(r6, (ushort)1); // s141 shfl - r7 = r6 * r2 + r7; // s142 mad - r2 = r2 * r3; // s143 mul - r7 = r7 + r4 + select(0xf4b1a8deu, 0xb98942fau, ((sel >> 29u) & 1u) != 0u); // s144 add - r7 = rotl_imm(r7, 15u); // s145 rotl - r7 = r7 ^ r5; // s146 xor - r4 = r7 * r4 + r4; // s147 mad - r6 = rotr_var(r6, r5); // s148 rotr - r1 = r2 * r4 + r1; // s149 mad - r1 = r1 + r7 + select(0x2bef10f2u, 0x0d48ba42u, ((sel >> 17u) & 1u) != 0u); // s150 add - r5 = r5 ^ r4; // s151 xor - r7 = r7 + r0 + select(0xdc5cc080u, 0xc98dea9cu, ((sel >> 30u) & 1u) != 0u); // s152 add - r4 = r4 * r6; // s153 mul - r0 = r0 * r2; // s154 mul - r6 = r6 ^ r5; // s155 xor - r4 = rotr_var(r4, r2); // s156 rotr - r1 = rotl_imm(r1, 11u); // s157 rotl - r5 = r5 + r0 + select(0x18197438u, 0x6c752dcbu, ((sel >> 11u) & 1u) != 0u); // s158 add - r4 = r1 * r5 + r4; // s159 mad - r4 = rotl_imm(r4, 26u); // s160 rotl - r3 = r0 * r7 + r3; // s161 mad - r3 = rotr_var(r3, r1); // s162 rotr - r4 = r4 + r0 + select(0xf1c46574u, 0x8e481727u, ((sel >> 3u) & 1u) != 0u); // s163 add - r0 = mulhi(r0, r4); // s164 mulhi - r2 = r2 + r6 + select(0xac578137u, 0x550ab406u, ((sel >> 20u) & 1u) != 0u); // s165 add - r0 = rotl_imm(r0, 13u); // s166 rotl - r3 = r3 + r1 + select(0xa33e6706u, 0xaebb5966u, ((sel >> 14u) & 1u) != 0u); // s167 add - r3 = r3 | r5; // s168 or - r6 = rotr_var(r6, r2); // s169 rotr - r4 = r4 ^ r6; // s170 xor - r6 = r6 - r1; // s171 sub - r7 = rotl_imm(r7, 22u); // s172 rotl - r5 = rotl_imm(r5, 15u); // s173 rotl - r7 = r7 ^ simd_shuffle_xor(r0, (ushort)8); // s174 shfl - r0 = r0 ^ r5; // s175 xor - r7 = rotl_imm(r7, 6u); // s176 rotl - r7 = r7 - r0; // s177 sub - r3 = rotl_imm(r3, 30u); // s178 rotl - r7 = r6 * r1 + r7; // s179 mad - r6 = rotl_imm(r6, 9u); // s180 rotl - r2 = r2 ^ r4; // s181 xor - r2 = r2 ^ r7; // s182 xor - r7 = r7 ^ r2; // s183 xor - r1 = r1 + r2 + select(0xd94d55acu, 0x5bb7550fu, ((sel >> 21u) & 1u) != 0u); // s184 add - r3 = r3 | r5; // s185 or - r6 = mulhi(r6, r3); // s186 mulhi - r4 = r4 | r0; // s187 or - r7 = r7 ^ simd_shuffle_xor(r1, (ushort)2); // s188 shfl - r6 = r6 + r5 + select(0x89e747feu, 0x2a354e2du, ((sel >> 6u) & 1u) != 0u); // s189 add - r1 = r1 ^ r4; // s190 xor - r7 = mulhi(r7, r4); // s191 mulhi - r2 = rotl_imm(r2, 26u); // s192 rotl - r5 = rotl_imm(r5, 8u); // s193 rotl - r4 = r4 ^ simd_shuffle_xor(r5, (ushort)16); // s194 shfl - r4 = r4 ^ r5; // s195 xor - r1 = r1 * r3; // s196 mul - r5 = r5 + r2 + select(0xea03e8e7u, 0x10cfdc71u, ((sel >> 7u) & 1u) != 0u); // s197 add - r7 = r7 + r5 + select(0xa7ee0102u, 0x66e148d3u, ((sel >> 15u) & 1u) != 0u); // s198 add - r5 = r5 - r2; // s199 sub - r5 = r5 ^ simd_shuffle_xor(r1, (ushort)2); // s200 shfl - r7 = r7 ^ simd_shuffle_xor(r1, (ushort)8); // s201 shfl - r4 = rotr_var(r4, r5); // s202 rotr - r7 = r7 ^ r5; // s203 xor - r7 = r7 ^ r0; // s204 xor - r6 = r6 + r2 + select(0x8379a4deu, 0x8558b619u, ((sel >> 10u) & 1u) != 0u); // s205 add - r4 = rotl_imm(r4, 13u); // s206 rotl - r1 = mulhi(r1, r3); // s207 mulhi - r1 = r4 * r4 + r1; // s208 mad - r2 = r2 ^ simd_shuffle_xor(r0, (ushort)4); // s209 shfl - r7 = r7 + r0 + select(0xaa8cb14eu, 0xf4049c4cu, ((sel >> 11u) & 1u) != 0u); // s210 add - r7 = r7 ^ simd_shuffle_xor(r1, (ushort)16); // s211 shfl - r1 = r1 ^ r0; // s212 xor - r7 = rotl_imm(r7, 3u); // s213 rotl - r4 = rotr_var(r4, r7); // s214 rotr - r3 = r3 ^ simd_shuffle_xor(r2, (ushort)16); // s215 shfl - r5 = r5 | r1; // s216 or - r1 = r1 - r2; // s217 sub - r6 = r6 - r5; // s218 sub - r6 = rotl_imm(r6, 4u); // s219 rotl - r2 = mulhi(r2, r0); // s220 mulhi - r2 = r2 | r0; // s221 or - r5 = r5 ^ simd_shuffle_xor(r2, (ushort)8); // s222 shfl - r5 = mulhi(r5, r6); // s223 mulhi - r0 = r0 - r6; // s224 sub - r7 = rotl_imm(r7, 23u); // s225 rotl - r4 = r4 | r2; // s226 or - r2 = r2 * r4; // s227 mul - r3 = rotl_imm(r3, 12u); // s228 rotl - r0 = rotr_var(r0, r4); // s229 rotr - r0 = r0 + r6 + select(0x1d176220u, 0x2a7fecb2u, ((sel >> 16u) & 1u) != 0u); // s230 add - r1 = r1 ^ simd_shuffle_xor(r2, (ushort)4); // s231 shfl - r2 = r2 * r1; // s232 mul - r7 = r0 * r1 + r7; // s233 mad - r5 = rotl_imm(r5, 22u); // s234 rotl - r4 = rotr_var(r4, r6); // s235 rotr - r0 = r5 * r1 + r0; // s236 mad - r6 = r6 ^ r4; // s237 xor - r4 = r4 ^ r6; // s238 xor - r6 = rotl_imm(r6, 18u); // s239 rotl - r4 = r4 + r7 + select(0x17dafb4du, 0xadce39f3u, ((sel >> 25u) & 1u) != 0u); // s240 add - r0 = r0 - r7; // s241 sub - r1 = rotr_var(r1, r6); // s242 rotr - r3 = r3 + r0 + select(0xf2f77d26u, 0x0e7033b6u, ((sel >> 29u) & 1u) != 0u); // s243 add - r2 = r7 * r4 + r2; // s244 mad - r4 = r0 * r6 + r4; // s245 mad - r5 = r5 * r7; // s246 mul - r3 = r3 + r5 + select(0xfed2da4eu, 0x7b2ff6b7u, ((sel >> 31u) & 1u) != 0u); // s247 add - r0 = r0 + r7 + select(0x03b2891cu, 0xb5fad7b1u, ((sel >> 0u) & 1u) != 0u); // s248 add - r5 = mulhi(r5, r4); // s249 mulhi - r5 = r5 + r2 + select(0x042cd6e3u, 0xa7e71c2fu, ((sel >> 11u) & 1u) != 0u); // s250 add - r6 = r6 ^ simd_shuffle_xor(r1, (ushort)1); // s251 shfl - r6 = r6 ^ r1; // s252 xor - r2 = r2 * r5; // s253 mul - r0 = r0 + r6 + select(0x784a302bu, 0xb83d78deu, ((sel >> 16u) & 1u) != 0u); // s254 add - r2 = r2 - r4; // s255 sub - } - } - uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((ulong)hi << 32) | (ulong)lo; -} diff --git a/proto-cuda/packs-ca4/mx8_sh256x27/vectors.h b/proto-cuda/packs-ca4/mx8_sh256x27/vectors.h deleted file mode 100644 index 17ce44606..000000000 --- a/proto-cuda/packs-ca4/mx8_sh256x27/vectors.h +++ /dev/null @@ -1,57 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// Expected outputs: igneum-pow (Rust) CPU interpreter, generator v2, memory-hard dataset -#pragma once -#ifdef __cplusplus -#include -#else -#include -#endif - -#define IGNEUM_VEC_WARPS 3 -static const uint32_t IGNEUM_VEC_BASE[IGNEUM_VEC_WARPS] = { 0u, 4096u, 1000000u }; -static const uint64_t IGNEUM_VEC_OUT[IGNEUM_VEC_WARPS][32] = { - { // base nonce 0 - 0x2576769ee4a14c8dull, 0x6408f60b1f606a96ull, 0x8e1825542dbd3636ull, 0xb25c17d8820ae079ull, 0xb6e1a578d06323b2ull, 0x8bea2965eaa47f55ull, 0x3684da53df965532ull, 0x3392668d50a45aadull, - 0x5e78d14aaaadce6eull, 0xcb0a7992844c9192ull, 0xfe4ad7fbc328f9efull, 0x7e64515d7d5941a4ull, 0x425e4eedb7b0ca4bull, 0x4f57a8f93ce317b8ull, 0x081360e3cf765b3aull, 0x1e9e919c50331254ull, - 0x998e7a76718adcbaull, 0xde5600e2b1838d9full, 0x4f85d0539779a267ull, 0x40661123ffbce40cull, 0x95b247ca86ff61dcull, 0xbe2bf61b9fcd1362ull, 0xfd6d6687a07997daull, 0x9935965e43f25045ull, - 0x27fe1552873c0b40ull, 0x972c952c83cf5ea9ull, 0x525b975fb9610333ull, 0x4f81431408d27e2dull, 0x6375d14f804f061full, 0xff6492be48775872ull, 0x81b2e098e1a01f8full, 0xc58ddcb717dd3370ull - }, - { // base nonce 4096 - 0x1ce77a600ec573b4ull, 0xb126a41f83521e45ull, 0x875547a82d6145fcull, 0xd4b04bf258ce4941ull, 0xe24c23e2b2371ad6ull, 0x7687a4beddbbd270ull, 0x63b9346ba234a0beull, 0x2500fb30f5e7eb20ull, - 0x7a185d59a30e3333ull, 0xa3c14cec7e2bcce2ull, 0x5730e39d367c6b21ull, 0x03e10c84ac832a3full, 0xf6dd88e0208019efull, 0xd539425d3bc497fcull, 0x71461f5d0c23f626ull, 0xa4c64db97fb4a425ull, - 0x61f56436538b61f8ull, 0x749b10eccde2f0bcull, 0xbe447d39b5f3aa29ull, 0x0e4ada60a90088b7ull, 0x3eda0e2db0bc9f9eull, 0xf90d81a21a10089cull, 0x7cf50a8f14d2430eull, 0x14601dea6a5d8625ull, - 0x23c448ee2f31a26dull, 0x60e5c3d4b218c0c2ull, 0x46ed8a924ac431fbull, 0xcc77440ffbe5c23dull, 0x9a15a2da9d6ba946ull, 0x37117ce37ca8c606ull, 0x69b85ea283597190ull, 0x03600a05ffba0055ull - }, - { // base nonce 1000000 - 0x6b390e64bbdd91ceull, 0x9b34dd7b05214a6bull, 0x60c14bcda3df2768ull, 0x557f5df495f92ad5ull, 0x5ac883b3090f344bull, 0xdec4058df6f8b088ull, 0x6e3dcb56bbb79ec9ull, 0x6450bb96c6c686bdull, - 0xc2021c81ef714e23ull, 0xcc0b3f77c0cf7bf6ull, 0x03cb474759279b0dull, 0x93cf69a5cd961b9cull, 0x61318f0c14d7e3deull, 0x9df2144c49b171beull, 0xa335ecfaf465e9cdull, 0x4f8661257c1706c3ull, - 0xab374ea37f8e253cull, 0x117242d65e8ebdb4ull, 0x9022069313586cd7ull, 0xe97f096510d6ad03ull, 0xa93d76b12894ffc5ull, 0x7823119022da1590ull, 0xde6f389ece0c83e9ull, 0x84e97ea778e4fb15ull, - 0x23d0802d386a21c8ull, 0xbaacd8512e9bbb31ull, 0x10522970c58234bbull, 0x90f0b64b38eed0b1ull, 0xb250988d2986d15bull, 0xf81e101bb298710full, 0x20e9cca5ade09113ull, 0x91c944d603539c62ull - } -}; - -// Dataset self-test: dataset[0..15] and dataset[IGNEUM_MASK] (268435455). -static const uint32_t IGNEUM_DS_HEAD[16] = { - 0xfdad4319u, 0x1a7b68e1u, 0xde6db608u, 0x13d73892u, 0xd17f447au, 0xb2221ccfu, 0x9db004bdu, 0x57d7d367u, - 0xdbc4cf34u, 0x697c009au, 0xc43af1d4u, 0x97f12b2eu, 0x74c37cd0u, 0xc651ea15u, 0x665a6d29u, 0x22330a2du -}; -static const uint32_t IGNEUM_DS_LAST_INDEX = 268435455u; -static const uint32_t IGNEUM_DS_LAST = 0xa83e7aa6u; -// 64 sampled dataset words (index, value) computed on the Mac. -#define IGNEUM_DS_SAMPLES 64 -static const uint32_t IGNEUM_DS_SAMPLE_INDEX[IGNEUM_DS_SAMPLES] = { - 59471966u, 217795994u, 208353206u, 42483309u, 172547758u, 148076330u, 183853158u, 214389424u, 267488061u, 169781097u, 184093494u, 153880993u, 84977930u, 46426879u, 3093825u, 225364072u, 44593546u, 260713159u, 168250303u, 52384140u, 223401610u, 45554030u, 95410555u, 175039924u, 79171087u, 267580473u, 24168642u, 37981670u, 171551130u, 195559979u, 204611762u, 140997658u, 138925853u, 86637313u, 20736778u, 219665210u, 160430336u, 264654675u, 8013395u, 228945585u, 213884386u, 104419827u, 44185464u, 142737231u, 99284897u, 132475900u, 61861762u, 132056166u, 262388043u, 91878046u, 117353561u, 124768597u, 71352993u, 190698941u, 46055428u, 55281366u, 165145231u, 106810753u, 171985651u, 232085256u, 159510492u, 40072060u, 209107596u, 39023794u -}; -static const uint32_t IGNEUM_DS_SAMPLE_VALUE[IGNEUM_DS_SAMPLES] = { - 0x3230bc7bu, 0x7fbfe2c9u, 0xb2690991u, 0x1745c7c5u, 0x0ab0ccafu, 0x1bf87d6bu, 0x160139fdu, 0x719817acu, 0x0155df4bu, 0xbe1e86c3u, 0x680bcd6cu, 0x79c3dc6cu, 0x181e7e5fu, 0x0713a109u, 0xc705dd9fu, 0x3933b7a8u, 0xdd1c0431u, 0x50522b30u, 0xa0020b38u, 0xbff39e96u, 0x21b67e18u, 0x740f8db3u, 0x2baba568u, 0x2c9bef83u, 0x0ad9b671u, 0xc4327869u, 0x7b4fd7d0u, 0x2c29965fu, 0xec56f15fu, 0x61111746u, 0x303a1d6eu, 0xbddcfd1au, 0xf829a355u, 0x6d5df2a9u, 0x01ab8e44u, 0x06d13507u, 0xda8dcfc6u, 0x01a703e1u, 0xafe7d2c1u, 0xc091c3a2u, 0xac1814feu, 0x6e6ff62au, 0x8fdf01bau, 0xdd3f7159u, 0xdfa0d75cu, 0x26684c35u, 0x7f441e63u, 0x88df2570u, 0x8aa4d5ebu, 0xcc816c05u, 0x434df890u, 0xcd392ad6u, 0x1ab4cb63u, 0x595926fau, 0x7cd76b41u, 0x20cb95c4u, 0x13cf823fu, 0xf9daf901u, 0xff9af40au, 0x2c7dfa51u, 0x871206dbu, 0x938c116cu, 0xb64bf199u, 0x5751f874u -}; -// Cache self-test (memory-hard mode): cache[0..15], the last 16 words, and FNV-1a 64 over all 2^26 words. -static const uint32_t IGNEUM_CACHE_HEAD[16] = { - 0x355a86d2u, 0x7957db1cu, 0xd21772afu, 0x6fc1e09bu, 0xd55ce61du, 0x6e6a278bu, 0xd3f543ceu, 0x223d8e82u, - 0x143ab337u, 0x2e9f05bdu, 0x2eb389bfu, 0x0c6e449eu, 0x5cfa4222u, 0xba6560feu, 0x8e3e1aa4u, 0xdbcc1d53u -}; -static const uint32_t IGNEUM_CACHE_LAST[16] = { - 0x41190d91u, 0xbd277957u, 0x22ddbb49u, 0x6986f207u, 0xdf69a4d6u, 0x26401a3au, 0x818230fbu, 0xc417122du, - 0x3597b211u, 0xb553ce55u, 0xcf39cc0du, 0x3b7fc43au, 0x3fd43b00u, 0x67e1c80eu, 0xffa7ea7du, 0xca2960abu -}; -static const uint64_t IGNEUM_CACHE_FNV64 = 0x48c4f5bf24166b2eull; diff --git a/proto-cuda/packs-ca4/mx8_sh256x27/vectors.json b/proto-cuda/packs-ca4/mx8_sh256x27/vectors.json deleted file mode 100644 index c481669bf..000000000 --- a/proto-cuda/packs-ca4/mx8_sh256x27/vectors.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "seed": "igneum-genesis", - "day": "2026-10-03", - "dataset_mode": "memory-hard", - "dataset_log2_words": 28, - "mask": "0x0fffffff", - "lanes": 32, - "source": "igneum-pow (Rust) CPU interpreter, generator v2, memory-hard dataset", - "warps": [ - {"base_nonce": 0, "expected": [ - "0x2576769ee4a14c8d", "0x6408f60b1f606a96", "0x8e1825542dbd3636", "0xb25c17d8820ae079", "0xb6e1a578d06323b2", "0x8bea2965eaa47f55", "0x3684da53df965532", "0x3392668d50a45aad", - "0x5e78d14aaaadce6e", "0xcb0a7992844c9192", "0xfe4ad7fbc328f9ef", "0x7e64515d7d5941a4", "0x425e4eedb7b0ca4b", "0x4f57a8f93ce317b8", "0x081360e3cf765b3a", "0x1e9e919c50331254", - "0x998e7a76718adcba", "0xde5600e2b1838d9f", "0x4f85d0539779a267", "0x40661123ffbce40c", "0x95b247ca86ff61dc", "0xbe2bf61b9fcd1362", "0xfd6d6687a07997da", "0x9935965e43f25045", - "0x27fe1552873c0b40", "0x972c952c83cf5ea9", "0x525b975fb9610333", "0x4f81431408d27e2d", "0x6375d14f804f061f", "0xff6492be48775872", "0x81b2e098e1a01f8f", "0xc58ddcb717dd3370" - ]}, - {"base_nonce": 4096, "expected": [ - "0x1ce77a600ec573b4", "0xb126a41f83521e45", "0x875547a82d6145fc", "0xd4b04bf258ce4941", "0xe24c23e2b2371ad6", "0x7687a4beddbbd270", "0x63b9346ba234a0be", "0x2500fb30f5e7eb20", - "0x7a185d59a30e3333", "0xa3c14cec7e2bcce2", "0x5730e39d367c6b21", "0x03e10c84ac832a3f", "0xf6dd88e0208019ef", "0xd539425d3bc497fc", "0x71461f5d0c23f626", "0xa4c64db97fb4a425", - "0x61f56436538b61f8", "0x749b10eccde2f0bc", "0xbe447d39b5f3aa29", "0x0e4ada60a90088b7", "0x3eda0e2db0bc9f9e", "0xf90d81a21a10089c", "0x7cf50a8f14d2430e", "0x14601dea6a5d8625", - "0x23c448ee2f31a26d", "0x60e5c3d4b218c0c2", "0x46ed8a924ac431fb", "0xcc77440ffbe5c23d", "0x9a15a2da9d6ba946", "0x37117ce37ca8c606", "0x69b85ea283597190", "0x03600a05ffba0055" - ]}, - {"base_nonce": 1000000, "expected": [ - "0x6b390e64bbdd91ce", "0x9b34dd7b05214a6b", "0x60c14bcda3df2768", "0x557f5df495f92ad5", "0x5ac883b3090f344b", "0xdec4058df6f8b088", "0x6e3dcb56bbb79ec9", "0x6450bb96c6c686bd", - "0xc2021c81ef714e23", "0xcc0b3f77c0cf7bf6", "0x03cb474759279b0d", "0x93cf69a5cd961b9c", "0x61318f0c14d7e3de", "0x9df2144c49b171be", "0xa335ecfaf465e9cd", "0x4f8661257c1706c3", - "0xab374ea37f8e253c", "0x117242d65e8ebdb4", "0x9022069313586cd7", "0xe97f096510d6ad03", "0xa93d76b12894ffc5", "0x7823119022da1590", "0xde6f389ece0c83e9", "0x84e97ea778e4fb15", - "0x23d0802d386a21c8", "0xbaacd8512e9bbb31", "0x10522970c58234bb", "0x90f0b64b38eed0b1", "0xb250988d2986d15b", "0xf81e101bb298710f", "0x20e9cca5ade09113", "0x91c944d603539c62" - ]} - ], - "dataset_head": ["0xfdad4319", "0x1a7b68e1", "0xde6db608", "0x13d73892", "0xd17f447a", "0xb2221ccf", "0x9db004bd", "0x57d7d367", "0xdbc4cf34", "0x697c009a", "0xc43af1d4", "0x97f12b2e", "0x74c37cd0", "0xc651ea15", "0x665a6d29", "0x22330a2d"], - "dataset_last_index": 268435455, - "dataset_last": "0xa83e7aa6", - "dataset_samples": [{"index": 59471966, "value": "0x3230bc7b"}, {"index": 217795994, "value": "0x7fbfe2c9"}, {"index": 208353206, "value": "0xb2690991"}, {"index": 42483309, "value": "0x1745c7c5"}, {"index": 172547758, "value": "0x0ab0ccaf"}, {"index": 148076330, "value": "0x1bf87d6b"}, {"index": 183853158, "value": "0x160139fd"}, {"index": 214389424, "value": "0x719817ac"}, {"index": 267488061, "value": "0x0155df4b"}, {"index": 169781097, "value": "0xbe1e86c3"}, {"index": 184093494, "value": "0x680bcd6c"}, {"index": 153880993, "value": "0x79c3dc6c"}, {"index": 84977930, "value": "0x181e7e5f"}, {"index": 46426879, "value": "0x0713a109"}, {"index": 3093825, "value": "0xc705dd9f"}, {"index": 225364072, "value": "0x3933b7a8"}, {"index": 44593546, "value": "0xdd1c0431"}, {"index": 260713159, "value": "0x50522b30"}, {"index": 168250303, "value": "0xa0020b38"}, {"index": 52384140, "value": "0xbff39e96"}, {"index": 223401610, "value": "0x21b67e18"}, {"index": 45554030, "value": "0x740f8db3"}, {"index": 95410555, "value": "0x2baba568"}, {"index": 175039924, "value": "0x2c9bef83"}, {"index": 79171087, "value": "0x0ad9b671"}, {"index": 267580473, "value": "0xc4327869"}, {"index": 24168642, "value": "0x7b4fd7d0"}, {"index": 37981670, "value": "0x2c29965f"}, {"index": 171551130, "value": "0xec56f15f"}, {"index": 195559979, "value": "0x61111746"}, {"index": 204611762, "value": "0x303a1d6e"}, {"index": 140997658, "value": "0xbddcfd1a"}, {"index": 138925853, "value": "0xf829a355"}, {"index": 86637313, "value": "0x6d5df2a9"}, {"index": 20736778, "value": "0x01ab8e44"}, {"index": 219665210, "value": "0x06d13507"}, {"index": 160430336, "value": "0xda8dcfc6"}, {"index": 264654675, "value": "0x01a703e1"}, {"index": 8013395, "value": "0xafe7d2c1"}, {"index": 228945585, "value": "0xc091c3a2"}, {"index": 213884386, "value": "0xac1814fe"}, {"index": 104419827, "value": "0x6e6ff62a"}, {"index": 44185464, "value": "0x8fdf01ba"}, {"index": 142737231, "value": "0xdd3f7159"}, {"index": 99284897, "value": "0xdfa0d75c"}, {"index": 132475900, "value": "0x26684c35"}, {"index": 61861762, "value": "0x7f441e63"}, {"index": 132056166, "value": "0x88df2570"}, {"index": 262388043, "value": "0x8aa4d5eb"}, {"index": 91878046, "value": "0xcc816c05"}, {"index": 117353561, "value": "0x434df890"}, {"index": 124768597, "value": "0xcd392ad6"}, {"index": 71352993, "value": "0x1ab4cb63"}, {"index": 190698941, "value": "0x595926fa"}, {"index": 46055428, "value": "0x7cd76b41"}, {"index": 55281366, "value": "0x20cb95c4"}, {"index": 165145231, "value": "0x13cf823f"}, {"index": 106810753, "value": "0xf9daf901"}, {"index": 171985651, "value": "0xff9af40a"}, {"index": 232085256, "value": "0x2c7dfa51"}, {"index": 159510492, "value": "0x871206db"}, {"index": 40072060, "value": "0x938c116c"}, {"index": 209107596, "value": "0xb64bf199"}, {"index": 39023794, "value": "0x5751f874"}], - "cache_head": ["0x355a86d2", "0x7957db1c", "0xd21772af", "0x6fc1e09b", "0xd55ce61d", "0x6e6a278b", "0xd3f543ce", "0x223d8e82", "0x143ab337", "0x2e9f05bd", "0x2eb389bf", "0x0c6e449e", "0x5cfa4222", "0xba6560fe", "0x8e3e1aa4", "0xdbcc1d53"], - "cache_last_line": ["0x41190d91", "0xbd277957", "0x22ddbb49", "0x6986f207", "0xdf69a4d6", "0x26401a3a", "0x818230fb", "0xc417122d", "0x3597b211", "0xb553ce55", "0xcf39cc0d", "0x3b7fc43a", "0x3fd43b00", "0x67e1c80e", "0xffa7ea7d", "0xca2960ab"], - "cache_fnv1a64": "0x48c4f5bf24166b2e" -} diff --git a/proto-cuda/packs-ca4/mx8_shl256x27/kernel.cl b/proto-cuda/packs-ca4/mx8_shl256x27/kernel.cl deleted file mode 100644 index 932bfbf4a..000000000 --- a/proto-cuda/packs-ca4/mx8_shl256x27/kernel.cl +++ /dev/null @@ -1,583 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// OpenCL C twin of the Metal kernel for the same seed (see proto-opencl/README.md, WAVEFRONT.md and program.metal). -// Built from source at runtime by proto-opencl/host.c, which passes these defines: -// IGNEUM_GROUP work-group size of igneum_hash, a multiple of 32 (default 32: one work-group = one 32-lane unit) -// IGNEUM_EXCHANGE 0 = local-memory exchange with a barrier (any device, any wave width; the default) -// 1 = sub_group_shuffle_xor (cl_khr_subgroup_shuffle), only with IGNEUM_GROUP 32 and a sub-group size of exactly 32 -// 2 = intel_sub_group_shuffle_xor (cl_intel_subgroups), same condition -// The verification unit is always 32 lanes. A 64-wide hardware wave (AMD GCN/CDNA, RDNA in wave64) runs two units; -// the exchange masks are 1, 2, 4, 8, 16, so every partner lane lies inside the lane's own aligned run of 32. -#ifndef IGNEUM_GROUP -#define IGNEUM_GROUP 32 -#endif -#ifndef IGNEUM_EXCHANGE -#define IGNEUM_EXCHANGE 0 -#endif -#ifdef __OPENCL_VERSION__ -#define IGNEUM_KERNEL_HASH __kernel __attribute__((reqd_work_group_size(IGNEUM_GROUP, 1, 1))) -#define IGNEUM_LOCAL_WORDS(name, n) __local uint name[n] -#if IGNEUM_EXCHANGE == 1 -#ifdef cl_khr_subgroups -#pragma OPENCL EXTENSION cl_khr_subgroups : enable -#endif -#ifdef cl_khr_subgroup_shuffle -#pragma OPENCL EXTENSION cl_khr_subgroup_shuffle : enable -#endif -#elif IGNEUM_EXCHANGE == 2 -#pragma OPENCL EXTENSION cl_intel_subgroups : enable -#endif -#else -// Not an OpenCL compiler: proto-opencl/emu compiles this file as C++ and supplies the built-ins and these two macros. -#include "emu_opencl.h" -#endif - -#if IGNEUM_EXCHANGE == 1 -#define IGNEUM_SHFL_XOR(dst, a, m) dst = sub_group_shuffle_xor((a), (uint)(m)) -#define IGNEUM_BCAST0(dst, a) dst = sub_group_broadcast((a), 0u) -#elif IGNEUM_EXCHANGE == 2 -#define IGNEUM_SHFL_XOR(dst, a, m) dst = intel_sub_group_shuffle_xor((a), (uint)(m)) -#define IGNEUM_BCAST0(dst, a) dst = sub_group_broadcast((a), 0u) -#else -// Local-memory exchange. Two buffers of IGNEUM_GROUP words alternate (xk counts exchanges), so one barrier per -// exchange is enough: a lane can only overwrite buffer b at exchange k+2 after passing barrier k+1, and every lane -// reaches barrier k+1 only after its read of buffer b at exchange k. The partner lid ^ m stays inside the lane's -// aligned run of 32 because m < 32. Control flow is uniform, so every work-item reaches every barrier. -#define IGNEUM_SHFL_XOR(dst, a, m) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid ^ (uint)(m))]; xk += 1u; } -#define IGNEUM_BCAST0(dst, a) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid & ~31u)]; xk += 1u; } -#endif - -static inline uint splitmix32(uint x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -// n is a literal in 1..31 at every call site. OpenCL rotate() rotates left by n modulo 32. -static inline uint rotl_imm(uint x, uint n) { return rotate(x, n); } -// Right rotation by n modulo 32 as a left rotation by (32 - n) modulo 32; n == 0 gives x. -static inline uint rotr_var(uint x, uint n) { return rotate(x, (0u - n) & 31u); } -static inline uint ds_elem(uint i, uint d0, uint d1) { - uint x = i ^ d0; - x *= 0x9E3779B1u; x ^= x >> 15; - x += d1; - x *= 0x85EBCA77u; x ^= x >> 13; - x *= 0xC2B2AE3Du; x ^= x >> 16; - return x; -} - -// Memory-hard dataset core (MEMHARD.md). Cache: 2^26 words in 2^16 segments of 64 chained ChaCha12 lines. -// Item: 8 rounds of 8 x seed-parameterised mixer + one 64-byte cache read, then 8 x final mixer (class v3, mixer multiplier 8, -// docs/plans/mixer-x4.md: the round key of application j of round r is 0x9E3779B9 * (r * m + j + 1)). All parameters are literals. -#define MH_CACHE_LINE_MASK 0x003fffffu -#define MH_SEGMENT_LINES 64u -#define MH_QR(a, b, c, d, r1, r2, r3, r4) { a += b; d ^= a; d = mh_rotl(d, r1); c += d; b ^= c; b = mh_rotl(b, r2); a += b; d ^= a; d = mh_rotl(d, r3); c += d; b ^= c; b = mh_rotl(b, r4); } -static inline uint mh_rotl(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 at every call site - -// y = ChaCha12 core(x) + x -static inline void mh_chacha_block(const uint* x, uint* y) { - for (uint i = 0u; i < 16u; ++i) y[i] = x[i]; - for (uint r = 0u; r < 6u; ++r) { - MH_QR(y[0], y[4], y[8], y[12], 16u, 12u, 8u, 7u) MH_QR(y[1], y[5], y[9], y[13], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[6], y[10], y[14], 16u, 12u, 8u, 7u) MH_QR(y[3], y[7], y[11], y[15], 16u, 12u, 8u, 7u) - MH_QR(y[0], y[5], y[10], y[15], 16u, 12u, 8u, 7u) MH_QR(y[1], y[6], y[11], y[12], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[7], y[8], y[13], 16u, 12u, 8u, 7u) MH_QR(y[3], y[4], y[9], y[14], 16u, 12u, 8u, 7u) - } - for (uint i = 0u; i < 16u; ++i) y[i] += x[i]; -} - -// One cache segment: 64 chained lines written at cache[seg * 1024]. in_j = prev ^ (sigma || K || seg || j || tag), prev_0 = 0. -static inline void mh_cache_segment(__global uint* cache, uint seg) { - uint prev[16]; uint x[16]; uint y[16]; - for (uint i = 0u; i < 16u; ++i) prev[i] = 0u; - for (uint j = 0u; j < MH_SEGMENT_LINES; ++j) { - x[0] = 0x61707865u ^ prev[0]; x[1] = 0x3320646eu ^ prev[1]; x[2] = 0x79622d32u ^ prev[2]; x[3] = 0x6b206574u ^ prev[3]; - x[4] = 0x3067619fu ^ prev[4]; - x[5] = 0x3c269176u ^ prev[5]; - x[6] = 0x84a03b03u ^ prev[6]; - x[7] = 0xf8c63294u ^ prev[7]; - x[8] = 0xff977c5bu ^ prev[8]; - x[9] = 0xe60def3eu ^ prev[9]; - x[10] = 0x63630141u ^ prev[10]; - x[11] = 0xb8fbcb58u ^ prev[11]; - x[12] = seg ^ prev[12]; x[13] = j ^ prev[13]; x[14] = 0x49676e65u ^ prev[14]; x[15] = 0x756d4d48u ^ prev[15]; - mh_chacha_block(x, y); - __global uint* line = cache + ((seg * MH_SEGMENT_LINES + j) * 16u); - for (uint i = 0u; i < 16u; ++i) { line[i] = y[i]; prev[i] = y[i]; } - } -} - -// M_r: per word (s ^ (RC + rk)) * MUL, then a column round and a diagonal round with the seed-drawn rotations. -static inline void mh_mixer(uint* s, uint rk) { - s[0] = (s[0] ^ (0xbab68293u + rk)) * 0x42146205u; - s[1] = (s[1] ^ (0xcc162340u + rk)) * 0x52cbe0fbu; - s[2] = (s[2] ^ (0x6ce151ccu + rk)) * 0x7ecf4a03u; - s[3] = (s[3] ^ (0xe62b8997u + rk)) * 0x6728907fu; - s[4] = (s[4] ^ (0xc9c80297u + rk)) * 0xd81d9751u; - s[5] = (s[5] ^ (0xf74a1654u + rk)) * 0x132952c3u; - s[6] = (s[6] ^ (0x3d704af5u + rk)) * 0xf60de277u; - s[7] = (s[7] ^ (0x3cf522b7u + rk)) * 0x05358035u; - s[8] = (s[8] ^ (0x2b9cac04u + rk)) * 0xbaf6499du; - s[9] = (s[9] ^ (0xa880ac10u + rk)) * 0xe4db9667u; - s[10] = (s[10] ^ (0x13e5dd1du + rk)) * 0x3e98f45du; - s[11] = (s[11] ^ (0x6fc3e233u + rk)) * 0xd0004eddu; - s[12] = (s[12] ^ (0x2d83eeacu + rk)) * 0x2691630du; - s[13] = (s[13] ^ (0x9006e8bfu + rk)) * 0x9beb3bcfu; - s[14] = (s[14] ^ (0x2c4b5362u + rk)) * 0xab310379u; - s[15] = (s[15] ^ (0x31b49ee2u + rk)) * 0x99cfb423u; - MH_QR(s[0], s[4], s[8], s[12], 20u, 20u, 19u, 4u) MH_QR(s[1], s[5], s[9], s[13], 20u, 20u, 19u, 4u) - MH_QR(s[2], s[6], s[10], s[14], 20u, 20u, 19u, 4u) MH_QR(s[3], s[7], s[11], s[15], 20u, 20u, 19u, 4u) - MH_QR(s[0], s[5], s[10], s[15], 26u, 3u, 3u, 27u) MH_QR(s[1], s[6], s[11], s[12], 26u, 3u, 3u, 27u) - MH_QR(s[2], s[7], s[8], s[13], 26u, 3u, 3u, 27u) MH_QR(s[3], s[4], s[9], s[14], 26u, 3u, 3u, 27u) -} - -// Item t: 16 words. s = (K, t * MUL[i] + RC[i]); 8 rounds of 8 x mixer + cache line s[0] & mask; 8 x final mixer. -static inline void mh_item(__global const uint* cache, uint t, uint* s) { - s[0] = 0x3067619fu; - s[1] = 0x3c269176u; - s[2] = 0x84a03b03u; - s[3] = 0xf8c63294u; - s[4] = 0xff977c5bu; - s[5] = 0xe60def3eu; - s[6] = 0x63630141u; - s[7] = 0xb8fbcb58u; - s[8] = t * 0x42146205u + 0xbab68293u; - s[9] = t * 0x52cbe0fbu + 0xcc162340u; - s[10] = t * 0x7ecf4a03u + 0x6ce151ccu; - s[11] = t * 0x6728907fu + 0xe62b8997u; - s[12] = t * 0xd81d9751u + 0xc9c80297u; - s[13] = t * 0x132952c3u + 0xf74a1654u; - s[14] = t * 0xf60de277u + 0x3d704af5u; - s[15] = t * 0x05358035u + 0x3cf522b7u; - for (uint r = 0u; r < 8u; ++r) { - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (r * 8u + j + 1u)); - __global const uint* line = cache + ((s[0] & MH_CACHE_LINE_MASK) * 16u); - for (uint i = 0u; i < 16u; ++i) s[i] ^= line[i]; - } - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (64u + j + 1u)); -} -// dataset[w] without the dataset: derive item w >> 4 and take word w & 15. -static inline uint mh_word(__global const uint* cache, uint w) { uint s[16]; mh_item(cache, w >> 4u, s); return s[w & 15u]; } - -// Memory-hard dataset (MEMHARD.md). One work-item per cache segment; one work-item per 64-byte dataset item. -// The same constants as memhard.h in this pack (one emitter, three dialects). -__kernel void igneum_cache_fill(__global uint* cache, uint nSegments) { - uint seg = (uint)get_global_id(0); - if (seg < nSegments) mh_cache_segment(cache, seg); -} -__kernel void igneum_build(__global uint* ds, __global const uint* cache, uint nItems) { - uint t = (uint)get_global_id(0); - if (t < nItems) { - uint s[16]; - mh_item(cache, t, s); - __global uint* d = ds + ((ulong)t * 16u); - for (uint i = 0u; i < 16u; ++i) d[i] = s[i]; - } -} - -// One hash per work-item. IGNEUM_GROUP is a multiple of 32; lane = lid & 31 and every exchange stays inside the -// lane's own aligned run of 32 work-items, exactly like simd_shuffle_xor inside a 32-wide Metal SIMD group and -// __shfl_xor_sync inside a CUDA warp. Control flow is uniform (no branches at all). -IGNEUM_KERNEL_HASH void igneum_hash(__global const uint* ds, __global ulong* out, uint baseNonce, uint mask) { - uint gid = (uint)get_global_id(0); - uint lid = (uint)get_local_id(0); - uint nonce = baseNonce + gid; - uint r0, r1, r2, r3, r4, r5, r6, r7; -#if IGNEUM_EXCHANGE == 0 - IGNEUM_LOCAL_WORDS(xch, 2 * IGNEUM_GROUP); - uint xk = 0u; -#else - (void)lid; -#endif - { uint x = nonce ^ 0x67a9a7beu; x += 0x9e3779b9u; x = splitmix32(x); r0 = x ^ 0x1a155b25u; } // SEEDW[0], 0x9e3779b9u * 1u, SEEDW[1] - { uint x = nonce ^ 0x1a155b25u; x += 0x3c6ef372u; x = splitmix32(x); r1 = x ^ 0xfddfb732u; } // SEEDW[1], 0x9e3779b9u * 2u, SEEDW[2] - { uint x = nonce ^ 0xfddfb732u; x += 0xdaa66d2bu; x = splitmix32(x); r2 = x ^ 0x4b5af2e8u; } // SEEDW[2], 0x9e3779b9u * 3u, SEEDW[3] - { uint x = nonce ^ 0x4b5af2e8u; x += 0x78dde6e4u; x = splitmix32(x); r3 = x ^ 0xc55caf33u; } // SEEDW[3], 0x9e3779b9u * 4u, SEEDW[4] - { uint x = nonce ^ 0xc55caf33u; x += 0x1715609du; x = splitmix32(x); r4 = x ^ 0xa27c13b7u; } // SEEDW[4], 0x9e3779b9u * 5u, SEEDW[5] - { uint x = nonce ^ 0xa27c13b7u; x += 0xb54cda56u; x = splitmix32(x); r5 = x ^ 0x06628a48u; } // SEEDW[5], 0x9e3779b9u * 6u, SEEDW[6] - { uint x = nonce ^ 0x06628a48u; x += 0x5384540fu; x = splitmix32(x); r6 = x ^ 0x03852469u; } // SEEDW[6], 0x9e3779b9u * 7u, SEEDW[7] - { uint x = nonce ^ 0x03852469u; x += 0xf1bbcdc8u; x = splitmix32(x); r7 = x ^ 0x67a9a7beu; } // SEEDW[7], 0x9e3779b9u * 8u, SEEDW[0] - - for (uint it = 0u; it < 8u; ++it) { - uint sel = r0; - r2 = r3 * r4 + r2; // 0 mad - r2 = r1 * r1 + r2; // 1 mad - r2 = r3 * r2 + r2; // 2 mad - r3 = r3 ^ r5; // 3 xor - r7 = r7 ^ ds[r2 & mask]; // 4 load - // per-load shadow sub-block 0 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 0 - for (uint sh0 = 0u; sh0 < 27u; ++sh0) { - r5 = r5 + r2 + ((((sel >> 18u) & 1u) != 0u) ? 0x8bc12da9u : 0x92199f99u); // s0 add - r0 = r0 + r7 + ((((sel >> 26u) & 1u) != 0u) ? 0x63079e5au : 0x8d72d3adu); // s1 add - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 2u); r6 = r6 ^ t_; } // s2 shfl - r4 = r4 - r2; // s3 sub - r7 = r7 + r0 + ((((sel >> 31u) & 1u) != 0u) ? 0x5d4c7a60u : 0xb21b4babu); // s4 add - r0 = rotl_imm(r0, 11u); // s5 rotl - r0 = r0 + r1 + ((((sel >> 16u) & 1u) != 0u) ? 0xc88e2942u : 0x2fe0e98bu); // s6 add - r7 = r7 ^ r1; // s7 xor - r1 = r6 * r5 + r1; // s8 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 4u); r6 = r6 ^ t_; } // s9 shfl - r1 = r2 * r2 + r1; // s10 mad - r5 = r0 * r3 + r5; // s11 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 4u); r2 = r2 ^ t_; } // s12 shfl - r1 = r1 + r5 + ((((sel >> 25u) & 1u) != 0u) ? 0xbc48c63eu : 0xbdf8f9a5u); // s13 add - r1 = rotl_imm(r1, 29u); // s14 rotl - r1 = r1 - r4; // s15 sub - } - r5 = r5 ^ ds[r7 & mask]; // 5 load - // per-load shadow sub-block 1 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 1 - for (uint sh1 = 0u; sh1 < 27u; ++sh1) { - r7 = r7 | r1; // s16 or - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 8u); r2 = r2 ^ t_; } // s17 shfl - r7 = r7 ^ r4; // s18 xor - r6 = r6 * r1; // s19 mul - r5 = r6 * r0 + r5; // s20 mad - r3 = r3 - r1; // s21 sub - r6 = r6 * r0; // s22 mul - r2 = r2 + r0 + ((((sel >> 1u) & 1u) != 0u) ? 0x96e8f127u : 0x45c37cecu); // s23 add - r6 = r6 - r4; // s24 sub - r7 = r3 * r4 + r7; // s25 mad - r3 = rotl_imm(r3, 9u); // s26 rotl - r2 = r2 - r1; // s27 sub - r6 = mul_hi(r6, r0); // s28 mulhi - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 2u); r2 = r2 ^ t_; } // s29 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 1u); r1 = r1 ^ t_; } // s30 shfl - r1 = r1 + r6 + ((((sel >> 1u) & 1u) != 0u) ? 0xb1cdb2abu : 0x37985632u); // s31 add - } - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 8u); r1 = r1 ^ t_; } // 6 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 8u); r7 = r7 ^ t_; } // 7 shfl - r1 = mul_hi(r1, r5); // 8 mulhi - r6 = rotr_var(r6, r3); // 9 rotr - r3 = r3 | r4; // 10 or - r4 = r4 ^ ds[r3 & mask]; // 11 load - // per-load shadow sub-block 2 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 2 - for (uint sh2 = 0u; sh2 < 27u; ++sh2) { - r0 = rotr_var(r0, r1); // s32 rotr - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 2u); r3 = r3 ^ t_; } // s33 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 4u); r0 = r0 ^ t_; } // s34 shfl - r7 = r7 * r0; // s35 mul - r3 = r3 + r1 + ((((sel >> 0u) & 1u) != 0u) ? 0x856e0180u : 0x804e777eu); // s36 add - r1 = r1 ^ r6; // s37 xor - r2 = r2 * r7; // s38 mul - r6 = r6 + r5 + ((((sel >> 2u) & 1u) != 0u) ? 0xe9bd0cc4u : 0x0b06c8a7u); // s39 add - r5 = r5 * r7; // s40 mul - r2 = mul_hi(r2, r3); // s41 mulhi - r2 = r2 ^ r0; // s42 xor - r0 = rotl_imm(r0, 4u); // s43 rotl - r4 = mul_hi(r4, r3); // s44 mulhi - r6 = r6 + r7 + ((((sel >> 12u) & 1u) != 0u) ? 0xcdcb63f6u : 0xee822e17u); // s45 add - r3 = r2 * r0 + r3; // s46 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 8u); r4 = r4 ^ t_; } // s47 shfl - } - r0 = mul_hi(r0, r4); // 12 mulhi - r5 = r5 + r1 + ((((sel >> 30u) & 1u) != 0u) ? 0xd3177981u : 0xc7934706u); // 13 add - r0 = r0 ^ ds[r4 & mask]; // 14 load - // per-load shadow sub-block 3 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 3 - for (uint sh3 = 0u; sh3 < 27u; ++sh3) { - r6 = mul_hi(r6, r5); // s48 mulhi - r0 = r0 - r2; // s49 sub - r3 = r3 - r5; // s50 sub - r1 = rotr_var(r1, r4); // s51 rotr - r6 = r7 * r7 + r6; // s52 mad - r5 = r5 ^ r3; // s53 xor - r1 = r1 - r0; // s54 sub - r5 = r5 - r6; // s55 sub - r3 = r3 + r2 + ((((sel >> 10u) & 1u) != 0u) ? 0xd4758987u : 0x3dfad1b6u); // s56 add - r4 = r4 + r1 + ((((sel >> 0u) & 1u) != 0u) ? 0x0602d6beu : 0xfca75bc2u); // s57 add - r5 = mul_hi(r5, r6); // s58 mulhi - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 2u); r2 = r2 ^ t_; } // s59 shfl - r2 = rotl_imm(r2, 9u); // s60 rotl - r4 = r4 | r6; // s61 or - r6 = rotr_var(r6, r4); // s62 rotr - r2 = r2 * r4; // s63 mul - } - r2 = r2 - r4; // 15 sub - r2 = r2 ^ ds[r0 & mask]; // 16 load - // per-load shadow sub-block 4 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 4 - for (uint sh4 = 0u; sh4 < 27u; ++sh4) { - r0 = r0 ^ r5; // s64 xor - r2 = r2 ^ r7; // s65 xor - r2 = r2 + r1 + ((((sel >> 6u) & 1u) != 0u) ? 0x8596687au : 0xd71c02ffu); // s66 add - r1 = rotl_imm(r1, 21u); // s67 rotl - r3 = r3 * r2; // s68 mul - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 2u); r7 = r7 ^ t_; } // s69 shfl - r3 = r3 * r2; // s70 mul - r0 = r2 * r5 + r0; // s71 mad - r6 = r6 + r7 + ((((sel >> 0u) & 1u) != 0u) ? 0x08ac5733u : 0x3c6fe15du); // s72 add - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 8u); r3 = r3 ^ t_; } // s73 shfl - r3 = r3 * r6; // s74 mul - r6 = r6 ^ r7; // s75 xor - r3 = r3 | r0; // s76 or - r2 = r2 + r4 + ((((sel >> 1u) & 1u) != 0u) ? 0xc100b495u : 0x295d5faeu); // s77 add - r3 = mul_hi(r3, r7); // s78 mulhi - r4 = r4 | r1; // s79 or - } - r7 = r7 ^ ds[r2 & mask]; // 17 load - // per-load shadow sub-block 5 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 5 - for (uint sh5 = 0u; sh5 < 27u; ++sh5) { - r4 = rotr_var(r4, r3); // s80 rotr - r4 = r4 + r3 + ((((sel >> 15u) & 1u) != 0u) ? 0x5cc59530u : 0x274a9221u); // s81 add - r7 = r7 + r3 + ((((sel >> 5u) & 1u) != 0u) ? 0x141479ecu : 0xc8651f8eu); // s82 add - r3 = rotr_var(r3, r6); // s83 rotr - r2 = r4 * r7 + r2; // s84 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 16u); r2 = r2 ^ t_; } // s85 shfl - r3 = r3 ^ r2; // s86 xor - { uint t_; IGNEUM_SHFL_XOR(t_, r7, 2u); r5 = r5 ^ t_; } // s87 shfl - r0 = r0 ^ r4; // s88 xor - r3 = r3 + r2 + ((((sel >> 18u) & 1u) != 0u) ? 0x883c0c92u : 0x53f915c2u); // s89 add - r7 = rotr_var(r7, r5); // s90 rotr - r3 = r3 + r1 + ((((sel >> 31u) & 1u) != 0u) ? 0x3cc5bd20u : 0xe2be00a5u); // s91 add - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 1u); r7 = r7 ^ t_; } // s92 shfl - r6 = rotr_var(r6, r2); // s93 rotr - r7 = rotl_imm(r7, 14u); // s94 rotl - r4 = r5 * r5 + r4; // s95 mad - } - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 4u); r7 = r7 ^ t_; } // 18 shfl - r5 = r5 * r0; // 19 mul - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 2u); r3 = r3 ^ t_; } // 20 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 16u); r2 = r2 ^ t_; } // 21 shfl - r6 = mul_hi(r6, r2); // 22 mulhi - r6 = r6 ^ ds[r1 & mask]; // 23 load - // per-load shadow sub-block 6 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 6 - for (uint sh6 = 0u; sh6 < 27u; ++sh6) { - r2 = r4 * r5 + r2; // s96 mad - r1 = r1 ^ r0; // s97 xor - r5 = r5 * r4; // s98 mul - r2 = r2 - r0; // s99 sub - r7 = rotl_imm(r7, 30u); // s100 rotl - r5 = r5 + r6 + ((((sel >> 17u) & 1u) != 0u) ? 0xbfd646cbu : 0x423fd9c9u); // s101 add - r7 = r7 + r6 + ((((sel >> 11u) & 1u) != 0u) ? 0x19b74a43u : 0x8d3c011du); // s102 add - r5 = rotl_imm(r5, 6u); // s103 rotl - r0 = r0 * r4; // s104 mul - r0 = r0 | r5; // s105 or - r0 = r0 + r1 + ((((sel >> 15u) & 1u) != 0u) ? 0x7dcb7e18u : 0x8ce14721u); // s106 add - r0 = rotl_imm(r0, 15u); // s107 rotl - r4 = r4 - r2; // s108 sub - r2 = mul_hi(r2, r0); // s109 mulhi - r1 = mul_hi(r1, r0); // s110 mulhi - r0 = r0 + r2 + ((((sel >> 28u) & 1u) != 0u) ? 0x3c179ad8u : 0x2d9fc6b9u); // s111 add - } - r5 = r5 * r0; // 24 mul - r5 = rotl_imm(r5, 19u); // 25 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 2u); r7 = r7 ^ t_; } // 26 shfl - r0 = r0 ^ r5; // 27 xor - r0 = r0 ^ r4; // 28 xor - r3 = r3 - r0; // 29 sub - r5 = r5 * r1; // 30 mul - r7 = r7 ^ ds[r2 & mask]; // 31 load - // per-load shadow sub-block 7 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 7 - for (uint sh7 = 0u; sh7 < 27u; ++sh7) { - r2 = mul_hi(r2, r0); // s112 mulhi - r6 = r6 - r3; // s113 sub - r7 = r6 * r3 + r7; // s114 mad - r5 = rotr_var(r5, r1); // s115 rotr - r6 = rotr_var(r6, r4); // s116 rotr - r2 = r2 + r1 + ((((sel >> 22u) & 1u) != 0u) ? 0x47359729u : 0x502138e2u); // s117 add - r3 = r2 * r0 + r3; // s118 mad - r1 = r1 * r3; // s119 mul - r2 = r0 * r4 + r2; // s120 mad - r0 = r0 * r3; // s121 mul - r2 = mul_hi(r2, r0); // s122 mulhi - r5 = r5 * r6; // s123 mul - r4 = r2 * r4 + r4; // s124 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 2u); r4 = r4 ^ t_; } // s125 shfl - r2 = r2 ^ r0; // s126 xor - r1 = rotl_imm(r1, 7u); // s127 rotl - } - r1 = r1 ^ ds[r0 & mask]; // 32 load - // per-load shadow sub-block 8 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 8 - for (uint sh8 = 0u; sh8 < 27u; ++sh8) { - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 4u); r7 = r7 ^ t_; } // s128 shfl - r6 = rotl_imm(r6, 17u); // s129 rotl - r2 = r2 + r5 + ((((sel >> 15u) & 1u) != 0u) ? 0x6c57e4e7u : 0x33dff776u); // s130 add - r0 = r0 | r3; // s131 or - r5 = rotr_var(r5, r0); // s132 rotr - r2 = r2 + r3 + ((((sel >> 30u) & 1u) != 0u) ? 0xe8212c0cu : 0x5db25b34u); // s133 add - r4 = r4 ^ r3; // s134 xor - r6 = r6 ^ r1; // s135 xor - r1 = r1 - r7; // s136 sub - r2 = r6 * r2 + r2; // s137 mad - r0 = mul_hi(r0, r5); // s138 mulhi - r2 = r2 + r7 + ((((sel >> 10u) & 1u) != 0u) ? 0xd44ff710u : 0x218d4090u); // s139 add - r1 = rotr_var(r1, r4); // s140 rotr - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 1u); r3 = r3 ^ t_; } // s141 shfl - r7 = r6 * r2 + r7; // s142 mad - r2 = r2 * r3; // s143 mul - } - r5 = r5 ^ r6; // 33 xor - r5 = r5 ^ ds[r1 & mask]; // 34 load - // per-load shadow sub-block 9 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 9 - for (uint sh9 = 0u; sh9 < 27u; ++sh9) { - r7 = r7 + r4 + ((((sel >> 29u) & 1u) != 0u) ? 0xb98942fau : 0xf4b1a8deu); // s144 add - r7 = rotl_imm(r7, 15u); // s145 rotl - r7 = r7 ^ r5; // s146 xor - r4 = r7 * r4 + r4; // s147 mad - r6 = rotr_var(r6, r5); // s148 rotr - r1 = r2 * r4 + r1; // s149 mad - r1 = r1 + r7 + ((((sel >> 17u) & 1u) != 0u) ? 0x0d48ba42u : 0x2bef10f2u); // s150 add - r5 = r5 ^ r4; // s151 xor - r7 = r7 + r0 + ((((sel >> 30u) & 1u) != 0u) ? 0xc98dea9cu : 0xdc5cc080u); // s152 add - r4 = r4 * r6; // s153 mul - r0 = r0 * r2; // s154 mul - r6 = r6 ^ r5; // s155 xor - r4 = rotr_var(r4, r2); // s156 rotr - r1 = rotl_imm(r1, 11u); // s157 rotl - r5 = r5 + r0 + ((((sel >> 11u) & 1u) != 0u) ? 0x6c752dcbu : 0x18197438u); // s158 add - r4 = r1 * r5 + r4; // s159 mad - } - r0 = mul_hi(r0, r5); // 35 mulhi - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 4u); r5 = r5 ^ t_; } // 36 shfl - r7 = r7 ^ ds[r0 & mask]; // 37 load - // per-load shadow sub-block 10 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 10 - for (uint sh10 = 0u; sh10 < 27u; ++sh10) { - r4 = rotl_imm(r4, 26u); // s160 rotl - r3 = r0 * r7 + r3; // s161 mad - r3 = rotr_var(r3, r1); // s162 rotr - r4 = r4 + r0 + ((((sel >> 3u) & 1u) != 0u) ? 0x8e481727u : 0xf1c46574u); // s163 add - r0 = mul_hi(r0, r4); // s164 mulhi - r2 = r2 + r6 + ((((sel >> 20u) & 1u) != 0u) ? 0x550ab406u : 0xac578137u); // s165 add - r0 = rotl_imm(r0, 13u); // s166 rotl - r3 = r3 + r1 + ((((sel >> 14u) & 1u) != 0u) ? 0xaebb5966u : 0xa33e6706u); // s167 add - r3 = r3 | r5; // s168 or - r6 = rotr_var(r6, r2); // s169 rotr - r4 = r4 ^ r6; // s170 xor - r6 = r6 - r1; // s171 sub - r7 = rotl_imm(r7, 22u); // s172 rotl - r5 = rotl_imm(r5, 15u); // s173 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r0, 8u); r7 = r7 ^ t_; } // s174 shfl - r0 = r0 ^ r5; // s175 xor - } - r3 = r3 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0x230c005cu : 0x75ba2fadu); // 38 add - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 4u); r1 = r1 ^ t_; } // 39 shfl - r2 = r2 ^ r5; // 40 xor - r3 = r6 * r3 + r3; // 41 mad - r6 = r6 - r7; // 42 sub - r7 = r7 ^ r0; // 43 xor - r1 = r1 ^ ds[r7 & mask]; // 44 load - // per-load shadow sub-block 11 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 11 - for (uint sh11 = 0u; sh11 < 27u; ++sh11) { - r7 = rotl_imm(r7, 6u); // s176 rotl - r7 = r7 - r0; // s177 sub - r3 = rotl_imm(r3, 30u); // s178 rotl - r7 = r6 * r1 + r7; // s179 mad - r6 = rotl_imm(r6, 9u); // s180 rotl - r2 = r2 ^ r4; // s181 xor - r2 = r2 ^ r7; // s182 xor - r7 = r7 ^ r2; // s183 xor - r1 = r1 + r2 + ((((sel >> 21u) & 1u) != 0u) ? 0x5bb7550fu : 0xd94d55acu); // s184 add - r3 = r3 | r5; // s185 or - r6 = mul_hi(r6, r3); // s186 mulhi - r4 = r4 | r0; // s187 or - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 2u); r7 = r7 ^ t_; } // s188 shfl - r6 = r6 + r5 + ((((sel >> 6u) & 1u) != 0u) ? 0x2a354e2du : 0x89e747feu); // s189 add - r1 = r1 ^ r4; // s190 xor - r7 = mul_hi(r7, r4); // s191 mulhi - } - r2 = r2 * r3; // 45 mul - r1 = mul_hi(r1, r5); // 46 mulhi - r4 = r4 - r3; // 47 sub - r2 = rotr_var(r2, r6); // 48 rotr - r3 = r3 ^ ds[r5 & mask]; // 49 load - // per-load shadow sub-block 12 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 12 - for (uint sh12 = 0u; sh12 < 27u; ++sh12) { - r2 = rotl_imm(r2, 26u); // s192 rotl - r5 = rotl_imm(r5, 8u); // s193 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 16u); r4 = r4 ^ t_; } // s194 shfl - r4 = r4 ^ r5; // s195 xor - r1 = r1 * r3; // s196 mul - r5 = r5 + r2 + ((((sel >> 7u) & 1u) != 0u) ? 0x10cfdc71u : 0xea03e8e7u); // s197 add - r7 = r7 + r5 + ((((sel >> 15u) & 1u) != 0u) ? 0x66e148d3u : 0xa7ee0102u); // s198 add - r5 = r5 - r2; // s199 sub - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 2u); r5 = r5 ^ t_; } // s200 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 8u); r7 = r7 ^ t_; } // s201 shfl - r4 = rotr_var(r4, r5); // s202 rotr - r7 = r7 ^ r5; // s203 xor - r7 = r7 ^ r0; // s204 xor - r6 = r6 + r2 + ((((sel >> 10u) & 1u) != 0u) ? 0x8558b619u : 0x8379a4deu); // s205 add - r4 = rotl_imm(r4, 13u); // s206 rotl - r1 = mul_hi(r1, r3); // s207 mulhi - } - r1 = r1 + r5 + ((((sel >> 7u) & 1u) != 0u) ? 0x1907970cu : 0x81b8bc2cu); // 50 add - r0 = r0 * r2; // 51 mul - r0 = r0 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x699fd448u : 0x4f92b968u); // 52 add - r1 = r1 + r0 + ((((sel >> 12u) & 1u) != 0u) ? 0x77b1520du : 0x2bb965afu); // 53 add - r7 = rotl_imm(r7, 14u); // 54 rotl - r3 = r3 + r7 + ((((sel >> 1u) & 1u) != 0u) ? 0xa54c55a0u : 0x7b0fe07au); // 55 add - r6 = r6 ^ ds[r7 & mask]; // 56 load - // per-load shadow sub-block 13 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 13 - for (uint sh13 = 0u; sh13 < 27u; ++sh13) { - r1 = r4 * r4 + r1; // s208 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r0, 4u); r2 = r2 ^ t_; } // s209 shfl - r7 = r7 + r0 + ((((sel >> 11u) & 1u) != 0u) ? 0xf4049c4cu : 0xaa8cb14eu); // s210 add - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 16u); r7 = r7 ^ t_; } // s211 shfl - r1 = r1 ^ r0; // s212 xor - r7 = rotl_imm(r7, 3u); // s213 rotl - r4 = rotr_var(r4, r7); // s214 rotr - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 16u); r3 = r3 ^ t_; } // s215 shfl - r5 = r5 | r1; // s216 or - r1 = r1 - r2; // s217 sub - r6 = r6 - r5; // s218 sub - r6 = rotl_imm(r6, 4u); // s219 rotl - r2 = mul_hi(r2, r0); // s220 mulhi - r2 = r2 | r0; // s221 or - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 8u); r5 = r5 ^ t_; } // s222 shfl - r5 = mul_hi(r5, r6); // s223 mulhi - } - r1 = rotr_var(r1, r5); // 57 rotr - r5 = r5 ^ ds[r4 & mask]; // 58 load - // per-load shadow sub-block 14 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 14 - for (uint sh14 = 0u; sh14 < 27u; ++sh14) { - r0 = r0 - r6; // s224 sub - r7 = rotl_imm(r7, 23u); // s225 rotl - r4 = r4 | r2; // s226 or - r2 = r2 * r4; // s227 mul - r3 = rotl_imm(r3, 12u); // s228 rotl - r0 = rotr_var(r0, r4); // s229 rotr - r0 = r0 + r6 + ((((sel >> 16u) & 1u) != 0u) ? 0x2a7fecb2u : 0x1d176220u); // s230 add - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 4u); r1 = r1 ^ t_; } // s231 shfl - r2 = r2 * r1; // s232 mul - r7 = r0 * r1 + r7; // s233 mad - r5 = rotl_imm(r5, 22u); // s234 rotl - r4 = rotr_var(r4, r6); // s235 rotr - r0 = r5 * r1 + r0; // s236 mad - r6 = r6 ^ r4; // s237 xor - r4 = r4 ^ r6; // s238 xor - r6 = rotl_imm(r6, 18u); // s239 rotl - } - r6 = r6 ^ ds[r2 & mask]; // 59 load - // per-load shadow sub-block 15 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 15 - for (uint sh15 = 0u; sh15 < 27u; ++sh15) { - r4 = r4 + r7 + ((((sel >> 25u) & 1u) != 0u) ? 0xadce39f3u : 0x17dafb4du); // s240 add - r0 = r0 - r7; // s241 sub - r1 = rotr_var(r1, r6); // s242 rotr - r3 = r3 + r0 + ((((sel >> 29u) & 1u) != 0u) ? 0x0e7033b6u : 0xf2f77d26u); // s243 add - r2 = r7 * r4 + r2; // s244 mad - r4 = r0 * r6 + r4; // s245 mad - r5 = r5 * r7; // s246 mul - r3 = r3 + r5 + ((((sel >> 31u) & 1u) != 0u) ? 0x7b2ff6b7u : 0xfed2da4eu); // s247 add - r0 = r0 + r7 + ((((sel >> 0u) & 1u) != 0u) ? 0xb5fad7b1u : 0x03b2891cu); // s248 add - r5 = mul_hi(r5, r4); // s249 mulhi - r5 = r5 + r2 + ((((sel >> 11u) & 1u) != 0u) ? 0xa7e71c2fu : 0x042cd6e3u); // s250 add - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 1u); r6 = r6 ^ t_; } // s251 shfl - r6 = r6 ^ r1; // s252 xor - r2 = r2 * r5; // s253 mul - r0 = r0 + r6 + ((((sel >> 16u) & 1u) != 0u) ? 0xb83d78deu : 0x784a302bu); // s254 add - r2 = r2 - r4; // s255 sub - } - r3 = r5 * r0 + r3; // 60 mad - r5 = r5 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0xad7493e7u : 0xaf9dd72du); // 61 add - r4 = r4 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x1e07c3d9u : 0x89841d87u); // 62 add - r5 = rotl_imm(r5, 19u); // 63 rotl - } - uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((ulong)hi << 32) | (ulong)lo; -} - -#if IGNEUM_EXCHANGE != 0 -// Reports the sub-group size this device uses for a work-group of IGNEUM_GROUP items. host.c runs it only when the -// per-kernel query (clGetKernelSubGroupInfoKHR on igneum_hash) is unavailable; that query is preferred because a -// compiler may pick a different wave width per kernel (RDNA: wave32 or wave64). See WAVEFRONT.md. -IGNEUM_KERNEL_HASH void igneum_probe_subgroup(__global uint* out) { - if (get_local_id(0) == 0u) { out[0] = get_sub_group_size(); out[1] = get_num_sub_groups(); } -} -#endif diff --git a/proto-cuda/packs-ca4/mx8_shl256x27/kernel.cu b/proto-cuda/packs-ca4/mx8_shl256x27/kernel.cu deleted file mode 100644 index b16a7632a..000000000 --- a/proto-cuda/packs-ca4/mx8_shl256x27/kernel.cu +++ /dev/null @@ -1,468 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// Bit-exact twin of the Metal kernel for the same seed (see proto-cuda/CHECKLIST.md and program.metal). -// Compiled ahead of time by nvcc together with proto-cuda/host.cu. No NVRTC. -#include -#include -#include "program.h" -#include "memhard.h" - -__device__ __forceinline__ uint32_t splitmix32(uint32_t x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -// n is a literal in 1..31 at every call site, so both shift amounts are in 1..31. -__device__ __forceinline__ uint32_t rotl_imm(uint32_t x, uint32_t n) { return (x << n) | (x >> (32u - n)); } -// n is masked to 0..31; the second shift amount is masked too, so n == 0 gives x. -__device__ __forceinline__ uint32_t rotr_var(uint32_t x, uint32_t n) { n &= 31u; return (x >> n) | (x << ((32u - n) & 31u)); } -__device__ __forceinline__ uint32_t ds_elem(uint32_t i, uint32_t d0, uint32_t d1) { - uint32_t x = i ^ d0; - x *= 0x9E3779B1u; x ^= x >> 15; - x += d1; - x *= 0x85EBCA77u; x ^= x >> 13; - x *= 0xC2B2AE3Du; x ^= x >> 16; - return x; -} - -// Memory-hard dataset (MEMHARD.md). One thread per cache segment; one thread per 64-byte dataset item. -// The core functions (mh_cache_segment, mh_item) are in memhard.h and are also compiled for the host. -__global__ void igneum_cache_fill(uint32_t* cache, uint32_t nSegments) { - uint32_t seg = blockIdx.x * blockDim.x + threadIdx.x; - if (seg < nSegments) mh_cache_segment(cache, seg); -} -__global__ void igneum_build(uint32_t* ds, const uint32_t* cache, uint32_t nItems) { - uint32_t t = blockIdx.x * blockDim.x + threadIdx.x; - if (t < nItems) { - uint32_t s[16]; - mh_item(cache, t, s); - uint32_t* d = ds + (size_t)t * 16u; - for (uint32_t i = 0u; i < 16u; ++i) d[i] = s[i]; - } -} - -// One hash per thread. blockDim.x is a multiple of 32; lane = threadIdx.x & 31 and every -// __shfl_xor_sync stays inside the lane's own warp, exactly like simd_shuffle_xor inside a -// 32-wide Metal SIMD group. Control flow is uniform, so the full 0xffffffff member mask is valid. -__global__ void igneum_hash(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask) { - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t nonce = baseNonce + gid; - uint32_t r0, r1, r2, r3, r4, r5, r6, r7; - { uint32_t x = nonce ^ 0x67a9a7beu; x += 0x9e3779b9u; x = splitmix32(x); r0 = x ^ 0x1a155b25u; } // SEEDW[0], 0x9e3779b9u * 1u, SEEDW[1] - { uint32_t x = nonce ^ 0x1a155b25u; x += 0x3c6ef372u; x = splitmix32(x); r1 = x ^ 0xfddfb732u; } // SEEDW[1], 0x9e3779b9u * 2u, SEEDW[2] - { uint32_t x = nonce ^ 0xfddfb732u; x += 0xdaa66d2bu; x = splitmix32(x); r2 = x ^ 0x4b5af2e8u; } // SEEDW[2], 0x9e3779b9u * 3u, SEEDW[3] - { uint32_t x = nonce ^ 0x4b5af2e8u; x += 0x78dde6e4u; x = splitmix32(x); r3 = x ^ 0xc55caf33u; } // SEEDW[3], 0x9e3779b9u * 4u, SEEDW[4] - { uint32_t x = nonce ^ 0xc55caf33u; x += 0x1715609du; x = splitmix32(x); r4 = x ^ 0xa27c13b7u; } // SEEDW[4], 0x9e3779b9u * 5u, SEEDW[5] - { uint32_t x = nonce ^ 0xa27c13b7u; x += 0xb54cda56u; x = splitmix32(x); r5 = x ^ 0x06628a48u; } // SEEDW[5], 0x9e3779b9u * 6u, SEEDW[6] - { uint32_t x = nonce ^ 0x06628a48u; x += 0x5384540fu; x = splitmix32(x); r6 = x ^ 0x03852469u; } // SEEDW[6], 0x9e3779b9u * 7u, SEEDW[7] - { uint32_t x = nonce ^ 0x03852469u; x += 0xf1bbcdc8u; x = splitmix32(x); r7 = x ^ 0x67a9a7beu; } // SEEDW[7], 0x9e3779b9u * 8u, SEEDW[0] - - for (uint32_t it = 0u; it < 8u; ++it) { - uint32_t sel = r0; - r2 = r3 * r4 + r2; // 0 mad - r2 = r1 * r1 + r2; // 1 mad - r2 = r3 * r2 + r2; // 2 mad - r3 = r3 ^ r5; // 3 xor - r7 = r7 ^ ds[r2 & mask]; // 4 load - // per-load shadow sub-block 0 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 0 - for (uint32_t sh0 = 0u; sh0 < 27u; ++sh0) { - r5 = r5 + r2 + ((((sel >> 18u) & 1u) != 0u) ? 0x8bc12da9u : 0x92199f99u); // s0 add - r0 = r0 + r7 + ((((sel >> 26u) & 1u) != 0u) ? 0x63079e5au : 0x8d72d3adu); // s1 add - r6 = r6 ^ __shfl_xor_sync(0xffffffffu, r3, 2); // s2 shfl - r4 = r4 - r2; // s3 sub - r7 = r7 + r0 + ((((sel >> 31u) & 1u) != 0u) ? 0x5d4c7a60u : 0xb21b4babu); // s4 add - r0 = rotl_imm(r0, 11u); // s5 rotl - r0 = r0 + r1 + ((((sel >> 16u) & 1u) != 0u) ? 0xc88e2942u : 0x2fe0e98bu); // s6 add - r7 = r7 ^ r1; // s7 xor - r1 = r6 * r5 + r1; // s8 mad - r6 = r6 ^ __shfl_xor_sync(0xffffffffu, r1, 4); // s9 shfl - r1 = r2 * r2 + r1; // s10 mad - r5 = r0 * r3 + r5; // s11 mad - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r6, 4); // s12 shfl - r1 = r1 + r5 + ((((sel >> 25u) & 1u) != 0u) ? 0xbc48c63eu : 0xbdf8f9a5u); // s13 add - r1 = rotl_imm(r1, 29u); // s14 rotl - r1 = r1 - r4; // s15 sub - } - r5 = r5 ^ ds[r7 & mask]; // 5 load - // per-load shadow sub-block 1 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 1 - for (uint32_t sh1 = 0u; sh1 < 27u; ++sh1) { - r7 = r7 | r1; // s16 or - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r4, 8); // s17 shfl - r7 = r7 ^ r4; // s18 xor - r6 = r6 * r1; // s19 mul - r5 = r6 * r0 + r5; // s20 mad - r3 = r3 - r1; // s21 sub - r6 = r6 * r0; // s22 mul - r2 = r2 + r0 + ((((sel >> 1u) & 1u) != 0u) ? 0x96e8f127u : 0x45c37cecu); // s23 add - r6 = r6 - r4; // s24 sub - r7 = r3 * r4 + r7; // s25 mad - r3 = rotl_imm(r3, 9u); // s26 rotl - r2 = r2 - r1; // s27 sub - r6 = __umulhi(r6, r0); // s28 mulhi - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r4, 2); // s29 shfl - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r6, 1); // s30 shfl - r1 = r1 + r6 + ((((sel >> 1u) & 1u) != 0u) ? 0xb1cdb2abu : 0x37985632u); // s31 add - } - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r4, 8); // 6 shfl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r3, 8); // 7 shfl - r1 = __umulhi(r1, r5); // 8 mulhi - r6 = rotr_var(r6, r3); // 9 rotr - r3 = r3 | r4; // 10 or - r4 = r4 ^ ds[r3 & mask]; // 11 load - // per-load shadow sub-block 2 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 2 - for (uint32_t sh2 = 0u; sh2 < 27u; ++sh2) { - r0 = rotr_var(r0, r1); // s32 rotr - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r4, 2); // s33 shfl - r0 = r0 ^ __shfl_xor_sync(0xffffffffu, r3, 4); // s34 shfl - r7 = r7 * r0; // s35 mul - r3 = r3 + r1 + ((((sel >> 0u) & 1u) != 0u) ? 0x856e0180u : 0x804e777eu); // s36 add - r1 = r1 ^ r6; // s37 xor - r2 = r2 * r7; // s38 mul - r6 = r6 + r5 + ((((sel >> 2u) & 1u) != 0u) ? 0xe9bd0cc4u : 0x0b06c8a7u); // s39 add - r5 = r5 * r7; // s40 mul - r2 = __umulhi(r2, r3); // s41 mulhi - r2 = r2 ^ r0; // s42 xor - r0 = rotl_imm(r0, 4u); // s43 rotl - r4 = __umulhi(r4, r3); // s44 mulhi - r6 = r6 + r7 + ((((sel >> 12u) & 1u) != 0u) ? 0xcdcb63f6u : 0xee822e17u); // s45 add - r3 = r2 * r0 + r3; // s46 mad - r4 = r4 ^ __shfl_xor_sync(0xffffffffu, r3, 8); // s47 shfl - } - r0 = __umulhi(r0, r4); // 12 mulhi - r5 = r5 + r1 + ((((sel >> 30u) & 1u) != 0u) ? 0xd3177981u : 0xc7934706u); // 13 add - r0 = r0 ^ ds[r4 & mask]; // 14 load - // per-load shadow sub-block 3 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 3 - for (uint32_t sh3 = 0u; sh3 < 27u; ++sh3) { - r6 = __umulhi(r6, r5); // s48 mulhi - r0 = r0 - r2; // s49 sub - r3 = r3 - r5; // s50 sub - r1 = rotr_var(r1, r4); // s51 rotr - r6 = r7 * r7 + r6; // s52 mad - r5 = r5 ^ r3; // s53 xor - r1 = r1 - r0; // s54 sub - r5 = r5 - r6; // s55 sub - r3 = r3 + r2 + ((((sel >> 10u) & 1u) != 0u) ? 0xd4758987u : 0x3dfad1b6u); // s56 add - r4 = r4 + r1 + ((((sel >> 0u) & 1u) != 0u) ? 0x0602d6beu : 0xfca75bc2u); // s57 add - r5 = __umulhi(r5, r6); // s58 mulhi - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r1, 2); // s59 shfl - r2 = rotl_imm(r2, 9u); // s60 rotl - r4 = r4 | r6; // s61 or - r6 = rotr_var(r6, r4); // s62 rotr - r2 = r2 * r4; // s63 mul - } - r2 = r2 - r4; // 15 sub - r2 = r2 ^ ds[r0 & mask]; // 16 load - // per-load shadow sub-block 4 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 4 - for (uint32_t sh4 = 0u; sh4 < 27u; ++sh4) { - r0 = r0 ^ r5; // s64 xor - r2 = r2 ^ r7; // s65 xor - r2 = r2 + r1 + ((((sel >> 6u) & 1u) != 0u) ? 0x8596687au : 0xd71c02ffu); // s66 add - r1 = rotl_imm(r1, 21u); // s67 rotl - r3 = r3 * r2; // s68 mul - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r5, 2); // s69 shfl - r3 = r3 * r2; // s70 mul - r0 = r2 * r5 + r0; // s71 mad - r6 = r6 + r7 + ((((sel >> 0u) & 1u) != 0u) ? 0x08ac5733u : 0x3c6fe15du); // s72 add - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r2, 8); // s73 shfl - r3 = r3 * r6; // s74 mul - r6 = r6 ^ r7; // s75 xor - r3 = r3 | r0; // s76 or - r2 = r2 + r4 + ((((sel >> 1u) & 1u) != 0u) ? 0xc100b495u : 0x295d5faeu); // s77 add - r3 = __umulhi(r3, r7); // s78 mulhi - r4 = r4 | r1; // s79 or - } - r7 = r7 ^ ds[r2 & mask]; // 17 load - // per-load shadow sub-block 5 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 5 - for (uint32_t sh5 = 0u; sh5 < 27u; ++sh5) { - r4 = rotr_var(r4, r3); // s80 rotr - r4 = r4 + r3 + ((((sel >> 15u) & 1u) != 0u) ? 0x5cc59530u : 0x274a9221u); // s81 add - r7 = r7 + r3 + ((((sel >> 5u) & 1u) != 0u) ? 0x141479ecu : 0xc8651f8eu); // s82 add - r3 = rotr_var(r3, r6); // s83 rotr - r2 = r4 * r7 + r2; // s84 mad - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r5, 16); // s85 shfl - r3 = r3 ^ r2; // s86 xor - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r7, 2); // s87 shfl - r0 = r0 ^ r4; // s88 xor - r3 = r3 + r2 + ((((sel >> 18u) & 1u) != 0u) ? 0x883c0c92u : 0x53f915c2u); // s89 add - r7 = rotr_var(r7, r5); // s90 rotr - r3 = r3 + r1 + ((((sel >> 31u) & 1u) != 0u) ? 0x3cc5bd20u : 0xe2be00a5u); // s91 add - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r2, 1); // s92 shfl - r6 = rotr_var(r6, r2); // s93 rotr - r7 = rotl_imm(r7, 14u); // s94 rotl - r4 = r5 * r5 + r4; // s95 mad - } - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r3, 4); // 18 shfl - r5 = r5 * r0; // 19 mul - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r4, 2); // 20 shfl - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r4, 16); // 21 shfl - r6 = __umulhi(r6, r2); // 22 mulhi - r6 = r6 ^ ds[r1 & mask]; // 23 load - // per-load shadow sub-block 6 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 6 - for (uint32_t sh6 = 0u; sh6 < 27u; ++sh6) { - r2 = r4 * r5 + r2; // s96 mad - r1 = r1 ^ r0; // s97 xor - r5 = r5 * r4; // s98 mul - r2 = r2 - r0; // s99 sub - r7 = rotl_imm(r7, 30u); // s100 rotl - r5 = r5 + r6 + ((((sel >> 17u) & 1u) != 0u) ? 0xbfd646cbu : 0x423fd9c9u); // s101 add - r7 = r7 + r6 + ((((sel >> 11u) & 1u) != 0u) ? 0x19b74a43u : 0x8d3c011du); // s102 add - r5 = rotl_imm(r5, 6u); // s103 rotl - r0 = r0 * r4; // s104 mul - r0 = r0 | r5; // s105 or - r0 = r0 + r1 + ((((sel >> 15u) & 1u) != 0u) ? 0x7dcb7e18u : 0x8ce14721u); // s106 add - r0 = rotl_imm(r0, 15u); // s107 rotl - r4 = r4 - r2; // s108 sub - r2 = __umulhi(r2, r0); // s109 mulhi - r1 = __umulhi(r1, r0); // s110 mulhi - r0 = r0 + r2 + ((((sel >> 28u) & 1u) != 0u) ? 0x3c179ad8u : 0x2d9fc6b9u); // s111 add - } - r5 = r5 * r0; // 24 mul - r5 = rotl_imm(r5, 19u); // 25 rotl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r6, 2); // 26 shfl - r0 = r0 ^ r5; // 27 xor - r0 = r0 ^ r4; // 28 xor - r3 = r3 - r0; // 29 sub - r5 = r5 * r1; // 30 mul - r7 = r7 ^ ds[r2 & mask]; // 31 load - // per-load shadow sub-block 7 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 7 - for (uint32_t sh7 = 0u; sh7 < 27u; ++sh7) { - r2 = __umulhi(r2, r0); // s112 mulhi - r6 = r6 - r3; // s113 sub - r7 = r6 * r3 + r7; // s114 mad - r5 = rotr_var(r5, r1); // s115 rotr - r6 = rotr_var(r6, r4); // s116 rotr - r2 = r2 + r1 + ((((sel >> 22u) & 1u) != 0u) ? 0x47359729u : 0x502138e2u); // s117 add - r3 = r2 * r0 + r3; // s118 mad - r1 = r1 * r3; // s119 mul - r2 = r0 * r4 + r2; // s120 mad - r0 = r0 * r3; // s121 mul - r2 = __umulhi(r2, r0); // s122 mulhi - r5 = r5 * r6; // s123 mul - r4 = r2 * r4 + r4; // s124 mad - r4 = r4 ^ __shfl_xor_sync(0xffffffffu, r2, 2); // s125 shfl - r2 = r2 ^ r0; // s126 xor - r1 = rotl_imm(r1, 7u); // s127 rotl - } - r1 = r1 ^ ds[r0 & mask]; // 32 load - // per-load shadow sub-block 8 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 8 - for (uint32_t sh8 = 0u; sh8 < 27u; ++sh8) { - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r2, 4); // s128 shfl - r6 = rotl_imm(r6, 17u); // s129 rotl - r2 = r2 + r5 + ((((sel >> 15u) & 1u) != 0u) ? 0x6c57e4e7u : 0x33dff776u); // s130 add - r0 = r0 | r3; // s131 or - r5 = rotr_var(r5, r0); // s132 rotr - r2 = r2 + r3 + ((((sel >> 30u) & 1u) != 0u) ? 0xe8212c0cu : 0x5db25b34u); // s133 add - r4 = r4 ^ r3; // s134 xor - r6 = r6 ^ r1; // s135 xor - r1 = r1 - r7; // s136 sub - r2 = r6 * r2 + r2; // s137 mad - r0 = __umulhi(r0, r5); // s138 mulhi - r2 = r2 + r7 + ((((sel >> 10u) & 1u) != 0u) ? 0xd44ff710u : 0x218d4090u); // s139 add - r1 = rotr_var(r1, r4); // s140 rotr - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r6, 1); // s141 shfl - r7 = r6 * r2 + r7; // s142 mad - r2 = r2 * r3; // s143 mul - } - r5 = r5 ^ r6; // 33 xor - r5 = r5 ^ ds[r1 & mask]; // 34 load - // per-load shadow sub-block 9 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 9 - for (uint32_t sh9 = 0u; sh9 < 27u; ++sh9) { - r7 = r7 + r4 + ((((sel >> 29u) & 1u) != 0u) ? 0xb98942fau : 0xf4b1a8deu); // s144 add - r7 = rotl_imm(r7, 15u); // s145 rotl - r7 = r7 ^ r5; // s146 xor - r4 = r7 * r4 + r4; // s147 mad - r6 = rotr_var(r6, r5); // s148 rotr - r1 = r2 * r4 + r1; // s149 mad - r1 = r1 + r7 + ((((sel >> 17u) & 1u) != 0u) ? 0x0d48ba42u : 0x2bef10f2u); // s150 add - r5 = r5 ^ r4; // s151 xor - r7 = r7 + r0 + ((((sel >> 30u) & 1u) != 0u) ? 0xc98dea9cu : 0xdc5cc080u); // s152 add - r4 = r4 * r6; // s153 mul - r0 = r0 * r2; // s154 mul - r6 = r6 ^ r5; // s155 xor - r4 = rotr_var(r4, r2); // s156 rotr - r1 = rotl_imm(r1, 11u); // s157 rotl - r5 = r5 + r0 + ((((sel >> 11u) & 1u) != 0u) ? 0x6c752dcbu : 0x18197438u); // s158 add - r4 = r1 * r5 + r4; // s159 mad - } - r0 = __umulhi(r0, r5); // 35 mulhi - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r2, 4); // 36 shfl - r7 = r7 ^ ds[r0 & mask]; // 37 load - // per-load shadow sub-block 10 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 10 - for (uint32_t sh10 = 0u; sh10 < 27u; ++sh10) { - r4 = rotl_imm(r4, 26u); // s160 rotl - r3 = r0 * r7 + r3; // s161 mad - r3 = rotr_var(r3, r1); // s162 rotr - r4 = r4 + r0 + ((((sel >> 3u) & 1u) != 0u) ? 0x8e481727u : 0xf1c46574u); // s163 add - r0 = __umulhi(r0, r4); // s164 mulhi - r2 = r2 + r6 + ((((sel >> 20u) & 1u) != 0u) ? 0x550ab406u : 0xac578137u); // s165 add - r0 = rotl_imm(r0, 13u); // s166 rotl - r3 = r3 + r1 + ((((sel >> 14u) & 1u) != 0u) ? 0xaebb5966u : 0xa33e6706u); // s167 add - r3 = r3 | r5; // s168 or - r6 = rotr_var(r6, r2); // s169 rotr - r4 = r4 ^ r6; // s170 xor - r6 = r6 - r1; // s171 sub - r7 = rotl_imm(r7, 22u); // s172 rotl - r5 = rotl_imm(r5, 15u); // s173 rotl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r0, 8); // s174 shfl - r0 = r0 ^ r5; // s175 xor - } - r3 = r3 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0x230c005cu : 0x75ba2fadu); // 38 add - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r5, 4); // 39 shfl - r2 = r2 ^ r5; // 40 xor - r3 = r6 * r3 + r3; // 41 mad - r6 = r6 - r7; // 42 sub - r7 = r7 ^ r0; // 43 xor - r1 = r1 ^ ds[r7 & mask]; // 44 load - // per-load shadow sub-block 11 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 11 - for (uint32_t sh11 = 0u; sh11 < 27u; ++sh11) { - r7 = rotl_imm(r7, 6u); // s176 rotl - r7 = r7 - r0; // s177 sub - r3 = rotl_imm(r3, 30u); // s178 rotl - r7 = r6 * r1 + r7; // s179 mad - r6 = rotl_imm(r6, 9u); // s180 rotl - r2 = r2 ^ r4; // s181 xor - r2 = r2 ^ r7; // s182 xor - r7 = r7 ^ r2; // s183 xor - r1 = r1 + r2 + ((((sel >> 21u) & 1u) != 0u) ? 0x5bb7550fu : 0xd94d55acu); // s184 add - r3 = r3 | r5; // s185 or - r6 = __umulhi(r6, r3); // s186 mulhi - r4 = r4 | r0; // s187 or - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r1, 2); // s188 shfl - r6 = r6 + r5 + ((((sel >> 6u) & 1u) != 0u) ? 0x2a354e2du : 0x89e747feu); // s189 add - r1 = r1 ^ r4; // s190 xor - r7 = __umulhi(r7, r4); // s191 mulhi - } - r2 = r2 * r3; // 45 mul - r1 = __umulhi(r1, r5); // 46 mulhi - r4 = r4 - r3; // 47 sub - r2 = rotr_var(r2, r6); // 48 rotr - r3 = r3 ^ ds[r5 & mask]; // 49 load - // per-load shadow sub-block 12 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 12 - for (uint32_t sh12 = 0u; sh12 < 27u; ++sh12) { - r2 = rotl_imm(r2, 26u); // s192 rotl - r5 = rotl_imm(r5, 8u); // s193 rotl - r4 = r4 ^ __shfl_xor_sync(0xffffffffu, r5, 16); // s194 shfl - r4 = r4 ^ r5; // s195 xor - r1 = r1 * r3; // s196 mul - r5 = r5 + r2 + ((((sel >> 7u) & 1u) != 0u) ? 0x10cfdc71u : 0xea03e8e7u); // s197 add - r7 = r7 + r5 + ((((sel >> 15u) & 1u) != 0u) ? 0x66e148d3u : 0xa7ee0102u); // s198 add - r5 = r5 - r2; // s199 sub - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r1, 2); // s200 shfl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r1, 8); // s201 shfl - r4 = rotr_var(r4, r5); // s202 rotr - r7 = r7 ^ r5; // s203 xor - r7 = r7 ^ r0; // s204 xor - r6 = r6 + r2 + ((((sel >> 10u) & 1u) != 0u) ? 0x8558b619u : 0x8379a4deu); // s205 add - r4 = rotl_imm(r4, 13u); // s206 rotl - r1 = __umulhi(r1, r3); // s207 mulhi - } - r1 = r1 + r5 + ((((sel >> 7u) & 1u) != 0u) ? 0x1907970cu : 0x81b8bc2cu); // 50 add - r0 = r0 * r2; // 51 mul - r0 = r0 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x699fd448u : 0x4f92b968u); // 52 add - r1 = r1 + r0 + ((((sel >> 12u) & 1u) != 0u) ? 0x77b1520du : 0x2bb965afu); // 53 add - r7 = rotl_imm(r7, 14u); // 54 rotl - r3 = r3 + r7 + ((((sel >> 1u) & 1u) != 0u) ? 0xa54c55a0u : 0x7b0fe07au); // 55 add - r6 = r6 ^ ds[r7 & mask]; // 56 load - // per-load shadow sub-block 13 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 13 - for (uint32_t sh13 = 0u; sh13 < 27u; ++sh13) { - r1 = r4 * r4 + r1; // s208 mad - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r0, 4); // s209 shfl - r7 = r7 + r0 + ((((sel >> 11u) & 1u) != 0u) ? 0xf4049c4cu : 0xaa8cb14eu); // s210 add - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r1, 16); // s211 shfl - r1 = r1 ^ r0; // s212 xor - r7 = rotl_imm(r7, 3u); // s213 rotl - r4 = rotr_var(r4, r7); // s214 rotr - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r2, 16); // s215 shfl - r5 = r5 | r1; // s216 or - r1 = r1 - r2; // s217 sub - r6 = r6 - r5; // s218 sub - r6 = rotl_imm(r6, 4u); // s219 rotl - r2 = __umulhi(r2, r0); // s220 mulhi - r2 = r2 | r0; // s221 or - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r2, 8); // s222 shfl - r5 = __umulhi(r5, r6); // s223 mulhi - } - r1 = rotr_var(r1, r5); // 57 rotr - r5 = r5 ^ ds[r4 & mask]; // 58 load - // per-load shadow sub-block 14 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 14 - for (uint32_t sh14 = 0u; sh14 < 27u; ++sh14) { - r0 = r0 - r6; // s224 sub - r7 = rotl_imm(r7, 23u); // s225 rotl - r4 = r4 | r2; // s226 or - r2 = r2 * r4; // s227 mul - r3 = rotl_imm(r3, 12u); // s228 rotl - r0 = rotr_var(r0, r4); // s229 rotr - r0 = r0 + r6 + ((((sel >> 16u) & 1u) != 0u) ? 0x2a7fecb2u : 0x1d176220u); // s230 add - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r2, 4); // s231 shfl - r2 = r2 * r1; // s232 mul - r7 = r0 * r1 + r7; // s233 mad - r5 = rotl_imm(r5, 22u); // s234 rotl - r4 = rotr_var(r4, r6); // s235 rotr - r0 = r5 * r1 + r0; // s236 mad - r6 = r6 ^ r4; // s237 xor - r4 = r4 ^ r6; // s238 xor - r6 = rotl_imm(r6, 18u); // s239 rotl - } - r6 = r6 ^ ds[r2 & mask]; // 59 load - // per-load shadow sub-block 15 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 15 - for (uint32_t sh15 = 0u; sh15 < 27u; ++sh15) { - r4 = r4 + r7 + ((((sel >> 25u) & 1u) != 0u) ? 0xadce39f3u : 0x17dafb4du); // s240 add - r0 = r0 - r7; // s241 sub - r1 = rotr_var(r1, r6); // s242 rotr - r3 = r3 + r0 + ((((sel >> 29u) & 1u) != 0u) ? 0x0e7033b6u : 0xf2f77d26u); // s243 add - r2 = r7 * r4 + r2; // s244 mad - r4 = r0 * r6 + r4; // s245 mad - r5 = r5 * r7; // s246 mul - r3 = r3 + r5 + ((((sel >> 31u) & 1u) != 0u) ? 0x7b2ff6b7u : 0xfed2da4eu); // s247 add - r0 = r0 + r7 + ((((sel >> 0u) & 1u) != 0u) ? 0xb5fad7b1u : 0x03b2891cu); // s248 add - r5 = __umulhi(r5, r4); // s249 mulhi - r5 = r5 + r2 + ((((sel >> 11u) & 1u) != 0u) ? 0xa7e71c2fu : 0x042cd6e3u); // s250 add - r6 = r6 ^ __shfl_xor_sync(0xffffffffu, r1, 1); // s251 shfl - r6 = r6 ^ r1; // s252 xor - r2 = r2 * r5; // s253 mul - r0 = r0 + r6 + ((((sel >> 16u) & 1u) != 0u) ? 0xb83d78deu : 0x784a302bu); // s254 add - r2 = r2 - r4; // s255 sub - } - r3 = r5 * r0 + r3; // 60 mad - r5 = r5 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0xad7493e7u : 0xaf9dd72du); // 61 add - r4 = r4 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x1e07c3d9u : 0x89841d87u); // 62 add - r5 = rotl_imm(r5, 19u); // 63 rotl - } - uint32_t lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint32_t hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((uint64_t)hi << 32) | (uint64_t)lo; -} - -// Host-side launch wrappers. Declared in program.h, called from host.cu. -cudaError_t igneum_launch_cache_fill(uint32_t* cache, uint32_t nSegments) { - if (nSegments == 0u) return cudaErrorInvalidValue; - uint32_t block = 256u; - uint32_t grid = (nSegments + block - 1u) / block; - igneum_cache_fill<<>>(cache, nSegments); - return cudaGetLastError(); -} - -cudaError_t igneum_launch_build(uint32_t* ds, const uint32_t* cache, uint32_t nItems) { - if (nItems == 0u) return cudaErrorInvalidValue; - uint32_t block = 256u; - uint32_t grid = (nItems + block - 1u) / block; - igneum_build<<>>(ds, cache, nItems); - return cudaGetLastError(); -} - -cudaError_t igneum_launch_hash(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask, - uint32_t nonces, uint32_t blockWarps) { - if (blockWarps == 0u || blockWarps > 32u) return cudaErrorInvalidValue; - uint32_t block = 32u * blockWarps; - if (nonces == 0u || (nonces % block) != 0u) return cudaErrorInvalidValue; - igneum_hash<<>>(ds, out, baseNonce, mask); - return cudaGetLastError(); -} - -cudaError_t igneum_hash_info(int* numRegs, int* blocksPerSM, uint32_t blockWarps) { - cudaFuncAttributes attr; - cudaError_t e = cudaFuncGetAttributes(&attr, igneum_hash); - if (e != cudaSuccess) return e; - *numRegs = attr.numRegs; - return cudaOccupancyMaxActiveBlocksPerMultiprocessor(blocksPerSM, igneum_hash, (int)(32u * blockWarps), 0); -} diff --git a/proto-cuda/packs-ca4/mx8_shl256x27/kernel_bound.cl b/proto-cuda/packs-ca4/mx8_shl256x27/kernel_bound.cl deleted file mode 100644 index 9093ee946..000000000 --- a/proto-cuda/packs-ca4/mx8_shl256x27/kernel_bound.cl +++ /dev/null @@ -1,981 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// OpenCL C twin of the Metal kernel for the same seed (see proto-opencl/README.md, WAVEFRONT.md and program.metal). -// Built from source at runtime by proto-opencl/host.c, which passes these defines: -// IGNEUM_GROUP work-group size of igneum_hash, a multiple of 32 (default 32: one work-group = one 32-lane unit) -// IGNEUM_EXCHANGE 0 = local-memory exchange with a barrier (any device, any wave width; the default) -// 1 = sub_group_shuffle_xor (cl_khr_subgroup_shuffle), only with IGNEUM_GROUP 32 and a sub-group size of exactly 32 -// 2 = intel_sub_group_shuffle_xor (cl_intel_subgroups), same condition -// The verification unit is always 32 lanes. A 64-wide hardware wave (AMD GCN/CDNA, RDNA in wave64) runs two units; -// the exchange masks are 1, 2, 4, 8, 16, so every partner lane lies inside the lane's own aligned run of 32. -#ifndef IGNEUM_GROUP -#define IGNEUM_GROUP 32 -#endif -#ifndef IGNEUM_EXCHANGE -#define IGNEUM_EXCHANGE 0 -#endif -#ifdef __OPENCL_VERSION__ -#define IGNEUM_KERNEL_HASH __kernel __attribute__((reqd_work_group_size(IGNEUM_GROUP, 1, 1))) -#define IGNEUM_LOCAL_WORDS(name, n) __local uint name[n] -#if IGNEUM_EXCHANGE == 1 -#ifdef cl_khr_subgroups -#pragma OPENCL EXTENSION cl_khr_subgroups : enable -#endif -#ifdef cl_khr_subgroup_shuffle -#pragma OPENCL EXTENSION cl_khr_subgroup_shuffle : enable -#endif -#elif IGNEUM_EXCHANGE == 2 -#pragma OPENCL EXTENSION cl_intel_subgroups : enable -#endif -#else -// Not an OpenCL compiler: proto-opencl/emu compiles this file as C++ and supplies the built-ins and these two macros. -#include "emu_opencl.h" -#endif - -#if IGNEUM_EXCHANGE == 1 -#define IGNEUM_SHFL_XOR(dst, a, m) dst = sub_group_shuffle_xor((a), (uint)(m)) -#define IGNEUM_BCAST0(dst, a) dst = sub_group_broadcast((a), 0u) -#elif IGNEUM_EXCHANGE == 2 -#define IGNEUM_SHFL_XOR(dst, a, m) dst = intel_sub_group_shuffle_xor((a), (uint)(m)) -#define IGNEUM_BCAST0(dst, a) dst = sub_group_broadcast((a), 0u) -#else -// Local-memory exchange. Two buffers of IGNEUM_GROUP words alternate (xk counts exchanges), so one barrier per -// exchange is enough: a lane can only overwrite buffer b at exchange k+2 after passing barrier k+1, and every lane -// reaches barrier k+1 only after its read of buffer b at exchange k. The partner lid ^ m stays inside the lane's -// aligned run of 32 because m < 32. Control flow is uniform, so every work-item reaches every barrier. -#define IGNEUM_SHFL_XOR(dst, a, m) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid ^ (uint)(m))]; xk += 1u; } -#define IGNEUM_BCAST0(dst, a) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid & ~31u)]; xk += 1u; } -#endif - -static inline uint splitmix32(uint x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -// n is a literal in 1..31 at every call site. OpenCL rotate() rotates left by n modulo 32. -static inline uint rotl_imm(uint x, uint n) { return rotate(x, n); } -// Right rotation by n modulo 32 as a left rotation by (32 - n) modulo 32; n == 0 gives x. -static inline uint rotr_var(uint x, uint n) { return rotate(x, (0u - n) & 31u); } -static inline uint ds_elem(uint i, uint d0, uint d1) { - uint x = i ^ d0; - x *= 0x9E3779B1u; x ^= x >> 15; - x += d1; - x *= 0x85EBCA77u; x ^= x >> 13; - x *= 0xC2B2AE3Du; x ^= x >> 16; - return x; -} - -// Memory-hard dataset core (MEMHARD.md). Cache: 2^26 words in 2^16 segments of 64 chained ChaCha12 lines. -// Item: 8 rounds of 8 x seed-parameterised mixer + one 64-byte cache read, then 8 x final mixer (class v3, mixer multiplier 8, -// docs/plans/mixer-x4.md: the round key of application j of round r is 0x9E3779B9 * (r * m + j + 1)). All parameters are literals. -#define MH_CACHE_LINE_MASK 0x003fffffu -#define MH_SEGMENT_LINES 64u -#define MH_QR(a, b, c, d, r1, r2, r3, r4) { a += b; d ^= a; d = mh_rotl(d, r1); c += d; b ^= c; b = mh_rotl(b, r2); a += b; d ^= a; d = mh_rotl(d, r3); c += d; b ^= c; b = mh_rotl(b, r4); } -static inline uint mh_rotl(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 at every call site - -// y = ChaCha12 core(x) + x -static inline void mh_chacha_block(const uint* x, uint* y) { - for (uint i = 0u; i < 16u; ++i) y[i] = x[i]; - for (uint r = 0u; r < 6u; ++r) { - MH_QR(y[0], y[4], y[8], y[12], 16u, 12u, 8u, 7u) MH_QR(y[1], y[5], y[9], y[13], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[6], y[10], y[14], 16u, 12u, 8u, 7u) MH_QR(y[3], y[7], y[11], y[15], 16u, 12u, 8u, 7u) - MH_QR(y[0], y[5], y[10], y[15], 16u, 12u, 8u, 7u) MH_QR(y[1], y[6], y[11], y[12], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[7], y[8], y[13], 16u, 12u, 8u, 7u) MH_QR(y[3], y[4], y[9], y[14], 16u, 12u, 8u, 7u) - } - for (uint i = 0u; i < 16u; ++i) y[i] += x[i]; -} - -// One cache segment: 64 chained lines written at cache[seg * 1024]. in_j = prev ^ (sigma || K || seg || j || tag), prev_0 = 0. -static inline void mh_cache_segment(__global uint* cache, uint seg) { - uint prev[16]; uint x[16]; uint y[16]; - for (uint i = 0u; i < 16u; ++i) prev[i] = 0u; - for (uint j = 0u; j < MH_SEGMENT_LINES; ++j) { - x[0] = 0x61707865u ^ prev[0]; x[1] = 0x3320646eu ^ prev[1]; x[2] = 0x79622d32u ^ prev[2]; x[3] = 0x6b206574u ^ prev[3]; - x[4] = 0x3067619fu ^ prev[4]; - x[5] = 0x3c269176u ^ prev[5]; - x[6] = 0x84a03b03u ^ prev[6]; - x[7] = 0xf8c63294u ^ prev[7]; - x[8] = 0xff977c5bu ^ prev[8]; - x[9] = 0xe60def3eu ^ prev[9]; - x[10] = 0x63630141u ^ prev[10]; - x[11] = 0xb8fbcb58u ^ prev[11]; - x[12] = seg ^ prev[12]; x[13] = j ^ prev[13]; x[14] = 0x49676e65u ^ prev[14]; x[15] = 0x756d4d48u ^ prev[15]; - mh_chacha_block(x, y); - __global uint* line = cache + ((seg * MH_SEGMENT_LINES + j) * 16u); - for (uint i = 0u; i < 16u; ++i) { line[i] = y[i]; prev[i] = y[i]; } - } -} - -// M_r: per word (s ^ (RC + rk)) * MUL, then a column round and a diagonal round with the seed-drawn rotations. -static inline void mh_mixer(uint* s, uint rk) { - s[0] = (s[0] ^ (0xbab68293u + rk)) * 0x42146205u; - s[1] = (s[1] ^ (0xcc162340u + rk)) * 0x52cbe0fbu; - s[2] = (s[2] ^ (0x6ce151ccu + rk)) * 0x7ecf4a03u; - s[3] = (s[3] ^ (0xe62b8997u + rk)) * 0x6728907fu; - s[4] = (s[4] ^ (0xc9c80297u + rk)) * 0xd81d9751u; - s[5] = (s[5] ^ (0xf74a1654u + rk)) * 0x132952c3u; - s[6] = (s[6] ^ (0x3d704af5u + rk)) * 0xf60de277u; - s[7] = (s[7] ^ (0x3cf522b7u + rk)) * 0x05358035u; - s[8] = (s[8] ^ (0x2b9cac04u + rk)) * 0xbaf6499du; - s[9] = (s[9] ^ (0xa880ac10u + rk)) * 0xe4db9667u; - s[10] = (s[10] ^ (0x13e5dd1du + rk)) * 0x3e98f45du; - s[11] = (s[11] ^ (0x6fc3e233u + rk)) * 0xd0004eddu; - s[12] = (s[12] ^ (0x2d83eeacu + rk)) * 0x2691630du; - s[13] = (s[13] ^ (0x9006e8bfu + rk)) * 0x9beb3bcfu; - s[14] = (s[14] ^ (0x2c4b5362u + rk)) * 0xab310379u; - s[15] = (s[15] ^ (0x31b49ee2u + rk)) * 0x99cfb423u; - MH_QR(s[0], s[4], s[8], s[12], 20u, 20u, 19u, 4u) MH_QR(s[1], s[5], s[9], s[13], 20u, 20u, 19u, 4u) - MH_QR(s[2], s[6], s[10], s[14], 20u, 20u, 19u, 4u) MH_QR(s[3], s[7], s[11], s[15], 20u, 20u, 19u, 4u) - MH_QR(s[0], s[5], s[10], s[15], 26u, 3u, 3u, 27u) MH_QR(s[1], s[6], s[11], s[12], 26u, 3u, 3u, 27u) - MH_QR(s[2], s[7], s[8], s[13], 26u, 3u, 3u, 27u) MH_QR(s[3], s[4], s[9], s[14], 26u, 3u, 3u, 27u) -} - -// Item t: 16 words. s = (K, t * MUL[i] + RC[i]); 8 rounds of 8 x mixer + cache line s[0] & mask; 8 x final mixer. -static inline void mh_item(__global const uint* cache, uint t, uint* s) { - s[0] = 0x3067619fu; - s[1] = 0x3c269176u; - s[2] = 0x84a03b03u; - s[3] = 0xf8c63294u; - s[4] = 0xff977c5bu; - s[5] = 0xe60def3eu; - s[6] = 0x63630141u; - s[7] = 0xb8fbcb58u; - s[8] = t * 0x42146205u + 0xbab68293u; - s[9] = t * 0x52cbe0fbu + 0xcc162340u; - s[10] = t * 0x7ecf4a03u + 0x6ce151ccu; - s[11] = t * 0x6728907fu + 0xe62b8997u; - s[12] = t * 0xd81d9751u + 0xc9c80297u; - s[13] = t * 0x132952c3u + 0xf74a1654u; - s[14] = t * 0xf60de277u + 0x3d704af5u; - s[15] = t * 0x05358035u + 0x3cf522b7u; - for (uint r = 0u; r < 8u; ++r) { - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (r * 8u + j + 1u)); - __global const uint* line = cache + ((s[0] & MH_CACHE_LINE_MASK) * 16u); - for (uint i = 0u; i < 16u; ++i) s[i] ^= line[i]; - } - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (64u + j + 1u)); -} -// dataset[w] without the dataset: derive item w >> 4 and take word w & 15. -static inline uint mh_word(__global const uint* cache, uint w) { uint s[16]; mh_item(cache, w >> 4u, s); return s[w & 15u]; } - -// Memory-hard dataset (MEMHARD.md). One work-item per cache segment; one work-item per 64-byte dataset item. -// The same constants as memhard.h in this pack (one emitter, three dialects). -__kernel void igneum_cache_fill(__global uint* cache, uint nSegments) { - uint seg = (uint)get_global_id(0); - if (seg < nSegments) mh_cache_segment(cache, seg); -} -__kernel void igneum_build(__global uint* ds, __global const uint* cache, uint nItems) { - uint t = (uint)get_global_id(0); - if (t < nItems) { - uint s[16]; - mh_item(cache, t, s); - __global uint* d = ds + ((ulong)t * 16u); - for (uint i = 0u; i < 16u; ++i) d[i] = s[i]; - } -} - -// One hash per work-item. IGNEUM_GROUP is a multiple of 32; lane = lid & 31 and every exchange stays inside the -// lane's own aligned run of 32 work-items, exactly like simd_shuffle_xor inside a 32-wide Metal SIMD group and -// __shfl_xor_sync inside a CUDA warp. Control flow is uniform (no branches at all). -IGNEUM_KERNEL_HASH void igneum_hash(__global const uint* ds, __global ulong* out, uint baseNonce, uint mask) { - uint gid = (uint)get_global_id(0); - uint lid = (uint)get_local_id(0); - uint nonce = baseNonce + gid; - uint r0, r1, r2, r3, r4, r5, r6, r7; -#if IGNEUM_EXCHANGE == 0 - IGNEUM_LOCAL_WORDS(xch, 2 * IGNEUM_GROUP); - uint xk = 0u; -#else - (void)lid; -#endif - { uint x = nonce ^ 0x67a9a7beu; x += 0x9e3779b9u; x = splitmix32(x); r0 = x ^ 0x1a155b25u; } // SEEDW[0], 0x9e3779b9u * 1u, SEEDW[1] - { uint x = nonce ^ 0x1a155b25u; x += 0x3c6ef372u; x = splitmix32(x); r1 = x ^ 0xfddfb732u; } // SEEDW[1], 0x9e3779b9u * 2u, SEEDW[2] - { uint x = nonce ^ 0xfddfb732u; x += 0xdaa66d2bu; x = splitmix32(x); r2 = x ^ 0x4b5af2e8u; } // SEEDW[2], 0x9e3779b9u * 3u, SEEDW[3] - { uint x = nonce ^ 0x4b5af2e8u; x += 0x78dde6e4u; x = splitmix32(x); r3 = x ^ 0xc55caf33u; } // SEEDW[3], 0x9e3779b9u * 4u, SEEDW[4] - { uint x = nonce ^ 0xc55caf33u; x += 0x1715609du; x = splitmix32(x); r4 = x ^ 0xa27c13b7u; } // SEEDW[4], 0x9e3779b9u * 5u, SEEDW[5] - { uint x = nonce ^ 0xa27c13b7u; x += 0xb54cda56u; x = splitmix32(x); r5 = x ^ 0x06628a48u; } // SEEDW[5], 0x9e3779b9u * 6u, SEEDW[6] - { uint x = nonce ^ 0x06628a48u; x += 0x5384540fu; x = splitmix32(x); r6 = x ^ 0x03852469u; } // SEEDW[6], 0x9e3779b9u * 7u, SEEDW[7] - { uint x = nonce ^ 0x03852469u; x += 0xf1bbcdc8u; x = splitmix32(x); r7 = x ^ 0x67a9a7beu; } // SEEDW[7], 0x9e3779b9u * 8u, SEEDW[0] - - for (uint it = 0u; it < 8u; ++it) { - uint sel = r0; - r2 = r3 * r4 + r2; // 0 mad - r2 = r1 * r1 + r2; // 1 mad - r2 = r3 * r2 + r2; // 2 mad - r3 = r3 ^ r5; // 3 xor - r7 = r7 ^ ds[r2 & mask]; // 4 load - // per-load shadow sub-block 0 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 0 - for (uint sh0 = 0u; sh0 < 27u; ++sh0) { - r5 = r5 + r2 + ((((sel >> 18u) & 1u) != 0u) ? 0x8bc12da9u : 0x92199f99u); // s0 add - r0 = r0 + r7 + ((((sel >> 26u) & 1u) != 0u) ? 0x63079e5au : 0x8d72d3adu); // s1 add - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 2u); r6 = r6 ^ t_; } // s2 shfl - r4 = r4 - r2; // s3 sub - r7 = r7 + r0 + ((((sel >> 31u) & 1u) != 0u) ? 0x5d4c7a60u : 0xb21b4babu); // s4 add - r0 = rotl_imm(r0, 11u); // s5 rotl - r0 = r0 + r1 + ((((sel >> 16u) & 1u) != 0u) ? 0xc88e2942u : 0x2fe0e98bu); // s6 add - r7 = r7 ^ r1; // s7 xor - r1 = r6 * r5 + r1; // s8 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 4u); r6 = r6 ^ t_; } // s9 shfl - r1 = r2 * r2 + r1; // s10 mad - r5 = r0 * r3 + r5; // s11 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 4u); r2 = r2 ^ t_; } // s12 shfl - r1 = r1 + r5 + ((((sel >> 25u) & 1u) != 0u) ? 0xbc48c63eu : 0xbdf8f9a5u); // s13 add - r1 = rotl_imm(r1, 29u); // s14 rotl - r1 = r1 - r4; // s15 sub - } - r5 = r5 ^ ds[r7 & mask]; // 5 load - // per-load shadow sub-block 1 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 1 - for (uint sh1 = 0u; sh1 < 27u; ++sh1) { - r7 = r7 | r1; // s16 or - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 8u); r2 = r2 ^ t_; } // s17 shfl - r7 = r7 ^ r4; // s18 xor - r6 = r6 * r1; // s19 mul - r5 = r6 * r0 + r5; // s20 mad - r3 = r3 - r1; // s21 sub - r6 = r6 * r0; // s22 mul - r2 = r2 + r0 + ((((sel >> 1u) & 1u) != 0u) ? 0x96e8f127u : 0x45c37cecu); // s23 add - r6 = r6 - r4; // s24 sub - r7 = r3 * r4 + r7; // s25 mad - r3 = rotl_imm(r3, 9u); // s26 rotl - r2 = r2 - r1; // s27 sub - r6 = mul_hi(r6, r0); // s28 mulhi - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 2u); r2 = r2 ^ t_; } // s29 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 1u); r1 = r1 ^ t_; } // s30 shfl - r1 = r1 + r6 + ((((sel >> 1u) & 1u) != 0u) ? 0xb1cdb2abu : 0x37985632u); // s31 add - } - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 8u); r1 = r1 ^ t_; } // 6 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 8u); r7 = r7 ^ t_; } // 7 shfl - r1 = mul_hi(r1, r5); // 8 mulhi - r6 = rotr_var(r6, r3); // 9 rotr - r3 = r3 | r4; // 10 or - r4 = r4 ^ ds[r3 & mask]; // 11 load - // per-load shadow sub-block 2 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 2 - for (uint sh2 = 0u; sh2 < 27u; ++sh2) { - r0 = rotr_var(r0, r1); // s32 rotr - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 2u); r3 = r3 ^ t_; } // s33 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 4u); r0 = r0 ^ t_; } // s34 shfl - r7 = r7 * r0; // s35 mul - r3 = r3 + r1 + ((((sel >> 0u) & 1u) != 0u) ? 0x856e0180u : 0x804e777eu); // s36 add - r1 = r1 ^ r6; // s37 xor - r2 = r2 * r7; // s38 mul - r6 = r6 + r5 + ((((sel >> 2u) & 1u) != 0u) ? 0xe9bd0cc4u : 0x0b06c8a7u); // s39 add - r5 = r5 * r7; // s40 mul - r2 = mul_hi(r2, r3); // s41 mulhi - r2 = r2 ^ r0; // s42 xor - r0 = rotl_imm(r0, 4u); // s43 rotl - r4 = mul_hi(r4, r3); // s44 mulhi - r6 = r6 + r7 + ((((sel >> 12u) & 1u) != 0u) ? 0xcdcb63f6u : 0xee822e17u); // s45 add - r3 = r2 * r0 + r3; // s46 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 8u); r4 = r4 ^ t_; } // s47 shfl - } - r0 = mul_hi(r0, r4); // 12 mulhi - r5 = r5 + r1 + ((((sel >> 30u) & 1u) != 0u) ? 0xd3177981u : 0xc7934706u); // 13 add - r0 = r0 ^ ds[r4 & mask]; // 14 load - // per-load shadow sub-block 3 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 3 - for (uint sh3 = 0u; sh3 < 27u; ++sh3) { - r6 = mul_hi(r6, r5); // s48 mulhi - r0 = r0 - r2; // s49 sub - r3 = r3 - r5; // s50 sub - r1 = rotr_var(r1, r4); // s51 rotr - r6 = r7 * r7 + r6; // s52 mad - r5 = r5 ^ r3; // s53 xor - r1 = r1 - r0; // s54 sub - r5 = r5 - r6; // s55 sub - r3 = r3 + r2 + ((((sel >> 10u) & 1u) != 0u) ? 0xd4758987u : 0x3dfad1b6u); // s56 add - r4 = r4 + r1 + ((((sel >> 0u) & 1u) != 0u) ? 0x0602d6beu : 0xfca75bc2u); // s57 add - r5 = mul_hi(r5, r6); // s58 mulhi - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 2u); r2 = r2 ^ t_; } // s59 shfl - r2 = rotl_imm(r2, 9u); // s60 rotl - r4 = r4 | r6; // s61 or - r6 = rotr_var(r6, r4); // s62 rotr - r2 = r2 * r4; // s63 mul - } - r2 = r2 - r4; // 15 sub - r2 = r2 ^ ds[r0 & mask]; // 16 load - // per-load shadow sub-block 4 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 4 - for (uint sh4 = 0u; sh4 < 27u; ++sh4) { - r0 = r0 ^ r5; // s64 xor - r2 = r2 ^ r7; // s65 xor - r2 = r2 + r1 + ((((sel >> 6u) & 1u) != 0u) ? 0x8596687au : 0xd71c02ffu); // s66 add - r1 = rotl_imm(r1, 21u); // s67 rotl - r3 = r3 * r2; // s68 mul - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 2u); r7 = r7 ^ t_; } // s69 shfl - r3 = r3 * r2; // s70 mul - r0 = r2 * r5 + r0; // s71 mad - r6 = r6 + r7 + ((((sel >> 0u) & 1u) != 0u) ? 0x08ac5733u : 0x3c6fe15du); // s72 add - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 8u); r3 = r3 ^ t_; } // s73 shfl - r3 = r3 * r6; // s74 mul - r6 = r6 ^ r7; // s75 xor - r3 = r3 | r0; // s76 or - r2 = r2 + r4 + ((((sel >> 1u) & 1u) != 0u) ? 0xc100b495u : 0x295d5faeu); // s77 add - r3 = mul_hi(r3, r7); // s78 mulhi - r4 = r4 | r1; // s79 or - } - r7 = r7 ^ ds[r2 & mask]; // 17 load - // per-load shadow sub-block 5 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 5 - for (uint sh5 = 0u; sh5 < 27u; ++sh5) { - r4 = rotr_var(r4, r3); // s80 rotr - r4 = r4 + r3 + ((((sel >> 15u) & 1u) != 0u) ? 0x5cc59530u : 0x274a9221u); // s81 add - r7 = r7 + r3 + ((((sel >> 5u) & 1u) != 0u) ? 0x141479ecu : 0xc8651f8eu); // s82 add - r3 = rotr_var(r3, r6); // s83 rotr - r2 = r4 * r7 + r2; // s84 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 16u); r2 = r2 ^ t_; } // s85 shfl - r3 = r3 ^ r2; // s86 xor - { uint t_; IGNEUM_SHFL_XOR(t_, r7, 2u); r5 = r5 ^ t_; } // s87 shfl - r0 = r0 ^ r4; // s88 xor - r3 = r3 + r2 + ((((sel >> 18u) & 1u) != 0u) ? 0x883c0c92u : 0x53f915c2u); // s89 add - r7 = rotr_var(r7, r5); // s90 rotr - r3 = r3 + r1 + ((((sel >> 31u) & 1u) != 0u) ? 0x3cc5bd20u : 0xe2be00a5u); // s91 add - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 1u); r7 = r7 ^ t_; } // s92 shfl - r6 = rotr_var(r6, r2); // s93 rotr - r7 = rotl_imm(r7, 14u); // s94 rotl - r4 = r5 * r5 + r4; // s95 mad - } - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 4u); r7 = r7 ^ t_; } // 18 shfl - r5 = r5 * r0; // 19 mul - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 2u); r3 = r3 ^ t_; } // 20 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 16u); r2 = r2 ^ t_; } // 21 shfl - r6 = mul_hi(r6, r2); // 22 mulhi - r6 = r6 ^ ds[r1 & mask]; // 23 load - // per-load shadow sub-block 6 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 6 - for (uint sh6 = 0u; sh6 < 27u; ++sh6) { - r2 = r4 * r5 + r2; // s96 mad - r1 = r1 ^ r0; // s97 xor - r5 = r5 * r4; // s98 mul - r2 = r2 - r0; // s99 sub - r7 = rotl_imm(r7, 30u); // s100 rotl - r5 = r5 + r6 + ((((sel >> 17u) & 1u) != 0u) ? 0xbfd646cbu : 0x423fd9c9u); // s101 add - r7 = r7 + r6 + ((((sel >> 11u) & 1u) != 0u) ? 0x19b74a43u : 0x8d3c011du); // s102 add - r5 = rotl_imm(r5, 6u); // s103 rotl - r0 = r0 * r4; // s104 mul - r0 = r0 | r5; // s105 or - r0 = r0 + r1 + ((((sel >> 15u) & 1u) != 0u) ? 0x7dcb7e18u : 0x8ce14721u); // s106 add - r0 = rotl_imm(r0, 15u); // s107 rotl - r4 = r4 - r2; // s108 sub - r2 = mul_hi(r2, r0); // s109 mulhi - r1 = mul_hi(r1, r0); // s110 mulhi - r0 = r0 + r2 + ((((sel >> 28u) & 1u) != 0u) ? 0x3c179ad8u : 0x2d9fc6b9u); // s111 add - } - r5 = r5 * r0; // 24 mul - r5 = rotl_imm(r5, 19u); // 25 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 2u); r7 = r7 ^ t_; } // 26 shfl - r0 = r0 ^ r5; // 27 xor - r0 = r0 ^ r4; // 28 xor - r3 = r3 - r0; // 29 sub - r5 = r5 * r1; // 30 mul - r7 = r7 ^ ds[r2 & mask]; // 31 load - // per-load shadow sub-block 7 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 7 - for (uint sh7 = 0u; sh7 < 27u; ++sh7) { - r2 = mul_hi(r2, r0); // s112 mulhi - r6 = r6 - r3; // s113 sub - r7 = r6 * r3 + r7; // s114 mad - r5 = rotr_var(r5, r1); // s115 rotr - r6 = rotr_var(r6, r4); // s116 rotr - r2 = r2 + r1 + ((((sel >> 22u) & 1u) != 0u) ? 0x47359729u : 0x502138e2u); // s117 add - r3 = r2 * r0 + r3; // s118 mad - r1 = r1 * r3; // s119 mul - r2 = r0 * r4 + r2; // s120 mad - r0 = r0 * r3; // s121 mul - r2 = mul_hi(r2, r0); // s122 mulhi - r5 = r5 * r6; // s123 mul - r4 = r2 * r4 + r4; // s124 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 2u); r4 = r4 ^ t_; } // s125 shfl - r2 = r2 ^ r0; // s126 xor - r1 = rotl_imm(r1, 7u); // s127 rotl - } - r1 = r1 ^ ds[r0 & mask]; // 32 load - // per-load shadow sub-block 8 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 8 - for (uint sh8 = 0u; sh8 < 27u; ++sh8) { - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 4u); r7 = r7 ^ t_; } // s128 shfl - r6 = rotl_imm(r6, 17u); // s129 rotl - r2 = r2 + r5 + ((((sel >> 15u) & 1u) != 0u) ? 0x6c57e4e7u : 0x33dff776u); // s130 add - r0 = r0 | r3; // s131 or - r5 = rotr_var(r5, r0); // s132 rotr - r2 = r2 + r3 + ((((sel >> 30u) & 1u) != 0u) ? 0xe8212c0cu : 0x5db25b34u); // s133 add - r4 = r4 ^ r3; // s134 xor - r6 = r6 ^ r1; // s135 xor - r1 = r1 - r7; // s136 sub - r2 = r6 * r2 + r2; // s137 mad - r0 = mul_hi(r0, r5); // s138 mulhi - r2 = r2 + r7 + ((((sel >> 10u) & 1u) != 0u) ? 0xd44ff710u : 0x218d4090u); // s139 add - r1 = rotr_var(r1, r4); // s140 rotr - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 1u); r3 = r3 ^ t_; } // s141 shfl - r7 = r6 * r2 + r7; // s142 mad - r2 = r2 * r3; // s143 mul - } - r5 = r5 ^ r6; // 33 xor - r5 = r5 ^ ds[r1 & mask]; // 34 load - // per-load shadow sub-block 9 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 9 - for (uint sh9 = 0u; sh9 < 27u; ++sh9) { - r7 = r7 + r4 + ((((sel >> 29u) & 1u) != 0u) ? 0xb98942fau : 0xf4b1a8deu); // s144 add - r7 = rotl_imm(r7, 15u); // s145 rotl - r7 = r7 ^ r5; // s146 xor - r4 = r7 * r4 + r4; // s147 mad - r6 = rotr_var(r6, r5); // s148 rotr - r1 = r2 * r4 + r1; // s149 mad - r1 = r1 + r7 + ((((sel >> 17u) & 1u) != 0u) ? 0x0d48ba42u : 0x2bef10f2u); // s150 add - r5 = r5 ^ r4; // s151 xor - r7 = r7 + r0 + ((((sel >> 30u) & 1u) != 0u) ? 0xc98dea9cu : 0xdc5cc080u); // s152 add - r4 = r4 * r6; // s153 mul - r0 = r0 * r2; // s154 mul - r6 = r6 ^ r5; // s155 xor - r4 = rotr_var(r4, r2); // s156 rotr - r1 = rotl_imm(r1, 11u); // s157 rotl - r5 = r5 + r0 + ((((sel >> 11u) & 1u) != 0u) ? 0x6c752dcbu : 0x18197438u); // s158 add - r4 = r1 * r5 + r4; // s159 mad - } - r0 = mul_hi(r0, r5); // 35 mulhi - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 4u); r5 = r5 ^ t_; } // 36 shfl - r7 = r7 ^ ds[r0 & mask]; // 37 load - // per-load shadow sub-block 10 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 10 - for (uint sh10 = 0u; sh10 < 27u; ++sh10) { - r4 = rotl_imm(r4, 26u); // s160 rotl - r3 = r0 * r7 + r3; // s161 mad - r3 = rotr_var(r3, r1); // s162 rotr - r4 = r4 + r0 + ((((sel >> 3u) & 1u) != 0u) ? 0x8e481727u : 0xf1c46574u); // s163 add - r0 = mul_hi(r0, r4); // s164 mulhi - r2 = r2 + r6 + ((((sel >> 20u) & 1u) != 0u) ? 0x550ab406u : 0xac578137u); // s165 add - r0 = rotl_imm(r0, 13u); // s166 rotl - r3 = r3 + r1 + ((((sel >> 14u) & 1u) != 0u) ? 0xaebb5966u : 0xa33e6706u); // s167 add - r3 = r3 | r5; // s168 or - r6 = rotr_var(r6, r2); // s169 rotr - r4 = r4 ^ r6; // s170 xor - r6 = r6 - r1; // s171 sub - r7 = rotl_imm(r7, 22u); // s172 rotl - r5 = rotl_imm(r5, 15u); // s173 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r0, 8u); r7 = r7 ^ t_; } // s174 shfl - r0 = r0 ^ r5; // s175 xor - } - r3 = r3 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0x230c005cu : 0x75ba2fadu); // 38 add - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 4u); r1 = r1 ^ t_; } // 39 shfl - r2 = r2 ^ r5; // 40 xor - r3 = r6 * r3 + r3; // 41 mad - r6 = r6 - r7; // 42 sub - r7 = r7 ^ r0; // 43 xor - r1 = r1 ^ ds[r7 & mask]; // 44 load - // per-load shadow sub-block 11 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 11 - for (uint sh11 = 0u; sh11 < 27u; ++sh11) { - r7 = rotl_imm(r7, 6u); // s176 rotl - r7 = r7 - r0; // s177 sub - r3 = rotl_imm(r3, 30u); // s178 rotl - r7 = r6 * r1 + r7; // s179 mad - r6 = rotl_imm(r6, 9u); // s180 rotl - r2 = r2 ^ r4; // s181 xor - r2 = r2 ^ r7; // s182 xor - r7 = r7 ^ r2; // s183 xor - r1 = r1 + r2 + ((((sel >> 21u) & 1u) != 0u) ? 0x5bb7550fu : 0xd94d55acu); // s184 add - r3 = r3 | r5; // s185 or - r6 = mul_hi(r6, r3); // s186 mulhi - r4 = r4 | r0; // s187 or - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 2u); r7 = r7 ^ t_; } // s188 shfl - r6 = r6 + r5 + ((((sel >> 6u) & 1u) != 0u) ? 0x2a354e2du : 0x89e747feu); // s189 add - r1 = r1 ^ r4; // s190 xor - r7 = mul_hi(r7, r4); // s191 mulhi - } - r2 = r2 * r3; // 45 mul - r1 = mul_hi(r1, r5); // 46 mulhi - r4 = r4 - r3; // 47 sub - r2 = rotr_var(r2, r6); // 48 rotr - r3 = r3 ^ ds[r5 & mask]; // 49 load - // per-load shadow sub-block 12 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 12 - for (uint sh12 = 0u; sh12 < 27u; ++sh12) { - r2 = rotl_imm(r2, 26u); // s192 rotl - r5 = rotl_imm(r5, 8u); // s193 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 16u); r4 = r4 ^ t_; } // s194 shfl - r4 = r4 ^ r5; // s195 xor - r1 = r1 * r3; // s196 mul - r5 = r5 + r2 + ((((sel >> 7u) & 1u) != 0u) ? 0x10cfdc71u : 0xea03e8e7u); // s197 add - r7 = r7 + r5 + ((((sel >> 15u) & 1u) != 0u) ? 0x66e148d3u : 0xa7ee0102u); // s198 add - r5 = r5 - r2; // s199 sub - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 2u); r5 = r5 ^ t_; } // s200 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 8u); r7 = r7 ^ t_; } // s201 shfl - r4 = rotr_var(r4, r5); // s202 rotr - r7 = r7 ^ r5; // s203 xor - r7 = r7 ^ r0; // s204 xor - r6 = r6 + r2 + ((((sel >> 10u) & 1u) != 0u) ? 0x8558b619u : 0x8379a4deu); // s205 add - r4 = rotl_imm(r4, 13u); // s206 rotl - r1 = mul_hi(r1, r3); // s207 mulhi - } - r1 = r1 + r5 + ((((sel >> 7u) & 1u) != 0u) ? 0x1907970cu : 0x81b8bc2cu); // 50 add - r0 = r0 * r2; // 51 mul - r0 = r0 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x699fd448u : 0x4f92b968u); // 52 add - r1 = r1 + r0 + ((((sel >> 12u) & 1u) != 0u) ? 0x77b1520du : 0x2bb965afu); // 53 add - r7 = rotl_imm(r7, 14u); // 54 rotl - r3 = r3 + r7 + ((((sel >> 1u) & 1u) != 0u) ? 0xa54c55a0u : 0x7b0fe07au); // 55 add - r6 = r6 ^ ds[r7 & mask]; // 56 load - // per-load shadow sub-block 13 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 13 - for (uint sh13 = 0u; sh13 < 27u; ++sh13) { - r1 = r4 * r4 + r1; // s208 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r0, 4u); r2 = r2 ^ t_; } // s209 shfl - r7 = r7 + r0 + ((((sel >> 11u) & 1u) != 0u) ? 0xf4049c4cu : 0xaa8cb14eu); // s210 add - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 16u); r7 = r7 ^ t_; } // s211 shfl - r1 = r1 ^ r0; // s212 xor - r7 = rotl_imm(r7, 3u); // s213 rotl - r4 = rotr_var(r4, r7); // s214 rotr - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 16u); r3 = r3 ^ t_; } // s215 shfl - r5 = r5 | r1; // s216 or - r1 = r1 - r2; // s217 sub - r6 = r6 - r5; // s218 sub - r6 = rotl_imm(r6, 4u); // s219 rotl - r2 = mul_hi(r2, r0); // s220 mulhi - r2 = r2 | r0; // s221 or - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 8u); r5 = r5 ^ t_; } // s222 shfl - r5 = mul_hi(r5, r6); // s223 mulhi - } - r1 = rotr_var(r1, r5); // 57 rotr - r5 = r5 ^ ds[r4 & mask]; // 58 load - // per-load shadow sub-block 14 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 14 - for (uint sh14 = 0u; sh14 < 27u; ++sh14) { - r0 = r0 - r6; // s224 sub - r7 = rotl_imm(r7, 23u); // s225 rotl - r4 = r4 | r2; // s226 or - r2 = r2 * r4; // s227 mul - r3 = rotl_imm(r3, 12u); // s228 rotl - r0 = rotr_var(r0, r4); // s229 rotr - r0 = r0 + r6 + ((((sel >> 16u) & 1u) != 0u) ? 0x2a7fecb2u : 0x1d176220u); // s230 add - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 4u); r1 = r1 ^ t_; } // s231 shfl - r2 = r2 * r1; // s232 mul - r7 = r0 * r1 + r7; // s233 mad - r5 = rotl_imm(r5, 22u); // s234 rotl - r4 = rotr_var(r4, r6); // s235 rotr - r0 = r5 * r1 + r0; // s236 mad - r6 = r6 ^ r4; // s237 xor - r4 = r4 ^ r6; // s238 xor - r6 = rotl_imm(r6, 18u); // s239 rotl - } - r6 = r6 ^ ds[r2 & mask]; // 59 load - // per-load shadow sub-block 15 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 15 - for (uint sh15 = 0u; sh15 < 27u; ++sh15) { - r4 = r4 + r7 + ((((sel >> 25u) & 1u) != 0u) ? 0xadce39f3u : 0x17dafb4du); // s240 add - r0 = r0 - r7; // s241 sub - r1 = rotr_var(r1, r6); // s242 rotr - r3 = r3 + r0 + ((((sel >> 29u) & 1u) != 0u) ? 0x0e7033b6u : 0xf2f77d26u); // s243 add - r2 = r7 * r4 + r2; // s244 mad - r4 = r0 * r6 + r4; // s245 mad - r5 = r5 * r7; // s246 mul - r3 = r3 + r5 + ((((sel >> 31u) & 1u) != 0u) ? 0x7b2ff6b7u : 0xfed2da4eu); // s247 add - r0 = r0 + r7 + ((((sel >> 0u) & 1u) != 0u) ? 0xb5fad7b1u : 0x03b2891cu); // s248 add - r5 = mul_hi(r5, r4); // s249 mulhi - r5 = r5 + r2 + ((((sel >> 11u) & 1u) != 0u) ? 0xa7e71c2fu : 0x042cd6e3u); // s250 add - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 1u); r6 = r6 ^ t_; } // s251 shfl - r6 = r6 ^ r1; // s252 xor - r2 = r2 * r5; // s253 mul - r0 = r0 + r6 + ((((sel >> 16u) & 1u) != 0u) ? 0xb83d78deu : 0x784a302bu); // s254 add - r2 = r2 - r4; // s255 sub - } - r3 = r5 * r0 + r3; // 60 mad - r5 = r5 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0xad7493e7u : 0xaf9dd72du); // 61 add - r4 = r4 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x1e07c3d9u : 0x89841d87u); // 62 add - r5 = rotl_imm(r5, 19u); // 63 rotl - } - uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((ulong)hi << 32) | (ulong)lo; -} - -#if IGNEUM_EXCHANGE != 0 -// Reports the sub-group size this device uses for a work-group of IGNEUM_GROUP items. host.c runs it only when the -// per-kernel query (clGetKernelSubGroupInfoKHR on igneum_hash) is unavailable; that query is preferred because a -// compiler may pick a different wave width per kernel (RDNA: wave32 or wave64). See WAVEFRONT.md. -IGNEUM_KERNEL_HASH void igneum_probe_subgroup(__global uint* out) { - if (get_local_id(0) == 0u) { out[0] = get_sub_group_size(); out[1] = get_num_sub_groups(); } -} -#endif - -// Header-bound variant (bind.rs): the init words come from initw, not SEEDW. Same body as igneum_hash. -IGNEUM_KERNEL_HASH void igneum_hash_bound(__global const uint* ds, __global ulong* out, uint baseNonce, uint mask, __global const uint* initw) { - uint gid = (uint)get_global_id(0); - uint lid = (uint)get_local_id(0); - uint nonce = baseNonce + gid; - uint r0, r1, r2, r3, r4, r5, r6, r7; - uint iw0 = initw[0], iw1 = initw[1], iw2 = initw[2], iw3 = initw[3], iw4 = initw[4], iw5 = initw[5], iw6 = initw[6], iw7 = initw[7]; -#if IGNEUM_EXCHANGE == 0 - IGNEUM_LOCAL_WORDS(xch, 2 * IGNEUM_GROUP); - uint xk = 0u; -#else - (void)lid; -#endif - { uint x = nonce ^ iw0; x += 0x9e3779b9u * 1u; x = splitmix32(x); r0 = x ^ iw1; } - { uint x = nonce ^ iw1; x += 0x9e3779b9u * 2u; x = splitmix32(x); r1 = x ^ iw2; } - { uint x = nonce ^ iw2; x += 0x9e3779b9u * 3u; x = splitmix32(x); r2 = x ^ iw3; } - { uint x = nonce ^ iw3; x += 0x9e3779b9u * 4u; x = splitmix32(x); r3 = x ^ iw4; } - { uint x = nonce ^ iw4; x += 0x9e3779b9u * 5u; x = splitmix32(x); r4 = x ^ iw5; } - { uint x = nonce ^ iw5; x += 0x9e3779b9u * 6u; x = splitmix32(x); r5 = x ^ iw6; } - { uint x = nonce ^ iw6; x += 0x9e3779b9u * 7u; x = splitmix32(x); r6 = x ^ iw7; } - { uint x = nonce ^ iw7; x += 0x9e3779b9u * 8u; x = splitmix32(x); r7 = x ^ iw0; } - - for (uint it = 0u; it < 8u; ++it) { - uint sel = r0; - r2 = r3 * r4 + r2; // 0 mad - r2 = r1 * r1 + r2; // 1 mad - r2 = r3 * r2 + r2; // 2 mad - r3 = r3 ^ r5; // 3 xor - r7 = r7 ^ ds[r2 & mask]; // 4 load - // per-load shadow sub-block 0 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 0 - for (uint sh0 = 0u; sh0 < 27u; ++sh0) { - r5 = r5 + r2 + ((((sel >> 18u) & 1u) != 0u) ? 0x8bc12da9u : 0x92199f99u); // s0 add - r0 = r0 + r7 + ((((sel >> 26u) & 1u) != 0u) ? 0x63079e5au : 0x8d72d3adu); // s1 add - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 2u); r6 = r6 ^ t_; } // s2 shfl - r4 = r4 - r2; // s3 sub - r7 = r7 + r0 + ((((sel >> 31u) & 1u) != 0u) ? 0x5d4c7a60u : 0xb21b4babu); // s4 add - r0 = rotl_imm(r0, 11u); // s5 rotl - r0 = r0 + r1 + ((((sel >> 16u) & 1u) != 0u) ? 0xc88e2942u : 0x2fe0e98bu); // s6 add - r7 = r7 ^ r1; // s7 xor - r1 = r6 * r5 + r1; // s8 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 4u); r6 = r6 ^ t_; } // s9 shfl - r1 = r2 * r2 + r1; // s10 mad - r5 = r0 * r3 + r5; // s11 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 4u); r2 = r2 ^ t_; } // s12 shfl - r1 = r1 + r5 + ((((sel >> 25u) & 1u) != 0u) ? 0xbc48c63eu : 0xbdf8f9a5u); // s13 add - r1 = rotl_imm(r1, 29u); // s14 rotl - r1 = r1 - r4; // s15 sub - } - r5 = r5 ^ ds[r7 & mask]; // 5 load - // per-load shadow sub-block 1 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 1 - for (uint sh1 = 0u; sh1 < 27u; ++sh1) { - r7 = r7 | r1; // s16 or - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 8u); r2 = r2 ^ t_; } // s17 shfl - r7 = r7 ^ r4; // s18 xor - r6 = r6 * r1; // s19 mul - r5 = r6 * r0 + r5; // s20 mad - r3 = r3 - r1; // s21 sub - r6 = r6 * r0; // s22 mul - r2 = r2 + r0 + ((((sel >> 1u) & 1u) != 0u) ? 0x96e8f127u : 0x45c37cecu); // s23 add - r6 = r6 - r4; // s24 sub - r7 = r3 * r4 + r7; // s25 mad - r3 = rotl_imm(r3, 9u); // s26 rotl - r2 = r2 - r1; // s27 sub - r6 = mul_hi(r6, r0); // s28 mulhi - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 2u); r2 = r2 ^ t_; } // s29 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 1u); r1 = r1 ^ t_; } // s30 shfl - r1 = r1 + r6 + ((((sel >> 1u) & 1u) != 0u) ? 0xb1cdb2abu : 0x37985632u); // s31 add - } - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 8u); r1 = r1 ^ t_; } // 6 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 8u); r7 = r7 ^ t_; } // 7 shfl - r1 = mul_hi(r1, r5); // 8 mulhi - r6 = rotr_var(r6, r3); // 9 rotr - r3 = r3 | r4; // 10 or - r4 = r4 ^ ds[r3 & mask]; // 11 load - // per-load shadow sub-block 2 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 2 - for (uint sh2 = 0u; sh2 < 27u; ++sh2) { - r0 = rotr_var(r0, r1); // s32 rotr - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 2u); r3 = r3 ^ t_; } // s33 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 4u); r0 = r0 ^ t_; } // s34 shfl - r7 = r7 * r0; // s35 mul - r3 = r3 + r1 + ((((sel >> 0u) & 1u) != 0u) ? 0x856e0180u : 0x804e777eu); // s36 add - r1 = r1 ^ r6; // s37 xor - r2 = r2 * r7; // s38 mul - r6 = r6 + r5 + ((((sel >> 2u) & 1u) != 0u) ? 0xe9bd0cc4u : 0x0b06c8a7u); // s39 add - r5 = r5 * r7; // s40 mul - r2 = mul_hi(r2, r3); // s41 mulhi - r2 = r2 ^ r0; // s42 xor - r0 = rotl_imm(r0, 4u); // s43 rotl - r4 = mul_hi(r4, r3); // s44 mulhi - r6 = r6 + r7 + ((((sel >> 12u) & 1u) != 0u) ? 0xcdcb63f6u : 0xee822e17u); // s45 add - r3 = r2 * r0 + r3; // s46 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 8u); r4 = r4 ^ t_; } // s47 shfl - } - r0 = mul_hi(r0, r4); // 12 mulhi - r5 = r5 + r1 + ((((sel >> 30u) & 1u) != 0u) ? 0xd3177981u : 0xc7934706u); // 13 add - r0 = r0 ^ ds[r4 & mask]; // 14 load - // per-load shadow sub-block 3 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 3 - for (uint sh3 = 0u; sh3 < 27u; ++sh3) { - r6 = mul_hi(r6, r5); // s48 mulhi - r0 = r0 - r2; // s49 sub - r3 = r3 - r5; // s50 sub - r1 = rotr_var(r1, r4); // s51 rotr - r6 = r7 * r7 + r6; // s52 mad - r5 = r5 ^ r3; // s53 xor - r1 = r1 - r0; // s54 sub - r5 = r5 - r6; // s55 sub - r3 = r3 + r2 + ((((sel >> 10u) & 1u) != 0u) ? 0xd4758987u : 0x3dfad1b6u); // s56 add - r4 = r4 + r1 + ((((sel >> 0u) & 1u) != 0u) ? 0x0602d6beu : 0xfca75bc2u); // s57 add - r5 = mul_hi(r5, r6); // s58 mulhi - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 2u); r2 = r2 ^ t_; } // s59 shfl - r2 = rotl_imm(r2, 9u); // s60 rotl - r4 = r4 | r6; // s61 or - r6 = rotr_var(r6, r4); // s62 rotr - r2 = r2 * r4; // s63 mul - } - r2 = r2 - r4; // 15 sub - r2 = r2 ^ ds[r0 & mask]; // 16 load - // per-load shadow sub-block 4 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 4 - for (uint sh4 = 0u; sh4 < 27u; ++sh4) { - r0 = r0 ^ r5; // s64 xor - r2 = r2 ^ r7; // s65 xor - r2 = r2 + r1 + ((((sel >> 6u) & 1u) != 0u) ? 0x8596687au : 0xd71c02ffu); // s66 add - r1 = rotl_imm(r1, 21u); // s67 rotl - r3 = r3 * r2; // s68 mul - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 2u); r7 = r7 ^ t_; } // s69 shfl - r3 = r3 * r2; // s70 mul - r0 = r2 * r5 + r0; // s71 mad - r6 = r6 + r7 + ((((sel >> 0u) & 1u) != 0u) ? 0x08ac5733u : 0x3c6fe15du); // s72 add - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 8u); r3 = r3 ^ t_; } // s73 shfl - r3 = r3 * r6; // s74 mul - r6 = r6 ^ r7; // s75 xor - r3 = r3 | r0; // s76 or - r2 = r2 + r4 + ((((sel >> 1u) & 1u) != 0u) ? 0xc100b495u : 0x295d5faeu); // s77 add - r3 = mul_hi(r3, r7); // s78 mulhi - r4 = r4 | r1; // s79 or - } - r7 = r7 ^ ds[r2 & mask]; // 17 load - // per-load shadow sub-block 5 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 5 - for (uint sh5 = 0u; sh5 < 27u; ++sh5) { - r4 = rotr_var(r4, r3); // s80 rotr - r4 = r4 + r3 + ((((sel >> 15u) & 1u) != 0u) ? 0x5cc59530u : 0x274a9221u); // s81 add - r7 = r7 + r3 + ((((sel >> 5u) & 1u) != 0u) ? 0x141479ecu : 0xc8651f8eu); // s82 add - r3 = rotr_var(r3, r6); // s83 rotr - r2 = r4 * r7 + r2; // s84 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 16u); r2 = r2 ^ t_; } // s85 shfl - r3 = r3 ^ r2; // s86 xor - { uint t_; IGNEUM_SHFL_XOR(t_, r7, 2u); r5 = r5 ^ t_; } // s87 shfl - r0 = r0 ^ r4; // s88 xor - r3 = r3 + r2 + ((((sel >> 18u) & 1u) != 0u) ? 0x883c0c92u : 0x53f915c2u); // s89 add - r7 = rotr_var(r7, r5); // s90 rotr - r3 = r3 + r1 + ((((sel >> 31u) & 1u) != 0u) ? 0x3cc5bd20u : 0xe2be00a5u); // s91 add - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 1u); r7 = r7 ^ t_; } // s92 shfl - r6 = rotr_var(r6, r2); // s93 rotr - r7 = rotl_imm(r7, 14u); // s94 rotl - r4 = r5 * r5 + r4; // s95 mad - } - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 4u); r7 = r7 ^ t_; } // 18 shfl - r5 = r5 * r0; // 19 mul - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 2u); r3 = r3 ^ t_; } // 20 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 16u); r2 = r2 ^ t_; } // 21 shfl - r6 = mul_hi(r6, r2); // 22 mulhi - r6 = r6 ^ ds[r1 & mask]; // 23 load - // per-load shadow sub-block 6 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 6 - for (uint sh6 = 0u; sh6 < 27u; ++sh6) { - r2 = r4 * r5 + r2; // s96 mad - r1 = r1 ^ r0; // s97 xor - r5 = r5 * r4; // s98 mul - r2 = r2 - r0; // s99 sub - r7 = rotl_imm(r7, 30u); // s100 rotl - r5 = r5 + r6 + ((((sel >> 17u) & 1u) != 0u) ? 0xbfd646cbu : 0x423fd9c9u); // s101 add - r7 = r7 + r6 + ((((sel >> 11u) & 1u) != 0u) ? 0x19b74a43u : 0x8d3c011du); // s102 add - r5 = rotl_imm(r5, 6u); // s103 rotl - r0 = r0 * r4; // s104 mul - r0 = r0 | r5; // s105 or - r0 = r0 + r1 + ((((sel >> 15u) & 1u) != 0u) ? 0x7dcb7e18u : 0x8ce14721u); // s106 add - r0 = rotl_imm(r0, 15u); // s107 rotl - r4 = r4 - r2; // s108 sub - r2 = mul_hi(r2, r0); // s109 mulhi - r1 = mul_hi(r1, r0); // s110 mulhi - r0 = r0 + r2 + ((((sel >> 28u) & 1u) != 0u) ? 0x3c179ad8u : 0x2d9fc6b9u); // s111 add - } - r5 = r5 * r0; // 24 mul - r5 = rotl_imm(r5, 19u); // 25 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 2u); r7 = r7 ^ t_; } // 26 shfl - r0 = r0 ^ r5; // 27 xor - r0 = r0 ^ r4; // 28 xor - r3 = r3 - r0; // 29 sub - r5 = r5 * r1; // 30 mul - r7 = r7 ^ ds[r2 & mask]; // 31 load - // per-load shadow sub-block 7 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 7 - for (uint sh7 = 0u; sh7 < 27u; ++sh7) { - r2 = mul_hi(r2, r0); // s112 mulhi - r6 = r6 - r3; // s113 sub - r7 = r6 * r3 + r7; // s114 mad - r5 = rotr_var(r5, r1); // s115 rotr - r6 = rotr_var(r6, r4); // s116 rotr - r2 = r2 + r1 + ((((sel >> 22u) & 1u) != 0u) ? 0x47359729u : 0x502138e2u); // s117 add - r3 = r2 * r0 + r3; // s118 mad - r1 = r1 * r3; // s119 mul - r2 = r0 * r4 + r2; // s120 mad - r0 = r0 * r3; // s121 mul - r2 = mul_hi(r2, r0); // s122 mulhi - r5 = r5 * r6; // s123 mul - r4 = r2 * r4 + r4; // s124 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 2u); r4 = r4 ^ t_; } // s125 shfl - r2 = r2 ^ r0; // s126 xor - r1 = rotl_imm(r1, 7u); // s127 rotl - } - r1 = r1 ^ ds[r0 & mask]; // 32 load - // per-load shadow sub-block 8 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 8 - for (uint sh8 = 0u; sh8 < 27u; ++sh8) { - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 4u); r7 = r7 ^ t_; } // s128 shfl - r6 = rotl_imm(r6, 17u); // s129 rotl - r2 = r2 + r5 + ((((sel >> 15u) & 1u) != 0u) ? 0x6c57e4e7u : 0x33dff776u); // s130 add - r0 = r0 | r3; // s131 or - r5 = rotr_var(r5, r0); // s132 rotr - r2 = r2 + r3 + ((((sel >> 30u) & 1u) != 0u) ? 0xe8212c0cu : 0x5db25b34u); // s133 add - r4 = r4 ^ r3; // s134 xor - r6 = r6 ^ r1; // s135 xor - r1 = r1 - r7; // s136 sub - r2 = r6 * r2 + r2; // s137 mad - r0 = mul_hi(r0, r5); // s138 mulhi - r2 = r2 + r7 + ((((sel >> 10u) & 1u) != 0u) ? 0xd44ff710u : 0x218d4090u); // s139 add - r1 = rotr_var(r1, r4); // s140 rotr - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 1u); r3 = r3 ^ t_; } // s141 shfl - r7 = r6 * r2 + r7; // s142 mad - r2 = r2 * r3; // s143 mul - } - r5 = r5 ^ r6; // 33 xor - r5 = r5 ^ ds[r1 & mask]; // 34 load - // per-load shadow sub-block 9 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 9 - for (uint sh9 = 0u; sh9 < 27u; ++sh9) { - r7 = r7 + r4 + ((((sel >> 29u) & 1u) != 0u) ? 0xb98942fau : 0xf4b1a8deu); // s144 add - r7 = rotl_imm(r7, 15u); // s145 rotl - r7 = r7 ^ r5; // s146 xor - r4 = r7 * r4 + r4; // s147 mad - r6 = rotr_var(r6, r5); // s148 rotr - r1 = r2 * r4 + r1; // s149 mad - r1 = r1 + r7 + ((((sel >> 17u) & 1u) != 0u) ? 0x0d48ba42u : 0x2bef10f2u); // s150 add - r5 = r5 ^ r4; // s151 xor - r7 = r7 + r0 + ((((sel >> 30u) & 1u) != 0u) ? 0xc98dea9cu : 0xdc5cc080u); // s152 add - r4 = r4 * r6; // s153 mul - r0 = r0 * r2; // s154 mul - r6 = r6 ^ r5; // s155 xor - r4 = rotr_var(r4, r2); // s156 rotr - r1 = rotl_imm(r1, 11u); // s157 rotl - r5 = r5 + r0 + ((((sel >> 11u) & 1u) != 0u) ? 0x6c752dcbu : 0x18197438u); // s158 add - r4 = r1 * r5 + r4; // s159 mad - } - r0 = mul_hi(r0, r5); // 35 mulhi - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 4u); r5 = r5 ^ t_; } // 36 shfl - r7 = r7 ^ ds[r0 & mask]; // 37 load - // per-load shadow sub-block 10 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 10 - for (uint sh10 = 0u; sh10 < 27u; ++sh10) { - r4 = rotl_imm(r4, 26u); // s160 rotl - r3 = r0 * r7 + r3; // s161 mad - r3 = rotr_var(r3, r1); // s162 rotr - r4 = r4 + r0 + ((((sel >> 3u) & 1u) != 0u) ? 0x8e481727u : 0xf1c46574u); // s163 add - r0 = mul_hi(r0, r4); // s164 mulhi - r2 = r2 + r6 + ((((sel >> 20u) & 1u) != 0u) ? 0x550ab406u : 0xac578137u); // s165 add - r0 = rotl_imm(r0, 13u); // s166 rotl - r3 = r3 + r1 + ((((sel >> 14u) & 1u) != 0u) ? 0xaebb5966u : 0xa33e6706u); // s167 add - r3 = r3 | r5; // s168 or - r6 = rotr_var(r6, r2); // s169 rotr - r4 = r4 ^ r6; // s170 xor - r6 = r6 - r1; // s171 sub - r7 = rotl_imm(r7, 22u); // s172 rotl - r5 = rotl_imm(r5, 15u); // s173 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r0, 8u); r7 = r7 ^ t_; } // s174 shfl - r0 = r0 ^ r5; // s175 xor - } - r3 = r3 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0x230c005cu : 0x75ba2fadu); // 38 add - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 4u); r1 = r1 ^ t_; } // 39 shfl - r2 = r2 ^ r5; // 40 xor - r3 = r6 * r3 + r3; // 41 mad - r6 = r6 - r7; // 42 sub - r7 = r7 ^ r0; // 43 xor - r1 = r1 ^ ds[r7 & mask]; // 44 load - // per-load shadow sub-block 11 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 11 - for (uint sh11 = 0u; sh11 < 27u; ++sh11) { - r7 = rotl_imm(r7, 6u); // s176 rotl - r7 = r7 - r0; // s177 sub - r3 = rotl_imm(r3, 30u); // s178 rotl - r7 = r6 * r1 + r7; // s179 mad - r6 = rotl_imm(r6, 9u); // s180 rotl - r2 = r2 ^ r4; // s181 xor - r2 = r2 ^ r7; // s182 xor - r7 = r7 ^ r2; // s183 xor - r1 = r1 + r2 + ((((sel >> 21u) & 1u) != 0u) ? 0x5bb7550fu : 0xd94d55acu); // s184 add - r3 = r3 | r5; // s185 or - r6 = mul_hi(r6, r3); // s186 mulhi - r4 = r4 | r0; // s187 or - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 2u); r7 = r7 ^ t_; } // s188 shfl - r6 = r6 + r5 + ((((sel >> 6u) & 1u) != 0u) ? 0x2a354e2du : 0x89e747feu); // s189 add - r1 = r1 ^ r4; // s190 xor - r7 = mul_hi(r7, r4); // s191 mulhi - } - r2 = r2 * r3; // 45 mul - r1 = mul_hi(r1, r5); // 46 mulhi - r4 = r4 - r3; // 47 sub - r2 = rotr_var(r2, r6); // 48 rotr - r3 = r3 ^ ds[r5 & mask]; // 49 load - // per-load shadow sub-block 12 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 12 - for (uint sh12 = 0u; sh12 < 27u; ++sh12) { - r2 = rotl_imm(r2, 26u); // s192 rotl - r5 = rotl_imm(r5, 8u); // s193 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 16u); r4 = r4 ^ t_; } // s194 shfl - r4 = r4 ^ r5; // s195 xor - r1 = r1 * r3; // s196 mul - r5 = r5 + r2 + ((((sel >> 7u) & 1u) != 0u) ? 0x10cfdc71u : 0xea03e8e7u); // s197 add - r7 = r7 + r5 + ((((sel >> 15u) & 1u) != 0u) ? 0x66e148d3u : 0xa7ee0102u); // s198 add - r5 = r5 - r2; // s199 sub - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 2u); r5 = r5 ^ t_; } // s200 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 8u); r7 = r7 ^ t_; } // s201 shfl - r4 = rotr_var(r4, r5); // s202 rotr - r7 = r7 ^ r5; // s203 xor - r7 = r7 ^ r0; // s204 xor - r6 = r6 + r2 + ((((sel >> 10u) & 1u) != 0u) ? 0x8558b619u : 0x8379a4deu); // s205 add - r4 = rotl_imm(r4, 13u); // s206 rotl - r1 = mul_hi(r1, r3); // s207 mulhi - } - r1 = r1 + r5 + ((((sel >> 7u) & 1u) != 0u) ? 0x1907970cu : 0x81b8bc2cu); // 50 add - r0 = r0 * r2; // 51 mul - r0 = r0 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x699fd448u : 0x4f92b968u); // 52 add - r1 = r1 + r0 + ((((sel >> 12u) & 1u) != 0u) ? 0x77b1520du : 0x2bb965afu); // 53 add - r7 = rotl_imm(r7, 14u); // 54 rotl - r3 = r3 + r7 + ((((sel >> 1u) & 1u) != 0u) ? 0xa54c55a0u : 0x7b0fe07au); // 55 add - r6 = r6 ^ ds[r7 & mask]; // 56 load - // per-load shadow sub-block 13 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 13 - for (uint sh13 = 0u; sh13 < 27u; ++sh13) { - r1 = r4 * r4 + r1; // s208 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r0, 4u); r2 = r2 ^ t_; } // s209 shfl - r7 = r7 + r0 + ((((sel >> 11u) & 1u) != 0u) ? 0xf4049c4cu : 0xaa8cb14eu); // s210 add - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 16u); r7 = r7 ^ t_; } // s211 shfl - r1 = r1 ^ r0; // s212 xor - r7 = rotl_imm(r7, 3u); // s213 rotl - r4 = rotr_var(r4, r7); // s214 rotr - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 16u); r3 = r3 ^ t_; } // s215 shfl - r5 = r5 | r1; // s216 or - r1 = r1 - r2; // s217 sub - r6 = r6 - r5; // s218 sub - r6 = rotl_imm(r6, 4u); // s219 rotl - r2 = mul_hi(r2, r0); // s220 mulhi - r2 = r2 | r0; // s221 or - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 8u); r5 = r5 ^ t_; } // s222 shfl - r5 = mul_hi(r5, r6); // s223 mulhi - } - r1 = rotr_var(r1, r5); // 57 rotr - r5 = r5 ^ ds[r4 & mask]; // 58 load - // per-load shadow sub-block 14 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 14 - for (uint sh14 = 0u; sh14 < 27u; ++sh14) { - r0 = r0 - r6; // s224 sub - r7 = rotl_imm(r7, 23u); // s225 rotl - r4 = r4 | r2; // s226 or - r2 = r2 * r4; // s227 mul - r3 = rotl_imm(r3, 12u); // s228 rotl - r0 = rotr_var(r0, r4); // s229 rotr - r0 = r0 + r6 + ((((sel >> 16u) & 1u) != 0u) ? 0x2a7fecb2u : 0x1d176220u); // s230 add - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 4u); r1 = r1 ^ t_; } // s231 shfl - r2 = r2 * r1; // s232 mul - r7 = r0 * r1 + r7; // s233 mad - r5 = rotl_imm(r5, 22u); // s234 rotl - r4 = rotr_var(r4, r6); // s235 rotr - r0 = r5 * r1 + r0; // s236 mad - r6 = r6 ^ r4; // s237 xor - r4 = r4 ^ r6; // s238 xor - r6 = rotl_imm(r6, 18u); // s239 rotl - } - r6 = r6 ^ ds[r2 & mask]; // 59 load - // per-load shadow sub-block 15 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 15 - for (uint sh15 = 0u; sh15 < 27u; ++sh15) { - r4 = r4 + r7 + ((((sel >> 25u) & 1u) != 0u) ? 0xadce39f3u : 0x17dafb4du); // s240 add - r0 = r0 - r7; // s241 sub - r1 = rotr_var(r1, r6); // s242 rotr - r3 = r3 + r0 + ((((sel >> 29u) & 1u) != 0u) ? 0x0e7033b6u : 0xf2f77d26u); // s243 add - r2 = r7 * r4 + r2; // s244 mad - r4 = r0 * r6 + r4; // s245 mad - r5 = r5 * r7; // s246 mul - r3 = r3 + r5 + ((((sel >> 31u) & 1u) != 0u) ? 0x7b2ff6b7u : 0xfed2da4eu); // s247 add - r0 = r0 + r7 + ((((sel >> 0u) & 1u) != 0u) ? 0xb5fad7b1u : 0x03b2891cu); // s248 add - r5 = mul_hi(r5, r4); // s249 mulhi - r5 = r5 + r2 + ((((sel >> 11u) & 1u) != 0u) ? 0xa7e71c2fu : 0x042cd6e3u); // s250 add - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 1u); r6 = r6 ^ t_; } // s251 shfl - r6 = r6 ^ r1; // s252 xor - r2 = r2 * r5; // s253 mul - r0 = r0 + r6 + ((((sel >> 16u) & 1u) != 0u) ? 0xb83d78deu : 0x784a302bu); // s254 add - r2 = r2 - r4; // s255 sub - } - r3 = r5 * r0 + r3; // 60 mad - r5 = r5 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0xad7493e7u : 0xaf9dd72du); // 61 add - r4 = r4 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x1e07c3d9u : 0x89841d87u); // 62 add - r5 = rotl_imm(r5, 19u); // 63 rotl - } - uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((ulong)hi << 32) | (ulong)lo; -} diff --git a/proto-cuda/packs-ca4/mx8_shl256x27/kernel_bound.cu b/proto-cuda/packs-ca4/mx8_shl256x27/kernel_bound.cu deleted file mode 100644 index e8ac5125f..000000000 --- a/proto-cuda/packs-ca4/mx8_shl256x27/kernel_bound.cu +++ /dev/null @@ -1,427 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// Header-bound twin of igneum_hash in kernel.cu: the init words come from a kernel argument, not SEEDW. -// Host declarations (also in program_bound.h if present): -// struct IgneumInitWords { uint32_t w[8]; }; -// cudaError_t igneum_launch_hash_bound(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask, -// IgneumInitWords iw, uint32_t nonces, uint32_t blockWarps); -// cudaError_t igneum_hash_bound_info(int* numRegs, int* blocksPerSM, uint32_t blockWarps); -#include -#include -#include "program.h" - -struct IgneumInitWords { uint32_t w[8]; }; - -__device__ __forceinline__ uint32_t splitmix32(uint32_t x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -__device__ __forceinline__ uint32_t rotl_imm(uint32_t x, uint32_t n) { return (x << n) | (x >> (32u - n)); } -__device__ __forceinline__ uint32_t rotr_var(uint32_t x, uint32_t n) { n &= 31u; return (x >> n) | (x << ((32u - n) & 31u)); } - -__global__ void igneum_hash_bound(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask, IgneumInitWords iw) { - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t nonce = baseNonce + gid; - uint32_t r0, r1, r2, r3, r4, r5, r6, r7; - { uint32_t x = nonce ^ iw.w[0]; x += 0x9e3779b9u * 1u; x = splitmix32(x); r0 = x ^ iw.w[1]; } - { uint32_t x = nonce ^ iw.w[1]; x += 0x9e3779b9u * 2u; x = splitmix32(x); r1 = x ^ iw.w[2]; } - { uint32_t x = nonce ^ iw.w[2]; x += 0x9e3779b9u * 3u; x = splitmix32(x); r2 = x ^ iw.w[3]; } - { uint32_t x = nonce ^ iw.w[3]; x += 0x9e3779b9u * 4u; x = splitmix32(x); r3 = x ^ iw.w[4]; } - { uint32_t x = nonce ^ iw.w[4]; x += 0x9e3779b9u * 5u; x = splitmix32(x); r4 = x ^ iw.w[5]; } - { uint32_t x = nonce ^ iw.w[5]; x += 0x9e3779b9u * 6u; x = splitmix32(x); r5 = x ^ iw.w[6]; } - { uint32_t x = nonce ^ iw.w[6]; x += 0x9e3779b9u * 7u; x = splitmix32(x); r6 = x ^ iw.w[7]; } - { uint32_t x = nonce ^ iw.w[7]; x += 0x9e3779b9u * 8u; x = splitmix32(x); r7 = x ^ iw.w[0]; } - - for (uint32_t it = 0u; it < 8u; ++it) { - uint32_t sel = r0; - r2 = r3 * r4 + r2; // 0 mad - r2 = r1 * r1 + r2; // 1 mad - r2 = r3 * r2 + r2; // 2 mad - r3 = r3 ^ r5; // 3 xor - r7 = r7 ^ ds[r2 & mask]; // 4 load - // per-load shadow sub-block 0 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 0 - for (uint32_t sh0 = 0u; sh0 < 27u; ++sh0) { - r5 = r5 + r2 + ((((sel >> 18u) & 1u) != 0u) ? 0x8bc12da9u : 0x92199f99u); // s0 add - r0 = r0 + r7 + ((((sel >> 26u) & 1u) != 0u) ? 0x63079e5au : 0x8d72d3adu); // s1 add - r6 = r6 ^ __shfl_xor_sync(0xffffffffu, r3, 2); // s2 shfl - r4 = r4 - r2; // s3 sub - r7 = r7 + r0 + ((((sel >> 31u) & 1u) != 0u) ? 0x5d4c7a60u : 0xb21b4babu); // s4 add - r0 = rotl_imm(r0, 11u); // s5 rotl - r0 = r0 + r1 + ((((sel >> 16u) & 1u) != 0u) ? 0xc88e2942u : 0x2fe0e98bu); // s6 add - r7 = r7 ^ r1; // s7 xor - r1 = r6 * r5 + r1; // s8 mad - r6 = r6 ^ __shfl_xor_sync(0xffffffffu, r1, 4); // s9 shfl - r1 = r2 * r2 + r1; // s10 mad - r5 = r0 * r3 + r5; // s11 mad - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r6, 4); // s12 shfl - r1 = r1 + r5 + ((((sel >> 25u) & 1u) != 0u) ? 0xbc48c63eu : 0xbdf8f9a5u); // s13 add - r1 = rotl_imm(r1, 29u); // s14 rotl - r1 = r1 - r4; // s15 sub - } - r5 = r5 ^ ds[r7 & mask]; // 5 load - // per-load shadow sub-block 1 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 1 - for (uint32_t sh1 = 0u; sh1 < 27u; ++sh1) { - r7 = r7 | r1; // s16 or - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r4, 8); // s17 shfl - r7 = r7 ^ r4; // s18 xor - r6 = r6 * r1; // s19 mul - r5 = r6 * r0 + r5; // s20 mad - r3 = r3 - r1; // s21 sub - r6 = r6 * r0; // s22 mul - r2 = r2 + r0 + ((((sel >> 1u) & 1u) != 0u) ? 0x96e8f127u : 0x45c37cecu); // s23 add - r6 = r6 - r4; // s24 sub - r7 = r3 * r4 + r7; // s25 mad - r3 = rotl_imm(r3, 9u); // s26 rotl - r2 = r2 - r1; // s27 sub - r6 = __umulhi(r6, r0); // s28 mulhi - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r4, 2); // s29 shfl - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r6, 1); // s30 shfl - r1 = r1 + r6 + ((((sel >> 1u) & 1u) != 0u) ? 0xb1cdb2abu : 0x37985632u); // s31 add - } - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r4, 8); // 6 shfl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r3, 8); // 7 shfl - r1 = __umulhi(r1, r5); // 8 mulhi - r6 = rotr_var(r6, r3); // 9 rotr - r3 = r3 | r4; // 10 or - r4 = r4 ^ ds[r3 & mask]; // 11 load - // per-load shadow sub-block 2 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 2 - for (uint32_t sh2 = 0u; sh2 < 27u; ++sh2) { - r0 = rotr_var(r0, r1); // s32 rotr - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r4, 2); // s33 shfl - r0 = r0 ^ __shfl_xor_sync(0xffffffffu, r3, 4); // s34 shfl - r7 = r7 * r0; // s35 mul - r3 = r3 + r1 + ((((sel >> 0u) & 1u) != 0u) ? 0x856e0180u : 0x804e777eu); // s36 add - r1 = r1 ^ r6; // s37 xor - r2 = r2 * r7; // s38 mul - r6 = r6 + r5 + ((((sel >> 2u) & 1u) != 0u) ? 0xe9bd0cc4u : 0x0b06c8a7u); // s39 add - r5 = r5 * r7; // s40 mul - r2 = __umulhi(r2, r3); // s41 mulhi - r2 = r2 ^ r0; // s42 xor - r0 = rotl_imm(r0, 4u); // s43 rotl - r4 = __umulhi(r4, r3); // s44 mulhi - r6 = r6 + r7 + ((((sel >> 12u) & 1u) != 0u) ? 0xcdcb63f6u : 0xee822e17u); // s45 add - r3 = r2 * r0 + r3; // s46 mad - r4 = r4 ^ __shfl_xor_sync(0xffffffffu, r3, 8); // s47 shfl - } - r0 = __umulhi(r0, r4); // 12 mulhi - r5 = r5 + r1 + ((((sel >> 30u) & 1u) != 0u) ? 0xd3177981u : 0xc7934706u); // 13 add - r0 = r0 ^ ds[r4 & mask]; // 14 load - // per-load shadow sub-block 3 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 3 - for (uint32_t sh3 = 0u; sh3 < 27u; ++sh3) { - r6 = __umulhi(r6, r5); // s48 mulhi - r0 = r0 - r2; // s49 sub - r3 = r3 - r5; // s50 sub - r1 = rotr_var(r1, r4); // s51 rotr - r6 = r7 * r7 + r6; // s52 mad - r5 = r5 ^ r3; // s53 xor - r1 = r1 - r0; // s54 sub - r5 = r5 - r6; // s55 sub - r3 = r3 + r2 + ((((sel >> 10u) & 1u) != 0u) ? 0xd4758987u : 0x3dfad1b6u); // s56 add - r4 = r4 + r1 + ((((sel >> 0u) & 1u) != 0u) ? 0x0602d6beu : 0xfca75bc2u); // s57 add - r5 = __umulhi(r5, r6); // s58 mulhi - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r1, 2); // s59 shfl - r2 = rotl_imm(r2, 9u); // s60 rotl - r4 = r4 | r6; // s61 or - r6 = rotr_var(r6, r4); // s62 rotr - r2 = r2 * r4; // s63 mul - } - r2 = r2 - r4; // 15 sub - r2 = r2 ^ ds[r0 & mask]; // 16 load - // per-load shadow sub-block 4 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 4 - for (uint32_t sh4 = 0u; sh4 < 27u; ++sh4) { - r0 = r0 ^ r5; // s64 xor - r2 = r2 ^ r7; // s65 xor - r2 = r2 + r1 + ((((sel >> 6u) & 1u) != 0u) ? 0x8596687au : 0xd71c02ffu); // s66 add - r1 = rotl_imm(r1, 21u); // s67 rotl - r3 = r3 * r2; // s68 mul - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r5, 2); // s69 shfl - r3 = r3 * r2; // s70 mul - r0 = r2 * r5 + r0; // s71 mad - r6 = r6 + r7 + ((((sel >> 0u) & 1u) != 0u) ? 0x08ac5733u : 0x3c6fe15du); // s72 add - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r2, 8); // s73 shfl - r3 = r3 * r6; // s74 mul - r6 = r6 ^ r7; // s75 xor - r3 = r3 | r0; // s76 or - r2 = r2 + r4 + ((((sel >> 1u) & 1u) != 0u) ? 0xc100b495u : 0x295d5faeu); // s77 add - r3 = __umulhi(r3, r7); // s78 mulhi - r4 = r4 | r1; // s79 or - } - r7 = r7 ^ ds[r2 & mask]; // 17 load - // per-load shadow sub-block 5 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 5 - for (uint32_t sh5 = 0u; sh5 < 27u; ++sh5) { - r4 = rotr_var(r4, r3); // s80 rotr - r4 = r4 + r3 + ((((sel >> 15u) & 1u) != 0u) ? 0x5cc59530u : 0x274a9221u); // s81 add - r7 = r7 + r3 + ((((sel >> 5u) & 1u) != 0u) ? 0x141479ecu : 0xc8651f8eu); // s82 add - r3 = rotr_var(r3, r6); // s83 rotr - r2 = r4 * r7 + r2; // s84 mad - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r5, 16); // s85 shfl - r3 = r3 ^ r2; // s86 xor - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r7, 2); // s87 shfl - r0 = r0 ^ r4; // s88 xor - r3 = r3 + r2 + ((((sel >> 18u) & 1u) != 0u) ? 0x883c0c92u : 0x53f915c2u); // s89 add - r7 = rotr_var(r7, r5); // s90 rotr - r3 = r3 + r1 + ((((sel >> 31u) & 1u) != 0u) ? 0x3cc5bd20u : 0xe2be00a5u); // s91 add - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r2, 1); // s92 shfl - r6 = rotr_var(r6, r2); // s93 rotr - r7 = rotl_imm(r7, 14u); // s94 rotl - r4 = r5 * r5 + r4; // s95 mad - } - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r3, 4); // 18 shfl - r5 = r5 * r0; // 19 mul - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r4, 2); // 20 shfl - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r4, 16); // 21 shfl - r6 = __umulhi(r6, r2); // 22 mulhi - r6 = r6 ^ ds[r1 & mask]; // 23 load - // per-load shadow sub-block 6 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 6 - for (uint32_t sh6 = 0u; sh6 < 27u; ++sh6) { - r2 = r4 * r5 + r2; // s96 mad - r1 = r1 ^ r0; // s97 xor - r5 = r5 * r4; // s98 mul - r2 = r2 - r0; // s99 sub - r7 = rotl_imm(r7, 30u); // s100 rotl - r5 = r5 + r6 + ((((sel >> 17u) & 1u) != 0u) ? 0xbfd646cbu : 0x423fd9c9u); // s101 add - r7 = r7 + r6 + ((((sel >> 11u) & 1u) != 0u) ? 0x19b74a43u : 0x8d3c011du); // s102 add - r5 = rotl_imm(r5, 6u); // s103 rotl - r0 = r0 * r4; // s104 mul - r0 = r0 | r5; // s105 or - r0 = r0 + r1 + ((((sel >> 15u) & 1u) != 0u) ? 0x7dcb7e18u : 0x8ce14721u); // s106 add - r0 = rotl_imm(r0, 15u); // s107 rotl - r4 = r4 - r2; // s108 sub - r2 = __umulhi(r2, r0); // s109 mulhi - r1 = __umulhi(r1, r0); // s110 mulhi - r0 = r0 + r2 + ((((sel >> 28u) & 1u) != 0u) ? 0x3c179ad8u : 0x2d9fc6b9u); // s111 add - } - r5 = r5 * r0; // 24 mul - r5 = rotl_imm(r5, 19u); // 25 rotl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r6, 2); // 26 shfl - r0 = r0 ^ r5; // 27 xor - r0 = r0 ^ r4; // 28 xor - r3 = r3 - r0; // 29 sub - r5 = r5 * r1; // 30 mul - r7 = r7 ^ ds[r2 & mask]; // 31 load - // per-load shadow sub-block 7 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 7 - for (uint32_t sh7 = 0u; sh7 < 27u; ++sh7) { - r2 = __umulhi(r2, r0); // s112 mulhi - r6 = r6 - r3; // s113 sub - r7 = r6 * r3 + r7; // s114 mad - r5 = rotr_var(r5, r1); // s115 rotr - r6 = rotr_var(r6, r4); // s116 rotr - r2 = r2 + r1 + ((((sel >> 22u) & 1u) != 0u) ? 0x47359729u : 0x502138e2u); // s117 add - r3 = r2 * r0 + r3; // s118 mad - r1 = r1 * r3; // s119 mul - r2 = r0 * r4 + r2; // s120 mad - r0 = r0 * r3; // s121 mul - r2 = __umulhi(r2, r0); // s122 mulhi - r5 = r5 * r6; // s123 mul - r4 = r2 * r4 + r4; // s124 mad - r4 = r4 ^ __shfl_xor_sync(0xffffffffu, r2, 2); // s125 shfl - r2 = r2 ^ r0; // s126 xor - r1 = rotl_imm(r1, 7u); // s127 rotl - } - r1 = r1 ^ ds[r0 & mask]; // 32 load - // per-load shadow sub-block 8 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 8 - for (uint32_t sh8 = 0u; sh8 < 27u; ++sh8) { - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r2, 4); // s128 shfl - r6 = rotl_imm(r6, 17u); // s129 rotl - r2 = r2 + r5 + ((((sel >> 15u) & 1u) != 0u) ? 0x6c57e4e7u : 0x33dff776u); // s130 add - r0 = r0 | r3; // s131 or - r5 = rotr_var(r5, r0); // s132 rotr - r2 = r2 + r3 + ((((sel >> 30u) & 1u) != 0u) ? 0xe8212c0cu : 0x5db25b34u); // s133 add - r4 = r4 ^ r3; // s134 xor - r6 = r6 ^ r1; // s135 xor - r1 = r1 - r7; // s136 sub - r2 = r6 * r2 + r2; // s137 mad - r0 = __umulhi(r0, r5); // s138 mulhi - r2 = r2 + r7 + ((((sel >> 10u) & 1u) != 0u) ? 0xd44ff710u : 0x218d4090u); // s139 add - r1 = rotr_var(r1, r4); // s140 rotr - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r6, 1); // s141 shfl - r7 = r6 * r2 + r7; // s142 mad - r2 = r2 * r3; // s143 mul - } - r5 = r5 ^ r6; // 33 xor - r5 = r5 ^ ds[r1 & mask]; // 34 load - // per-load shadow sub-block 9 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 9 - for (uint32_t sh9 = 0u; sh9 < 27u; ++sh9) { - r7 = r7 + r4 + ((((sel >> 29u) & 1u) != 0u) ? 0xb98942fau : 0xf4b1a8deu); // s144 add - r7 = rotl_imm(r7, 15u); // s145 rotl - r7 = r7 ^ r5; // s146 xor - r4 = r7 * r4 + r4; // s147 mad - r6 = rotr_var(r6, r5); // s148 rotr - r1 = r2 * r4 + r1; // s149 mad - r1 = r1 + r7 + ((((sel >> 17u) & 1u) != 0u) ? 0x0d48ba42u : 0x2bef10f2u); // s150 add - r5 = r5 ^ r4; // s151 xor - r7 = r7 + r0 + ((((sel >> 30u) & 1u) != 0u) ? 0xc98dea9cu : 0xdc5cc080u); // s152 add - r4 = r4 * r6; // s153 mul - r0 = r0 * r2; // s154 mul - r6 = r6 ^ r5; // s155 xor - r4 = rotr_var(r4, r2); // s156 rotr - r1 = rotl_imm(r1, 11u); // s157 rotl - r5 = r5 + r0 + ((((sel >> 11u) & 1u) != 0u) ? 0x6c752dcbu : 0x18197438u); // s158 add - r4 = r1 * r5 + r4; // s159 mad - } - r0 = __umulhi(r0, r5); // 35 mulhi - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r2, 4); // 36 shfl - r7 = r7 ^ ds[r0 & mask]; // 37 load - // per-load shadow sub-block 10 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 10 - for (uint32_t sh10 = 0u; sh10 < 27u; ++sh10) { - r4 = rotl_imm(r4, 26u); // s160 rotl - r3 = r0 * r7 + r3; // s161 mad - r3 = rotr_var(r3, r1); // s162 rotr - r4 = r4 + r0 + ((((sel >> 3u) & 1u) != 0u) ? 0x8e481727u : 0xf1c46574u); // s163 add - r0 = __umulhi(r0, r4); // s164 mulhi - r2 = r2 + r6 + ((((sel >> 20u) & 1u) != 0u) ? 0x550ab406u : 0xac578137u); // s165 add - r0 = rotl_imm(r0, 13u); // s166 rotl - r3 = r3 + r1 + ((((sel >> 14u) & 1u) != 0u) ? 0xaebb5966u : 0xa33e6706u); // s167 add - r3 = r3 | r5; // s168 or - r6 = rotr_var(r6, r2); // s169 rotr - r4 = r4 ^ r6; // s170 xor - r6 = r6 - r1; // s171 sub - r7 = rotl_imm(r7, 22u); // s172 rotl - r5 = rotl_imm(r5, 15u); // s173 rotl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r0, 8); // s174 shfl - r0 = r0 ^ r5; // s175 xor - } - r3 = r3 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0x230c005cu : 0x75ba2fadu); // 38 add - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r5, 4); // 39 shfl - r2 = r2 ^ r5; // 40 xor - r3 = r6 * r3 + r3; // 41 mad - r6 = r6 - r7; // 42 sub - r7 = r7 ^ r0; // 43 xor - r1 = r1 ^ ds[r7 & mask]; // 44 load - // per-load shadow sub-block 11 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 11 - for (uint32_t sh11 = 0u; sh11 < 27u; ++sh11) { - r7 = rotl_imm(r7, 6u); // s176 rotl - r7 = r7 - r0; // s177 sub - r3 = rotl_imm(r3, 30u); // s178 rotl - r7 = r6 * r1 + r7; // s179 mad - r6 = rotl_imm(r6, 9u); // s180 rotl - r2 = r2 ^ r4; // s181 xor - r2 = r2 ^ r7; // s182 xor - r7 = r7 ^ r2; // s183 xor - r1 = r1 + r2 + ((((sel >> 21u) & 1u) != 0u) ? 0x5bb7550fu : 0xd94d55acu); // s184 add - r3 = r3 | r5; // s185 or - r6 = __umulhi(r6, r3); // s186 mulhi - r4 = r4 | r0; // s187 or - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r1, 2); // s188 shfl - r6 = r6 + r5 + ((((sel >> 6u) & 1u) != 0u) ? 0x2a354e2du : 0x89e747feu); // s189 add - r1 = r1 ^ r4; // s190 xor - r7 = __umulhi(r7, r4); // s191 mulhi - } - r2 = r2 * r3; // 45 mul - r1 = __umulhi(r1, r5); // 46 mulhi - r4 = r4 - r3; // 47 sub - r2 = rotr_var(r2, r6); // 48 rotr - r3 = r3 ^ ds[r5 & mask]; // 49 load - // per-load shadow sub-block 12 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 12 - for (uint32_t sh12 = 0u; sh12 < 27u; ++sh12) { - r2 = rotl_imm(r2, 26u); // s192 rotl - r5 = rotl_imm(r5, 8u); // s193 rotl - r4 = r4 ^ __shfl_xor_sync(0xffffffffu, r5, 16); // s194 shfl - r4 = r4 ^ r5; // s195 xor - r1 = r1 * r3; // s196 mul - r5 = r5 + r2 + ((((sel >> 7u) & 1u) != 0u) ? 0x10cfdc71u : 0xea03e8e7u); // s197 add - r7 = r7 + r5 + ((((sel >> 15u) & 1u) != 0u) ? 0x66e148d3u : 0xa7ee0102u); // s198 add - r5 = r5 - r2; // s199 sub - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r1, 2); // s200 shfl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r1, 8); // s201 shfl - r4 = rotr_var(r4, r5); // s202 rotr - r7 = r7 ^ r5; // s203 xor - r7 = r7 ^ r0; // s204 xor - r6 = r6 + r2 + ((((sel >> 10u) & 1u) != 0u) ? 0x8558b619u : 0x8379a4deu); // s205 add - r4 = rotl_imm(r4, 13u); // s206 rotl - r1 = __umulhi(r1, r3); // s207 mulhi - } - r1 = r1 + r5 + ((((sel >> 7u) & 1u) != 0u) ? 0x1907970cu : 0x81b8bc2cu); // 50 add - r0 = r0 * r2; // 51 mul - r0 = r0 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x699fd448u : 0x4f92b968u); // 52 add - r1 = r1 + r0 + ((((sel >> 12u) & 1u) != 0u) ? 0x77b1520du : 0x2bb965afu); // 53 add - r7 = rotl_imm(r7, 14u); // 54 rotl - r3 = r3 + r7 + ((((sel >> 1u) & 1u) != 0u) ? 0xa54c55a0u : 0x7b0fe07au); // 55 add - r6 = r6 ^ ds[r7 & mask]; // 56 load - // per-load shadow sub-block 13 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 13 - for (uint32_t sh13 = 0u; sh13 < 27u; ++sh13) { - r1 = r4 * r4 + r1; // s208 mad - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r0, 4); // s209 shfl - r7 = r7 + r0 + ((((sel >> 11u) & 1u) != 0u) ? 0xf4049c4cu : 0xaa8cb14eu); // s210 add - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r1, 16); // s211 shfl - r1 = r1 ^ r0; // s212 xor - r7 = rotl_imm(r7, 3u); // s213 rotl - r4 = rotr_var(r4, r7); // s214 rotr - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r2, 16); // s215 shfl - r5 = r5 | r1; // s216 or - r1 = r1 - r2; // s217 sub - r6 = r6 - r5; // s218 sub - r6 = rotl_imm(r6, 4u); // s219 rotl - r2 = __umulhi(r2, r0); // s220 mulhi - r2 = r2 | r0; // s221 or - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r2, 8); // s222 shfl - r5 = __umulhi(r5, r6); // s223 mulhi - } - r1 = rotr_var(r1, r5); // 57 rotr - r5 = r5 ^ ds[r4 & mask]; // 58 load - // per-load shadow sub-block 14 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 14 - for (uint32_t sh14 = 0u; sh14 < 27u; ++sh14) { - r0 = r0 - r6; // s224 sub - r7 = rotl_imm(r7, 23u); // s225 rotl - r4 = r4 | r2; // s226 or - r2 = r2 * r4; // s227 mul - r3 = rotl_imm(r3, 12u); // s228 rotl - r0 = rotr_var(r0, r4); // s229 rotr - r0 = r0 + r6 + ((((sel >> 16u) & 1u) != 0u) ? 0x2a7fecb2u : 0x1d176220u); // s230 add - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r2, 4); // s231 shfl - r2 = r2 * r1; // s232 mul - r7 = r0 * r1 + r7; // s233 mad - r5 = rotl_imm(r5, 22u); // s234 rotl - r4 = rotr_var(r4, r6); // s235 rotr - r0 = r5 * r1 + r0; // s236 mad - r6 = r6 ^ r4; // s237 xor - r4 = r4 ^ r6; // s238 xor - r6 = rotl_imm(r6, 18u); // s239 rotl - } - r6 = r6 ^ ds[r2 & mask]; // 59 load - // per-load shadow sub-block 15 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 15 - for (uint32_t sh15 = 0u; sh15 < 27u; ++sh15) { - r4 = r4 + r7 + ((((sel >> 25u) & 1u) != 0u) ? 0xadce39f3u : 0x17dafb4du); // s240 add - r0 = r0 - r7; // s241 sub - r1 = rotr_var(r1, r6); // s242 rotr - r3 = r3 + r0 + ((((sel >> 29u) & 1u) != 0u) ? 0x0e7033b6u : 0xf2f77d26u); // s243 add - r2 = r7 * r4 + r2; // s244 mad - r4 = r0 * r6 + r4; // s245 mad - r5 = r5 * r7; // s246 mul - r3 = r3 + r5 + ((((sel >> 31u) & 1u) != 0u) ? 0x7b2ff6b7u : 0xfed2da4eu); // s247 add - r0 = r0 + r7 + ((((sel >> 0u) & 1u) != 0u) ? 0xb5fad7b1u : 0x03b2891cu); // s248 add - r5 = __umulhi(r5, r4); // s249 mulhi - r5 = r5 + r2 + ((((sel >> 11u) & 1u) != 0u) ? 0xa7e71c2fu : 0x042cd6e3u); // s250 add - r6 = r6 ^ __shfl_xor_sync(0xffffffffu, r1, 1); // s251 shfl - r6 = r6 ^ r1; // s252 xor - r2 = r2 * r5; // s253 mul - r0 = r0 + r6 + ((((sel >> 16u) & 1u) != 0u) ? 0xb83d78deu : 0x784a302bu); // s254 add - r2 = r2 - r4; // s255 sub - } - r3 = r5 * r0 + r3; // 60 mad - r5 = r5 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0xad7493e7u : 0xaf9dd72du); // 61 add - r4 = r4 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x1e07c3d9u : 0x89841d87u); // 62 add - r5 = rotl_imm(r5, 19u); // 63 rotl - } - uint32_t lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint32_t hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((uint64_t)hi << 32) | (uint64_t)lo; -} - -cudaError_t igneum_launch_hash_bound(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask, - IgneumInitWords iw, uint32_t nonces, uint32_t blockWarps) { - if (blockWarps == 0u || blockWarps > 32u) return cudaErrorInvalidValue; - uint32_t block = 32u * blockWarps; - if (nonces == 0u || (nonces % block) != 0u) return cudaErrorInvalidValue; - igneum_hash_bound<<>>(ds, out, baseNonce, mask, iw); - return cudaGetLastError(); -} - -cudaError_t igneum_hash_bound_info(int* numRegs, int* blocksPerSM, uint32_t blockWarps) { - cudaFuncAttributes attr; - cudaError_t e = cudaFuncGetAttributes(&attr, igneum_hash_bound); - if (e != cudaSuccess) return e; - *numRegs = attr.numRegs; - return cudaOccupancyMaxActiveBlocksPerMultiprocessor(blocksPerSM, igneum_hash_bound, (int)(32u * blockWarps), 0); -} diff --git a/proto-cuda/packs-ca4/mx8_shl256x27/memhard.h b/proto-cuda/packs-ca4/mx8_shl256x27/memhard.h deleted file mode 100644 index f7f34c732..000000000 --- a/proto-cuda/packs-ca4/mx8_shl256x27/memhard.h +++ /dev/null @@ -1,109 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// Memory-hard dataset core, the same text that the Mac's Metal kernels and CPU verifier were checked against. -// Included by kernel.cu (device), host.cu (host reference) and proto-opencl/host.c (C99 host reference). -// See proto-metal/MEMHARD.md for the construction. kernel.cl carries the same text in OpenCL C. -#pragma once -#ifdef __cplusplus -#include -#else -#include -#endif -#if defined(__CUDACC__) -#define IGNEUM_HD __host__ __device__ __forceinline__ -#elif defined(_MSC_VER) && !defined(__cplusplus) -#define IGNEUM_HD static __inline -#else -#define IGNEUM_HD static inline -#endif -// Memory-hard dataset core (MEMHARD.md). Cache: 2^26 words in 2^16 segments of 64 chained ChaCha12 lines. -// Item: 8 rounds of 8 x seed-parameterised mixer + one 64-byte cache read, then 8 x final mixer (class v3, mixer multiplier 8, -// docs/plans/mixer-x4.md: the round key of application j of round r is 0x9E3779B9 * (r * m + j + 1)). All parameters are literals. -#define MH_CACHE_LINE_MASK 0x003fffffu -#define MH_SEGMENT_LINES 64u -#define MH_QR(a, b, c, d, r1, r2, r3, r4) { a += b; d ^= a; d = mh_rotl(d, r1); c += d; b ^= c; b = mh_rotl(b, r2); a += b; d ^= a; d = mh_rotl(d, r3); c += d; b ^= c; b = mh_rotl(b, r4); } -IGNEUM_HD uint32_t mh_rotl(uint32_t x, uint32_t n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 at every call site - -// y = ChaCha12 core(x) + x -IGNEUM_HD void mh_chacha_block(const uint32_t* x, uint32_t* y) { - for (uint32_t i = 0u; i < 16u; ++i) y[i] = x[i]; - for (uint32_t r = 0u; r < 6u; ++r) { - MH_QR(y[0], y[4], y[8], y[12], 16u, 12u, 8u, 7u) MH_QR(y[1], y[5], y[9], y[13], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[6], y[10], y[14], 16u, 12u, 8u, 7u) MH_QR(y[3], y[7], y[11], y[15], 16u, 12u, 8u, 7u) - MH_QR(y[0], y[5], y[10], y[15], 16u, 12u, 8u, 7u) MH_QR(y[1], y[6], y[11], y[12], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[7], y[8], y[13], 16u, 12u, 8u, 7u) MH_QR(y[3], y[4], y[9], y[14], 16u, 12u, 8u, 7u) - } - for (uint32_t i = 0u; i < 16u; ++i) y[i] += x[i]; -} - -// One cache segment: 64 chained lines written at cache[seg * 1024]. in_j = prev ^ (sigma || K || seg || j || tag), prev_0 = 0. -IGNEUM_HD void mh_cache_segment(uint32_t* cache, uint32_t seg) { - uint32_t prev[16]; uint32_t x[16]; uint32_t y[16]; - for (uint32_t i = 0u; i < 16u; ++i) prev[i] = 0u; - for (uint32_t j = 0u; j < MH_SEGMENT_LINES; ++j) { - x[0] = 0x61707865u ^ prev[0]; x[1] = 0x3320646eu ^ prev[1]; x[2] = 0x79622d32u ^ prev[2]; x[3] = 0x6b206574u ^ prev[3]; - x[4] = 0x3067619fu ^ prev[4]; - x[5] = 0x3c269176u ^ prev[5]; - x[6] = 0x84a03b03u ^ prev[6]; - x[7] = 0xf8c63294u ^ prev[7]; - x[8] = 0xff977c5bu ^ prev[8]; - x[9] = 0xe60def3eu ^ prev[9]; - x[10] = 0x63630141u ^ prev[10]; - x[11] = 0xb8fbcb58u ^ prev[11]; - x[12] = seg ^ prev[12]; x[13] = j ^ prev[13]; x[14] = 0x49676e65u ^ prev[14]; x[15] = 0x756d4d48u ^ prev[15]; - mh_chacha_block(x, y); - uint32_t* line = cache + ((seg * MH_SEGMENT_LINES + j) * 16u); - for (uint32_t i = 0u; i < 16u; ++i) { line[i] = y[i]; prev[i] = y[i]; } - } -} - -// M_r: per word (s ^ (RC + rk)) * MUL, then a column round and a diagonal round with the seed-drawn rotations. -IGNEUM_HD void mh_mixer(uint32_t* s, uint32_t rk) { - s[0] = (s[0] ^ (0xbab68293u + rk)) * 0x42146205u; - s[1] = (s[1] ^ (0xcc162340u + rk)) * 0x52cbe0fbu; - s[2] = (s[2] ^ (0x6ce151ccu + rk)) * 0x7ecf4a03u; - s[3] = (s[3] ^ (0xe62b8997u + rk)) * 0x6728907fu; - s[4] = (s[4] ^ (0xc9c80297u + rk)) * 0xd81d9751u; - s[5] = (s[5] ^ (0xf74a1654u + rk)) * 0x132952c3u; - s[6] = (s[6] ^ (0x3d704af5u + rk)) * 0xf60de277u; - s[7] = (s[7] ^ (0x3cf522b7u + rk)) * 0x05358035u; - s[8] = (s[8] ^ (0x2b9cac04u + rk)) * 0xbaf6499du; - s[9] = (s[9] ^ (0xa880ac10u + rk)) * 0xe4db9667u; - s[10] = (s[10] ^ (0x13e5dd1du + rk)) * 0x3e98f45du; - s[11] = (s[11] ^ (0x6fc3e233u + rk)) * 0xd0004eddu; - s[12] = (s[12] ^ (0x2d83eeacu + rk)) * 0x2691630du; - s[13] = (s[13] ^ (0x9006e8bfu + rk)) * 0x9beb3bcfu; - s[14] = (s[14] ^ (0x2c4b5362u + rk)) * 0xab310379u; - s[15] = (s[15] ^ (0x31b49ee2u + rk)) * 0x99cfb423u; - MH_QR(s[0], s[4], s[8], s[12], 20u, 20u, 19u, 4u) MH_QR(s[1], s[5], s[9], s[13], 20u, 20u, 19u, 4u) - MH_QR(s[2], s[6], s[10], s[14], 20u, 20u, 19u, 4u) MH_QR(s[3], s[7], s[11], s[15], 20u, 20u, 19u, 4u) - MH_QR(s[0], s[5], s[10], s[15], 26u, 3u, 3u, 27u) MH_QR(s[1], s[6], s[11], s[12], 26u, 3u, 3u, 27u) - MH_QR(s[2], s[7], s[8], s[13], 26u, 3u, 3u, 27u) MH_QR(s[3], s[4], s[9], s[14], 26u, 3u, 3u, 27u) -} - -// Item t: 16 words. s = (K, t * MUL[i] + RC[i]); 8 rounds of 8 x mixer + cache line s[0] & mask; 8 x final mixer. -IGNEUM_HD void mh_item(const uint32_t* cache, uint32_t t, uint32_t* s) { - s[0] = 0x3067619fu; - s[1] = 0x3c269176u; - s[2] = 0x84a03b03u; - s[3] = 0xf8c63294u; - s[4] = 0xff977c5bu; - s[5] = 0xe60def3eu; - s[6] = 0x63630141u; - s[7] = 0xb8fbcb58u; - s[8] = t * 0x42146205u + 0xbab68293u; - s[9] = t * 0x52cbe0fbu + 0xcc162340u; - s[10] = t * 0x7ecf4a03u + 0x6ce151ccu; - s[11] = t * 0x6728907fu + 0xe62b8997u; - s[12] = t * 0xd81d9751u + 0xc9c80297u; - s[13] = t * 0x132952c3u + 0xf74a1654u; - s[14] = t * 0xf60de277u + 0x3d704af5u; - s[15] = t * 0x05358035u + 0x3cf522b7u; - for (uint32_t r = 0u; r < 8u; ++r) { - for (uint32_t j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (r * 8u + j + 1u)); - const uint32_t* line = cache + ((s[0] & MH_CACHE_LINE_MASK) * 16u); - for (uint32_t i = 0u; i < 16u; ++i) s[i] ^= line[i]; - } - for (uint32_t j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (64u + j + 1u)); -} -// dataset[w] without the dataset: derive item w >> 4 and take word w & 15. -IGNEUM_HD uint32_t mh_word(const uint32_t* cache, uint32_t w) { uint32_t s[16]; mh_item(cache, w >> 4u, s); return s[w & 15u]; } diff --git a/proto-cuda/packs-ca4/mx8_shl256x27/memhard.metal b/proto-cuda/packs-ca4/mx8_shl256x27/memhard.metal deleted file mode 100644 index 01b263d6d..000000000 --- a/proto-cuda/packs-ca4/mx8_shl256x27/memhard.metal +++ /dev/null @@ -1,107 +0,0 @@ -#include -using namespace metal; -// Memory-hard dataset core (MEMHARD.md). Cache: 2^26 words in 2^16 segments of 64 chained ChaCha12 lines. -// Item: 8 rounds of 8 x seed-parameterised mixer + one 64-byte cache read, then 8 x final mixer (class v3, mixer multiplier 8, -// docs/plans/mixer-x4.md: the round key of application j of round r is 0x9E3779B9 * (r * m + j + 1)). All parameters are literals. -#define MH_CACHE_LINE_MASK 0x003fffffu -#define MH_SEGMENT_LINES 64u -#define MH_QR(a, b, c, d, r1, r2, r3, r4) { a += b; d ^= a; d = mh_rotl(d, r1); c += d; b ^= c; b = mh_rotl(b, r2); a += b; d ^= a; d = mh_rotl(d, r3); c += d; b ^= c; b = mh_rotl(b, r4); } -inline uint mh_rotl(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 at every call site - -// y = ChaCha12 core(x) + x -inline void mh_chacha_block(const thread uint* x, thread uint* y) { - for (uint i = 0u; i < 16u; ++i) y[i] = x[i]; - for (uint r = 0u; r < 6u; ++r) { - MH_QR(y[0], y[4], y[8], y[12], 16u, 12u, 8u, 7u) MH_QR(y[1], y[5], y[9], y[13], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[6], y[10], y[14], 16u, 12u, 8u, 7u) MH_QR(y[3], y[7], y[11], y[15], 16u, 12u, 8u, 7u) - MH_QR(y[0], y[5], y[10], y[15], 16u, 12u, 8u, 7u) MH_QR(y[1], y[6], y[11], y[12], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[7], y[8], y[13], 16u, 12u, 8u, 7u) MH_QR(y[3], y[4], y[9], y[14], 16u, 12u, 8u, 7u) - } - for (uint i = 0u; i < 16u; ++i) y[i] += x[i]; -} - -// One cache segment: 64 chained lines written at cache[seg * 1024]. in_j = prev ^ (sigma || K || seg || j || tag), prev_0 = 0. -inline void mh_cache_segment(device uint* cache, uint seg) { - uint prev[16]; uint x[16]; uint y[16]; - for (uint i = 0u; i < 16u; ++i) prev[i] = 0u; - for (uint j = 0u; j < MH_SEGMENT_LINES; ++j) { - x[0] = 0x61707865u ^ prev[0]; x[1] = 0x3320646eu ^ prev[1]; x[2] = 0x79622d32u ^ prev[2]; x[3] = 0x6b206574u ^ prev[3]; - x[4] = 0x3067619fu ^ prev[4]; - x[5] = 0x3c269176u ^ prev[5]; - x[6] = 0x84a03b03u ^ prev[6]; - x[7] = 0xf8c63294u ^ prev[7]; - x[8] = 0xff977c5bu ^ prev[8]; - x[9] = 0xe60def3eu ^ prev[9]; - x[10] = 0x63630141u ^ prev[10]; - x[11] = 0xb8fbcb58u ^ prev[11]; - x[12] = seg ^ prev[12]; x[13] = j ^ prev[13]; x[14] = 0x49676e65u ^ prev[14]; x[15] = 0x756d4d48u ^ prev[15]; - mh_chacha_block(x, y); - device uint* line = cache + ((seg * MH_SEGMENT_LINES + j) * 16u); - for (uint i = 0u; i < 16u; ++i) { line[i] = y[i]; prev[i] = y[i]; } - } -} - -// M_r: per word (s ^ (RC + rk)) * MUL, then a column round and a diagonal round with the seed-drawn rotations. -inline void mh_mixer(thread uint* s, uint rk) { - s[0] = (s[0] ^ (0xbab68293u + rk)) * 0x42146205u; - s[1] = (s[1] ^ (0xcc162340u + rk)) * 0x52cbe0fbu; - s[2] = (s[2] ^ (0x6ce151ccu + rk)) * 0x7ecf4a03u; - s[3] = (s[3] ^ (0xe62b8997u + rk)) * 0x6728907fu; - s[4] = (s[4] ^ (0xc9c80297u + rk)) * 0xd81d9751u; - s[5] = (s[5] ^ (0xf74a1654u + rk)) * 0x132952c3u; - s[6] = (s[6] ^ (0x3d704af5u + rk)) * 0xf60de277u; - s[7] = (s[7] ^ (0x3cf522b7u + rk)) * 0x05358035u; - s[8] = (s[8] ^ (0x2b9cac04u + rk)) * 0xbaf6499du; - s[9] = (s[9] ^ (0xa880ac10u + rk)) * 0xe4db9667u; - s[10] = (s[10] ^ (0x13e5dd1du + rk)) * 0x3e98f45du; - s[11] = (s[11] ^ (0x6fc3e233u + rk)) * 0xd0004eddu; - s[12] = (s[12] ^ (0x2d83eeacu + rk)) * 0x2691630du; - s[13] = (s[13] ^ (0x9006e8bfu + rk)) * 0x9beb3bcfu; - s[14] = (s[14] ^ (0x2c4b5362u + rk)) * 0xab310379u; - s[15] = (s[15] ^ (0x31b49ee2u + rk)) * 0x99cfb423u; - MH_QR(s[0], s[4], s[8], s[12], 20u, 20u, 19u, 4u) MH_QR(s[1], s[5], s[9], s[13], 20u, 20u, 19u, 4u) - MH_QR(s[2], s[6], s[10], s[14], 20u, 20u, 19u, 4u) MH_QR(s[3], s[7], s[11], s[15], 20u, 20u, 19u, 4u) - MH_QR(s[0], s[5], s[10], s[15], 26u, 3u, 3u, 27u) MH_QR(s[1], s[6], s[11], s[12], 26u, 3u, 3u, 27u) - MH_QR(s[2], s[7], s[8], s[13], 26u, 3u, 3u, 27u) MH_QR(s[3], s[4], s[9], s[14], 26u, 3u, 3u, 27u) -} - -// Item t: 16 words. s = (K, t * MUL[i] + RC[i]); 8 rounds of 8 x mixer + cache line s[0] & mask; 8 x final mixer. -inline void mh_item(device const uint* cache, uint t, thread uint* s) { - s[0] = 0x3067619fu; - s[1] = 0x3c269176u; - s[2] = 0x84a03b03u; - s[3] = 0xf8c63294u; - s[4] = 0xff977c5bu; - s[5] = 0xe60def3eu; - s[6] = 0x63630141u; - s[7] = 0xb8fbcb58u; - s[8] = t * 0x42146205u + 0xbab68293u; - s[9] = t * 0x52cbe0fbu + 0xcc162340u; - s[10] = t * 0x7ecf4a03u + 0x6ce151ccu; - s[11] = t * 0x6728907fu + 0xe62b8997u; - s[12] = t * 0xd81d9751u + 0xc9c80297u; - s[13] = t * 0x132952c3u + 0xf74a1654u; - s[14] = t * 0xf60de277u + 0x3d704af5u; - s[15] = t * 0x05358035u + 0x3cf522b7u; - for (uint r = 0u; r < 8u; ++r) { - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (r * 8u + j + 1u)); - device const uint* line = cache + ((s[0] & MH_CACHE_LINE_MASK) * 16u); - for (uint i = 0u; i < 16u; ++i) s[i] ^= line[i]; - } - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (64u + j + 1u)); -} -// dataset[w] without the dataset: derive item w >> 4 and take word w & 15. -inline uint mh_word(device const uint* cache, uint w) { uint s[16]; mh_item(cache, w >> 4u, s); return s[w & 15u]; } - -// One thread per segment (2^16 threads). -kernel void igneum_cache_fill(device uint* cache [[buffer(0)]], uint gid [[thread_position_in_grid]]) { - mh_cache_segment(cache, gid); -} -// One thread per 64-byte item (dataset words / 16 threads). -kernel void igneum_build(device const uint* cache [[buffer(0)]], device uint* dataset [[buffer(1)]], - uint gid [[thread_position_in_grid]]) { - uint s[16]; - mh_item(cache, gid, s); - device uint* d = dataset + gid * 16u; - for (uint i = 0u; i < 16u; ++i) d[i] = s[i]; -} diff --git a/proto-cuda/packs-ca4/mx8_shl256x27/program.h b/proto-cuda/packs-ca4/mx8_shl256x27/program.h deleted file mode 100644 index 98595b57e..000000000 --- a/proto-cuda/packs-ca4/mx8_shl256x27/program.h +++ /dev/null @@ -1,72 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// Program metadata for host.cu plus the launch wrappers defined in kernel.cu. -// Also included by proto-opencl/host.c (C99), which defines IGNEUM_NO_CUDA first and reads only the macros. -#pragma once -#ifdef __cplusplus -#include -#else -#include -#endif -#ifndef IGNEUM_NO_CUDA -#include -#endif - -#define IGNEUM_SEED_STRING "igneum-genesis" -#define IGNEUM_SEED_BYTES_HEX "69676e65756d2d67656e65736973" -#define IGNEUM_GENERATOR 2 -#define IGNEUM_PROGRAM_ATTEMPT 0 -#define IGNEUM_PROGRAM_ID 0x854050a4293f0615ull -#define IGNEUM_DAY_STRING "2026-10-03" -#define IGNEUM_DAY_BYTES_HEX "6461792f323032362d31302d3033" -#define IGNEUM_DAY0 0x3067619fu -#define IGNEUM_DAY1 0x3c269176u -#define IGNEUM_DATASET_LOG2 28 -#define IGNEUM_MASK 0x0fffffffu -#define IGNEUM_LANES 32 -#define IGNEUM_ITERATIONS 8 -#define IGNEUM_INSTR_COUNT 64 -#define IGNEUM_LOADS_PER_HASH 128 -#define IGNEUM_WIDE_LOADS_PER_HASH 0 -#define IGNEUM_OP_MIX "load=16 add=8 shfl=8 xor=6 mad=5 mul=5 mulhi=5 sub=4 rotl=3 rotr=3 or=1" -// Class v3 construction (Counter ASIC 2.0, 5 October 2026, docs/plans/mixer-x4.md): version 2 loads; the dataset item -// derivation applies the mixer IGNEUM_MIXER_MULT times per round (memhard.h), and the cache follows the growth rule. -#define IGNEUM_LOAD_CLASS "mx8+shl256x27" -#define IGNEUM_CLASS_MIXER_MULT 8 -#define IGNEUM_CACHE_GROWTH 1 // 1: cache words = 2^(26 + doublings(day)), doublings = floor(log2(1 + day / 1460)) -#define IGNEUM_LOAD_SLOTS 16 -#define IGNEUM_LOAD_MIX { 100, 0, 0 } -#define IGNEUM_LOAD_WIDTH_COUNTS { 16, 0, 0 } // loads of 4, 16, 64 bytes per program -#define IGNEUM_BYTES_PER_HASH 512 -#define IGNEUM_FOLD_ROT 11 -#define IGNEUM_FOLD_MUL 0x9e3779b1u -// Latency-shadow block (Counter ASIC 3.0 item 8, docs/analysis/latency-shadow-2026-10-06.md): NOT the lottery hash. A block of -// IGNEUM_SHADOW_INSTRS ALU instructions (the ten non-load families) runs IGNEUM_SHADOW_REPS times at the end of every -// iteration; the 16 loads, the acceptance rule and the base program are the class's without the shadow. -#define IGNEUM_SHADOW_INSTRS 256 -#define IGNEUM_SHADOW_REPS 27 -#define IGNEUM_SHADOW_INSTRS_PER_HASH 55296 -#define IGNEUM_SHADOW_OP_MIX "add=47 rotl=30 xor=30 shfl=29 mad=27 mul=22 sub=21 rotr=20 mulhi=18 or=12" -// Counter ASIC 4.0 research (experimental): the block is placed per load, sub-block j (instrs / 16) after the j-th load. -#define IGNEUM_SHADOW_PER_LOAD 1 -// 0 = closed-form dataset (ds_elem), 1 = memory-hard cache construction (MEMHARD.md, memhard.h) -#define IGNEUM_DATASET_MODE 1 - -#define IGNEUM_SEEDW_INIT { 0x67a9a7beu, 0x1a155b25u, 0xfddfb732u, 0x4b5af2e8u, 0xc55caf33u, 0xa27c13b7u, 0x06628a48u, 0x03852469u } -#define IGNEUM_KEY_INIT { 0x3067619fu, 0x3c269176u, 0x84a03b03u, 0xf8c63294u, 0xff977c5bu, 0xe60def3eu, 0x63630141u, 0xb8fbcb58u } -#define IGNEUM_CACHE_LOG2_WORDS 26 -#define IGNEUM_CACHE_SEGMENT_LOG2_LINES 6 -#define IGNEUM_CACHE_SEGMENTS 65536u -#define IGNEUM_ITEM_ROUNDS 8 -#define IGNEUM_MIXER_MULT 8 // mixer applications per round and after the last read (class v3, docs/plans/mixer-x4.md) -#define IGNEUM_MIX_ROT_INIT { 20u, 20u, 19u, 4u, 26u, 3u, 3u, 27u } -#define IGNEUM_MIX_MUL_INIT { 0x42146205u, 0x52cbe0fbu, 0x7ecf4a03u, 0x6728907fu, 0xd81d9751u, 0x132952c3u, 0xf60de277u, 0x05358035u, 0xbaf6499du, 0xe4db9667u, 0x3e98f45du, 0xd0004eddu, 0x2691630du, 0x9beb3bcfu, 0xab310379u, 0x99cfb423u } -#define IGNEUM_MIX_RC_INIT { 0xbab68293u, 0xcc162340u, 0x6ce151ccu, 0xe62b8997u, 0xc9c80297u, 0xf74a1654u, 0x3d704af5u, 0x3cf522b7u, 0x2b9cac04u, 0xa880ac10u, 0x13e5dd1du, 0x6fc3e233u, 0x2d83eeacu, 0x9006e8bfu, 0x2c4b5362u, 0x31b49ee2u } - -#ifndef IGNEUM_NO_CUDA -// Defined in kernel.cu. All launch on the default stream and return cudaGetLastError(). -cudaError_t igneum_launch_cache_fill(uint32_t* cache, uint32_t nSegments); -cudaError_t igneum_launch_build(uint32_t* ds, const uint32_t* cache, uint32_t nItems); -cudaError_t igneum_launch_hash(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask, - uint32_t nonces, uint32_t blockWarps); -cudaError_t igneum_hash_info(int* numRegs, int* blocksPerSM, uint32_t blockWarps); -#endif diff --git a/proto-cuda/packs-ca4/mx8_shl256x27/program.json b/proto-cuda/packs-ca4/mx8_shl256x27/program.json deleted file mode 100644 index a4232b87b..000000000 --- a/proto-cuda/packs-ca4/mx8_shl256x27/program.json +++ /dev/null @@ -1,390 +0,0 @@ -{ - "format": "igneum-program-pack-3", - "generator": 2, - "attempt": 0, - "program_id": "0x854050a4293f0615", - "program_id_derivation": "FNV-1a 64 over 'igneum-program/' || generator_le32 || seed_words as little-endian bytes || attempt_le32", - "dataset_mode": "memory-hard", - "seed": "igneum-genesis", - "seed_bytes": "69676e65756d2d67656e65736973", - "seed_words": ["0x67a9a7be", "0x1a155b25", "0xfddfb732", "0x4b5af2e8", "0xc55caf33", "0xa27c13b7", "0x06628a48", "0x03852469"], - "seed_derivation": "seed_words = FNV-1a 64 over seed_bytes (attempt 0) or seed_bytes || attempt_le32 (attempt k >= 1), basis ^ (salt * 0x9E3779B97F4A7C15) for salt 0..3, then h ^= h>>33; h *= 0xff51afd7ed558ccd; h ^= h>>33; words[2*salt] = low 32, words[2*salt+1] = high 32", - "generator_rule": "version 2: exactly 16 load slots drawn first from instructions 1..63 (partial Fisher-Yates), the other 48 ops from the ten non-load weights (sum 75); a load's source is drawn from the registers other than dst written by an earlier instruction and not read by a load since; the candidate must pass the acceptance rule of spec 01 section 1.4.6 (static: no cyclically stale load source, every register has an injecting write; dynamic: 64 units on the seed-keyed closed-form dataset with no constant register bit, no lane-constant load site, under 164 saturated final values, every output bit within 136 of 1024, distinct addresses above 245760), else the next attempt of the seed is tried", - "lanes": 32, - "registers": 8, - "iterations": 8, - "instruction_count": 64, - "loads_per_hash": 128, - "load_class": "mx8+shl256x27", - "mixer_mult": 8, - "cache_growth": true, - "mixer": "class v3 (Counter ASIC 2.0, 5 October 2026, docs/plans/mixer-x4.md): every mixer application of the item derivation is 8 applications with round keys (r * 8 + j + 1) * 0x9E3779B9, the 8 dependent cache reads per item unchanged; cache growth rule option C: cache words = 2^(26 + doublings(day)), dataset words = 2^(genesis_log2 + doublings(day)), doublings(day) = floor(log2(1 + day / 1460)) for day = days since genesis", - "load_slots": 16, - "load_mix_percent_4_16_64": [100, 0, 0], - "load_width_counts_4_16_64": [16, 0, 0], - "bytes_per_hash": 512, - "wide_load": "read-width experiment (5 October 2026, docs/plans/read-width.md), NOT the lottery hash: a load of W words (width field, 4 or 16) reads dataset[b .. b + W) with b = (src & mask) & ~(W - 1) and folds every word into dst: x = dst ^ w[0]; for j in 1..W: x = (rotl(x, 11) * 0x9e3779b1) ^ w[j]; dst = x; width 1 is the plain load; the width is drawn per instruction from the class mix with one extra below(100) draw after the nine of version 2, and the program id is FNV-1a 64 over 'igneum-program-rw/' || generator_le32 || seed words || attempt_le32 || mix[3] || load_slots", - "op_mix": {"load": 16, "add": 8, "shfl": 8, "xor": 6, "mad": 5, "mul": 5, "mulhi": 5, "sub": 4, "rotl": 3, "rotr": 3, "or": 1}, - "register_init": "for i in 0..7: x = nonce ^ seed_words[i]; x += 0x9e3779b9 * (i+1) (mod 2^32); x = splitmix32(x); r[i] = x ^ seed_words[(i+1) & 7]", - "splitmix32": "x ^= x>>16; x *= 0x7feb352d; x ^= x>>15; x *= 0x846ca68b; x ^= x>>16", - "iteration": "sel = r0 sampled once at the top of each iteration, then all instructions in order", - "output": "lo = r0 ^ rotl(r1,7) ^ rotl(r2,14) ^ rotl(r3,21); hi = r4 ^ rotl(r5,9) ^ rotl(r6,18) ^ rotl(r7,27); out = (hi << 32) | lo", - "op_semantics": { - "add": "dst = dst + src + (bit `bit` of sel ? imm2 : imm)", - "sub": "dst = dst - src", - "mul": "dst = dst * src (low 32)", - "mulhi": "dst = high 32 bits of dst * src", - "xor": "dst = dst ^ src", - "or": "dst = dst | src", - "rotl": "dst = rotl(dst, rot), rot in 1..31", - "rotr": "dst = rotr(dst, src & 31)", - "mad": "dst = src * src2 + dst", - "shfl": "dst = dst ^ (src of lane (lane ^ mask)), mask in {1,2,4,8,16}, within the 32-lane warp", - "load": "dst = dst ^ dataset[src & dataset.mask]", - "wload": "base = (src of lane 0 & dataset.mask) & ~31; dst = dst ^ dataset[base + lane] (warp-coalesced 128-byte load, lever b, only when --wide-frac > 0)" - }, - "dataset": { - "log2_words": 28, - "bytes": 1073741824, - "mask": "0x0fffffff", - "day": "2026-10-03", - "day_bytes": "6461792f323032362d31302d3033", - "day_words_from": "seed_words_from_bytes(day_bytes)", - "d0": "0x3067619f", - "d1": "0x3c269176", - "mode": "memory-hard", - "spec": "proto-metal/MEMHARD.md", - "key": ["0x3067619f", "0x3c269176", "0x84a03b03", "0xf8c63294", "0xff977c5b", "0xe60def3e", "0x63630141", "0xb8fbcb58"], - "key_derivation": "the 8 words of seed_words_from_bytes(day_bytes); d0, d1 are key[0], key[1]", - "cache": {"log2_words": 26, "bytes": 268435456, "line_words": 16, "segment_lines": 64, "segments": 65536, "block": "ChaCha12 core + feed-forward, rotations 16 12 8 7", "sigma": ["0x61707865", "0x3320646e", "0x79622d32", "0x6b206574"], "tag": ["0x49676e65", "0x756d4d48"], "chain": "in_j = prev_line ^ (sigma[0..3] || key[0..7] || seg || j || tag[0..1]); line_j = block(in_j); prev_0 = 0"}, - "mixer": {"draw": "SplitMix64 seeded with key[0] | key[1] << 32: rot[0..7] = 1 + next() % 31, mul[0..15] = low32(next()) | 1, rc[0..15] = low32(next())", "rot": [20, 20, 19, 4, 26, 3, 3, 27], "mul": ["0x42146205", "0x52cbe0fb", "0x7ecf4a03", "0x6728907f", "0xd81d9751", "0x132952c3", "0xf60de277", "0x05358035", "0xbaf6499d", "0xe4db9667", "0x3e98f45d", "0xd0004edd", "0x2691630d", "0x9beb3bcf", "0xab310379", "0x99cfb423"], "rc": ["0xbab68293", "0xcc162340", "0x6ce151cc", "0xe62b8997", "0xc9c80297", "0xf74a1654", "0x3d704af5", "0x3cf522b7", "0x2b9cac04", "0xa880ac10", "0x13e5dd1d", "0x6fc3e233", "0x2d83eeac", "0x9006e8bf", "0x2c4b5362", "0x31b49ee2"], "round": "for i in 0..15: s[i] = (s[i] ^ (rc[i] + (r+1) * 0x9E3779B9)) * mul[i]; then quarter rounds on columns (0,4,8,12) (1,5,9,13) (2,6,10,14) (3,7,11,15) with rot[0..3] and diagonals (0,5,10,15) (1,6,11,12) (2,7,8,13) (3,4,9,14) with rot[4..7]", "quarter_round": "a += b; d ^= a; d = rotl(d, r1); c += d; b ^= c; b = rotl(b, r2); a += b; d ^= a; d = rotl(d, r3); c += d; b ^= c; b = rotl(b, r4)"}, - "mixer_mult": 8, - "item": "s[0..7] = key; s[8+i] = t * mul[i] + rc[i] for i in 0..7; for r in 0..7: for j in 0..7: s = M(s, rk = (r * 8 + j + 1) * 0x9E3779B9); line = s[0] & 0x003fffff; s[i] ^= cache[line * 16 + i]; then for j in 0..7: s = M(s, rk = (64 + j + 1) * 0x9E3779B9); item(t) = s", - "word": "dataset[w] = item(w >> 4)[w & 15]" - }, - "shadow": {"instrs": 256, "reps": 27, "instrs_per_hash": 55296, "op_mix": {"add": 47, "rotl": 30, "xor": 30, "shfl": 29, "mad": 27, "mul": 22, "sub": 21, "rotr": 20, "mulhi": 18, "or": 12}, "rule": "Counter ASIC 3.0 item 8 (docs/analysis/latency-shadow-2026-10-06.md): after the 64 base instructions the program stream draws instrs more ALU instructions (the non-load table, nine draws each, the source as on an ALU slot); the block runs reps times at the end of every iteration with the iteration's sel; the acceptance rule interprets the base program only", "program_id_suffix": "'shadow/' || instrs_le16 || reps_le16", "instructions": [ - {"i": 0, "op": "add", "dst": 5, "src": 2, "src2": 1, "imm": "0x92199f99", "imm2": "0x8bc12da9", "rot": 9, "bit": 18, "mask": 4}, - {"i": 1, "op": "add", "dst": 0, "src": 7, "src2": 1, "imm": "0x8d72d3ad", "imm2": "0x63079e5a", "rot": 20, "bit": 26, "mask": 4}, - {"i": 2, "op": "shfl", "dst": 6, "src": 3, "src2": 6, "imm": "0x9beaeddf", "imm2": "0x744ecb00", "rot": 10, "bit": 4, "mask": 2}, - {"i": 3, "op": "sub", "dst": 4, "src": 2, "src2": 0, "imm": "0x2a3ddc67", "imm2": "0x72c80241", "rot": 30, "bit": 1, "mask": 16}, - {"i": 4, "op": "add", "dst": 7, "src": 0, "src2": 5, "imm": "0xb21b4bab", "imm2": "0x5d4c7a60", "rot": 17, "bit": 31, "mask": 4}, - {"i": 5, "op": "rotl", "dst": 0, "src": 6, "src2": 3, "imm": "0xe69d7919", "imm2": "0xa048c61e", "rot": 11, "bit": 1, "mask": 8}, - {"i": 6, "op": "add", "dst": 0, "src": 1, "src2": 3, "imm": "0x2fe0e98b", "imm2": "0xc88e2942", "rot": 5, "bit": 16, "mask": 16}, - {"i": 7, "op": "xor", "dst": 7, "src": 1, "src2": 0, "imm": "0xc16efe98", "imm2": "0x01a638c8", "rot": 8, "bit": 17, "mask": 1}, - {"i": 8, "op": "mad", "dst": 1, "src": 6, "src2": 5, "imm": "0x76ec7b8b", "imm2": "0x25663feb", "rot": 29, "bit": 30, "mask": 2}, - {"i": 9, "op": "shfl", "dst": 6, "src": 1, "src2": 0, "imm": "0x6c8ee3cb", "imm2": "0xea93237e", "rot": 27, "bit": 1, "mask": 4}, - {"i": 10, "op": "mad", "dst": 1, "src": 2, "src2": 2, "imm": "0x6dc4ea18", "imm2": "0x6efde6f5", "rot": 3, "bit": 20, "mask": 2}, - {"i": 11, "op": "mad", "dst": 5, "src": 0, "src2": 3, "imm": "0x023613fc", "imm2": "0x18c51939", "rot": 19, "bit": 31, "mask": 1}, - {"i": 12, "op": "shfl", "dst": 2, "src": 6, "src2": 1, "imm": "0x74aec8d2", "imm2": "0x7f7ad29c", "rot": 7, "bit": 8, "mask": 4}, - {"i": 13, "op": "add", "dst": 1, "src": 5, "src2": 6, "imm": "0xbdf8f9a5", "imm2": "0xbc48c63e", "rot": 6, "bit": 25, "mask": 2}, - {"i": 14, "op": "rotl", "dst": 1, "src": 2, "src2": 6, "imm": "0x7ad8ca8b", "imm2": "0x14c712ad", "rot": 29, "bit": 25, "mask": 8}, - {"i": 15, "op": "sub", "dst": 1, "src": 4, "src2": 5, "imm": "0xd3349f69", "imm2": "0x70aac45a", "rot": 29, "bit": 9, "mask": 16}, - {"i": 16, "op": "or", "dst": 7, "src": 1, "src2": 2, "imm": "0x08168ed1", "imm2": "0x28964037", "rot": 26, "bit": 0, "mask": 2}, - {"i": 17, "op": "shfl", "dst": 2, "src": 4, "src2": 6, "imm": "0x98d85f72", "imm2": "0x3dc87042", "rot": 29, "bit": 22, "mask": 8}, - {"i": 18, "op": "xor", "dst": 7, "src": 4, "src2": 0, "imm": "0x651d4a0e", "imm2": "0x93c198bd", "rot": 26, "bit": 12, "mask": 8}, - {"i": 19, "op": "mul", "dst": 6, "src": 1, "src2": 6, "imm": "0xd38ce89d", "imm2": "0x3dfad388", "rot": 5, "bit": 28, "mask": 8}, - {"i": 20, "op": "mad", "dst": 5, "src": 6, "src2": 0, "imm": "0x59d78b36", "imm2": "0xc4db274e", "rot": 25, "bit": 24, "mask": 1}, - {"i": 21, "op": "sub", "dst": 3, "src": 1, "src2": 7, "imm": "0xf6c6a6c7", "imm2": "0x8536f4e6", "rot": 6, "bit": 12, "mask": 4}, - {"i": 22, "op": "mul", "dst": 6, "src": 0, "src2": 7, "imm": "0x599445b4", "imm2": "0x632f8c32", "rot": 19, "bit": 4, "mask": 8}, - {"i": 23, "op": "add", "dst": 2, "src": 0, "src2": 7, "imm": "0x45c37cec", "imm2": "0x96e8f127", "rot": 15, "bit": 1, "mask": 4}, - {"i": 24, "op": "sub", "dst": 6, "src": 4, "src2": 3, "imm": "0xc1c44491", "imm2": "0x92e3ce57", "rot": 20, "bit": 25, "mask": 4}, - {"i": 25, "op": "mad", "dst": 7, "src": 3, "src2": 4, "imm": "0x89bfb8d3", "imm2": "0x19b5455e", "rot": 22, "bit": 2, "mask": 16}, - {"i": 26, "op": "rotl", "dst": 3, "src": 5, "src2": 7, "imm": "0x2f47ce8d", "imm2": "0x8b458ec5", "rot": 9, "bit": 0, "mask": 4}, - {"i": 27, "op": "sub", "dst": 2, "src": 1, "src2": 2, "imm": "0x9f0dce23", "imm2": "0x3cdca814", "rot": 25, "bit": 1, "mask": 2}, - {"i": 28, "op": "mulhi", "dst": 6, "src": 0, "src2": 3, "imm": "0x61fc9eb8", "imm2": "0x23202e9d", "rot": 30, "bit": 16, "mask": 16}, - {"i": 29, "op": "shfl", "dst": 2, "src": 4, "src2": 3, "imm": "0x339dbd65", "imm2": "0xc175f639", "rot": 15, "bit": 22, "mask": 2}, - {"i": 30, "op": "shfl", "dst": 1, "src": 6, "src2": 1, "imm": "0x5bd14589", "imm2": "0xa68a2bed", "rot": 31, "bit": 31, "mask": 1}, - {"i": 31, "op": "add", "dst": 1, "src": 6, "src2": 1, "imm": "0x37985632", "imm2": "0xb1cdb2ab", "rot": 29, "bit": 1, "mask": 8}, - {"i": 32, "op": "rotr", "dst": 0, "src": 1, "src2": 6, "imm": "0x4b2058f4", "imm2": "0xf06d8ac9", "rot": 9, "bit": 15, "mask": 16}, - {"i": 33, "op": "shfl", "dst": 3, "src": 4, "src2": 4, "imm": "0x2081626c", "imm2": "0x08d1bb87", "rot": 24, "bit": 29, "mask": 2}, - {"i": 34, "op": "shfl", "dst": 0, "src": 3, "src2": 6, "imm": "0xe4fcfe03", "imm2": "0x78a46b13", "rot": 15, "bit": 29, "mask": 4}, - {"i": 35, "op": "mul", "dst": 7, "src": 0, "src2": 4, "imm": "0x33148d30", "imm2": "0x5780a3d6", "rot": 6, "bit": 11, "mask": 2}, - {"i": 36, "op": "add", "dst": 3, "src": 1, "src2": 1, "imm": "0x804e777e", "imm2": "0x856e0180", "rot": 22, "bit": 0, "mask": 16}, - {"i": 37, "op": "xor", "dst": 1, "src": 6, "src2": 0, "imm": "0xc69aa2f6", "imm2": "0xafebe3e8", "rot": 15, "bit": 9, "mask": 16}, - {"i": 38, "op": "mul", "dst": 2, "src": 7, "src2": 4, "imm": "0xeb4c4082", "imm2": "0x3f1e0da7", "rot": 25, "bit": 20, "mask": 16}, - {"i": 39, "op": "add", "dst": 6, "src": 5, "src2": 5, "imm": "0x0b06c8a7", "imm2": "0xe9bd0cc4", "rot": 6, "bit": 2, "mask": 4}, - {"i": 40, "op": "mul", "dst": 5, "src": 7, "src2": 7, "imm": "0x070af1b6", "imm2": "0x6b648e75", "rot": 10, "bit": 6, "mask": 16}, - {"i": 41, "op": "mulhi", "dst": 2, "src": 3, "src2": 2, "imm": "0x389753b2", "imm2": "0x9e657305", "rot": 30, "bit": 2, "mask": 4}, - {"i": 42, "op": "xor", "dst": 2, "src": 0, "src2": 4, "imm": "0x0960e5b9", "imm2": "0xa925a406", "rot": 4, "bit": 6, "mask": 16}, - {"i": 43, "op": "rotl", "dst": 0, "src": 1, "src2": 1, "imm": "0x8eabe09f", "imm2": "0x76ef71e2", "rot": 4, "bit": 19, "mask": 8}, - {"i": 44, "op": "mulhi", "dst": 4, "src": 3, "src2": 7, "imm": "0x6a34768a", "imm2": "0x2e427c64", "rot": 21, "bit": 5, "mask": 4}, - {"i": 45, "op": "add", "dst": 6, "src": 7, "src2": 5, "imm": "0xee822e17", "imm2": "0xcdcb63f6", "rot": 27, "bit": 12, "mask": 16}, - {"i": 46, "op": "mad", "dst": 3, "src": 2, "src2": 0, "imm": "0xc89ead43", "imm2": "0x4d3108b2", "rot": 26, "bit": 17, "mask": 1}, - {"i": 47, "op": "shfl", "dst": 4, "src": 3, "src2": 0, "imm": "0xdd62be9e", "imm2": "0xf243f6f2", "rot": 8, "bit": 4, "mask": 8}, - {"i": 48, "op": "mulhi", "dst": 6, "src": 5, "src2": 0, "imm": "0xd3f7fa72", "imm2": "0x0debc83f", "rot": 25, "bit": 5, "mask": 2}, - {"i": 49, "op": "sub", "dst": 0, "src": 2, "src2": 6, "imm": "0x7afadd15", "imm2": "0xc07fc39c", "rot": 30, "bit": 29, "mask": 8}, - {"i": 50, "op": "sub", "dst": 3, "src": 5, "src2": 3, "imm": "0x179b78ec", "imm2": "0xaeccde37", "rot": 30, "bit": 17, "mask": 1}, - {"i": 51, "op": "rotr", "dst": 1, "src": 4, "src2": 1, "imm": "0xa4bfcea6", "imm2": "0xbf63bb2f", "rot": 2, "bit": 21, "mask": 2}, - {"i": 52, "op": "mad", "dst": 6, "src": 7, "src2": 7, "imm": "0xabf5ed10", "imm2": "0x8a285f51", "rot": 3, "bit": 23, "mask": 2}, - {"i": 53, "op": "xor", "dst": 5, "src": 3, "src2": 5, "imm": "0x88b416eb", "imm2": "0x36271d87", "rot": 19, "bit": 10, "mask": 2}, - {"i": 54, "op": "sub", "dst": 1, "src": 0, "src2": 1, "imm": "0xa3417dd3", "imm2": "0xcd98f620", "rot": 28, "bit": 2, "mask": 1}, - {"i": 55, "op": "sub", "dst": 5, "src": 6, "src2": 2, "imm": "0x45996c6f", "imm2": "0xe3d41087", "rot": 4, "bit": 31, "mask": 1}, - {"i": 56, "op": "add", "dst": 3, "src": 2, "src2": 0, "imm": "0x3dfad1b6", "imm2": "0xd4758987", "rot": 27, "bit": 10, "mask": 2}, - {"i": 57, "op": "add", "dst": 4, "src": 1, "src2": 3, "imm": "0xfca75bc2", "imm2": "0x0602d6be", "rot": 19, "bit": 0, "mask": 16}, - {"i": 58, "op": "mulhi", "dst": 5, "src": 6, "src2": 5, "imm": "0x83250a7b", "imm2": "0x2f93d53b", "rot": 25, "bit": 7, "mask": 2}, - {"i": 59, "op": "shfl", "dst": 2, "src": 1, "src2": 0, "imm": "0x13f4a089", "imm2": "0x145ea125", "rot": 12, "bit": 3, "mask": 2}, - {"i": 60, "op": "rotl", "dst": 2, "src": 5, "src2": 6, "imm": "0xad3170e3", "imm2": "0x15db04d1", "rot": 9, "bit": 13, "mask": 2}, - {"i": 61, "op": "or", "dst": 4, "src": 6, "src2": 2, "imm": "0x4b6305b2", "imm2": "0x6e7b2e9c", "rot": 27, "bit": 16, "mask": 4}, - {"i": 62, "op": "rotr", "dst": 6, "src": 4, "src2": 6, "imm": "0xfcd4b1c9", "imm2": "0xaf5c733b", "rot": 6, "bit": 21, "mask": 16}, - {"i": 63, "op": "mul", "dst": 2, "src": 4, "src2": 6, "imm": "0x95cebb3e", "imm2": "0xbba9cdfc", "rot": 19, "bit": 23, "mask": 1}, - {"i": 64, "op": "xor", "dst": 0, "src": 5, "src2": 7, "imm": "0x5f7579ff", "imm2": "0xb104e01a", "rot": 25, "bit": 12, "mask": 8}, - {"i": 65, "op": "xor", "dst": 2, "src": 7, "src2": 3, "imm": "0x749c7611", "imm2": "0xf17df3bb", "rot": 27, "bit": 26, "mask": 2}, - {"i": 66, "op": "add", "dst": 2, "src": 1, "src2": 6, "imm": "0xd71c02ff", "imm2": "0x8596687a", "rot": 4, "bit": 6, "mask": 2}, - {"i": 67, "op": "rotl", "dst": 1, "src": 3, "src2": 2, "imm": "0xd2864071", "imm2": "0xaa644ef3", "rot": 21, "bit": 5, "mask": 1}, - {"i": 68, "op": "mul", "dst": 3, "src": 2, "src2": 1, "imm": "0xd0689a5f", "imm2": "0xa3a1f063", "rot": 13, "bit": 28, "mask": 2}, - {"i": 69, "op": "shfl", "dst": 7, "src": 5, "src2": 6, "imm": "0xd01476eb", "imm2": "0x76abb5c2", "rot": 7, "bit": 30, "mask": 2}, - {"i": 70, "op": "mul", "dst": 3, "src": 2, "src2": 5, "imm": "0x59a75c10", "imm2": "0x1e1fbb72", "rot": 20, "bit": 16, "mask": 1}, - {"i": 71, "op": "mad", "dst": 0, "src": 2, "src2": 5, "imm": "0x39cca1df", "imm2": "0x22c057ac", "rot": 5, "bit": 23, "mask": 16}, - {"i": 72, "op": "add", "dst": 6, "src": 7, "src2": 3, "imm": "0x3c6fe15d", "imm2": "0x08ac5733", "rot": 14, "bit": 0, "mask": 4}, - {"i": 73, "op": "shfl", "dst": 3, "src": 2, "src2": 6, "imm": "0x2098627d", "imm2": "0xf4fecc81", "rot": 20, "bit": 30, "mask": 8}, - {"i": 74, "op": "mul", "dst": 3, "src": 6, "src2": 5, "imm": "0xf82e9e23", "imm2": "0x3ad62132", "rot": 10, "bit": 14, "mask": 16}, - {"i": 75, "op": "xor", "dst": 6, "src": 7, "src2": 4, "imm": "0x70548a91", "imm2": "0xa9715d2e", "rot": 6, "bit": 24, "mask": 1}, - {"i": 76, "op": "or", "dst": 3, "src": 0, "src2": 7, "imm": "0xa164325f", "imm2": "0xf300838b", "rot": 23, "bit": 23, "mask": 16}, - {"i": 77, "op": "add", "dst": 2, "src": 4, "src2": 6, "imm": "0x295d5fae", "imm2": "0xc100b495", "rot": 7, "bit": 1, "mask": 2}, - {"i": 78, "op": "mulhi", "dst": 3, "src": 7, "src2": 4, "imm": "0xb62cca87", "imm2": "0x2ebde415", "rot": 14, "bit": 7, "mask": 2}, - {"i": 79, "op": "or", "dst": 4, "src": 1, "src2": 7, "imm": "0xfcbc482d", "imm2": "0x8876e6cd", "rot": 29, "bit": 5, "mask": 2}, - {"i": 80, "op": "rotr", "dst": 4, "src": 3, "src2": 0, "imm": "0x6d64013b", "imm2": "0x675f4a8d", "rot": 26, "bit": 18, "mask": 8}, - {"i": 81, "op": "add", "dst": 4, "src": 3, "src2": 3, "imm": "0x274a9221", "imm2": "0x5cc59530", "rot": 15, "bit": 15, "mask": 2}, - {"i": 82, "op": "add", "dst": 7, "src": 3, "src2": 0, "imm": "0xc8651f8e", "imm2": "0x141479ec", "rot": 18, "bit": 5, "mask": 1}, - {"i": 83, "op": "rotr", "dst": 3, "src": 6, "src2": 0, "imm": "0xcc8a7766", "imm2": "0xc2eb5161", "rot": 4, "bit": 29, "mask": 2}, - {"i": 84, "op": "mad", "dst": 2, "src": 4, "src2": 7, "imm": "0xbc507d51", "imm2": "0x0b1196fd", "rot": 9, "bit": 7, "mask": 8}, - {"i": 85, "op": "shfl", "dst": 2, "src": 5, "src2": 5, "imm": "0x5a156c90", "imm2": "0xa6b3fbfa", "rot": 11, "bit": 2, "mask": 16}, - {"i": 86, "op": "xor", "dst": 3, "src": 2, "src2": 1, "imm": "0x8b042658", "imm2": "0xacf37a8f", "rot": 12, "bit": 23, "mask": 16}, - {"i": 87, "op": "shfl", "dst": 5, "src": 7, "src2": 2, "imm": "0x1a214238", "imm2": "0x017fdf5d", "rot": 29, "bit": 14, "mask": 2}, - {"i": 88, "op": "xor", "dst": 0, "src": 4, "src2": 2, "imm": "0xd523e612", "imm2": "0x2158c2ed", "rot": 30, "bit": 14, "mask": 4}, - {"i": 89, "op": "add", "dst": 3, "src": 2, "src2": 7, "imm": "0x53f915c2", "imm2": "0x883c0c92", "rot": 9, "bit": 18, "mask": 8}, - {"i": 90, "op": "rotr", "dst": 7, "src": 5, "src2": 5, "imm": "0xbd633b21", "imm2": "0xcf8c356c", "rot": 25, "bit": 31, "mask": 4}, - {"i": 91, "op": "add", "dst": 3, "src": 1, "src2": 1, "imm": "0xe2be00a5", "imm2": "0x3cc5bd20", "rot": 10, "bit": 31, "mask": 1}, - {"i": 92, "op": "shfl", "dst": 7, "src": 2, "src2": 4, "imm": "0x3d3da600", "imm2": "0x1f22df89", "rot": 4, "bit": 24, "mask": 1}, - {"i": 93, "op": "rotr", "dst": 6, "src": 2, "src2": 7, "imm": "0xb0f57471", "imm2": "0x8f2a7eca", "rot": 16, "bit": 25, "mask": 1}, - {"i": 94, "op": "rotl", "dst": 7, "src": 0, "src2": 4, "imm": "0x00a21815", "imm2": "0xeb4d7218", "rot": 14, "bit": 18, "mask": 16}, - {"i": 95, "op": "mad", "dst": 4, "src": 5, "src2": 5, "imm": "0xc4c3828e", "imm2": "0xfb7bba17", "rot": 16, "bit": 10, "mask": 16}, - {"i": 96, "op": "mad", "dst": 2, "src": 4, "src2": 5, "imm": "0xfc5bbc75", "imm2": "0xfc43468f", "rot": 31, "bit": 20, "mask": 16}, - {"i": 97, "op": "xor", "dst": 1, "src": 0, "src2": 2, "imm": "0x1fe9c249", "imm2": "0x164bb16b", "rot": 15, "bit": 9, "mask": 4}, - {"i": 98, "op": "mul", "dst": 5, "src": 4, "src2": 1, "imm": "0xeda725fa", "imm2": "0x66f7e9a2", "rot": 21, "bit": 6, "mask": 2}, - {"i": 99, "op": "sub", "dst": 2, "src": 0, "src2": 7, "imm": "0x8fb29f7d", "imm2": "0xf530eda1", "rot": 27, "bit": 30, "mask": 4}, - {"i": 100, "op": "rotl", "dst": 7, "src": 2, "src2": 0, "imm": "0x9f4de742", "imm2": "0x9b5ff871", "rot": 30, "bit": 21, "mask": 16}, - {"i": 101, "op": "add", "dst": 5, "src": 6, "src2": 7, "imm": "0x423fd9c9", "imm2": "0xbfd646cb", "rot": 17, "bit": 17, "mask": 2}, - {"i": 102, "op": "add", "dst": 7, "src": 6, "src2": 3, "imm": "0x8d3c011d", "imm2": "0x19b74a43", "rot": 12, "bit": 11, "mask": 4}, - {"i": 103, "op": "rotl", "dst": 5, "src": 6, "src2": 0, "imm": "0xcfc70303", "imm2": "0xf2b3e8ef", "rot": 6, "bit": 24, "mask": 1}, - {"i": 104, "op": "mul", "dst": 0, "src": 4, "src2": 5, "imm": "0x650475eb", "imm2": "0x11dcbd94", "rot": 15, "bit": 10, "mask": 1}, - {"i": 105, "op": "or", "dst": 0, "src": 5, "src2": 3, "imm": "0xaacaa145", "imm2": "0x9139d3fe", "rot": 4, "bit": 18, "mask": 2}, - {"i": 106, "op": "add", "dst": 0, "src": 1, "src2": 7, "imm": "0x8ce14721", "imm2": "0x7dcb7e18", "rot": 24, "bit": 15, "mask": 8}, - {"i": 107, "op": "rotl", "dst": 0, "src": 3, "src2": 3, "imm": "0xb36d6d98", "imm2": "0x2c3390c8", "rot": 15, "bit": 10, "mask": 16}, - {"i": 108, "op": "sub", "dst": 4, "src": 2, "src2": 5, "imm": "0xf6bbdaef", "imm2": "0x9db6f65e", "rot": 25, "bit": 10, "mask": 1}, - {"i": 109, "op": "mulhi", "dst": 2, "src": 0, "src2": 0, "imm": "0xb1721fd6", "imm2": "0xd96d52c9", "rot": 1, "bit": 27, "mask": 8}, - {"i": 110, "op": "mulhi", "dst": 1, "src": 0, "src2": 5, "imm": "0xfb4ca37f", "imm2": "0xb6ec7dbe", "rot": 27, "bit": 12, "mask": 8}, - {"i": 111, "op": "add", "dst": 0, "src": 2, "src2": 0, "imm": "0x2d9fc6b9", "imm2": "0x3c179ad8", "rot": 9, "bit": 28, "mask": 16}, - {"i": 112, "op": "mulhi", "dst": 2, "src": 0, "src2": 6, "imm": "0x71a9f6bd", "imm2": "0xd3417bf5", "rot": 2, "bit": 14, "mask": 8}, - {"i": 113, "op": "sub", "dst": 6, "src": 3, "src2": 5, "imm": "0xa3d19400", "imm2": "0x15df7330", "rot": 5, "bit": 2, "mask": 8}, - {"i": 114, "op": "mad", "dst": 7, "src": 6, "src2": 3, "imm": "0x1400d92e", "imm2": "0xfa4a158e", "rot": 16, "bit": 9, "mask": 1}, - {"i": 115, "op": "rotr", "dst": 5, "src": 1, "src2": 3, "imm": "0xd01684c3", "imm2": "0x6367febb", "rot": 23, "bit": 19, "mask": 2}, - {"i": 116, "op": "rotr", "dst": 6, "src": 4, "src2": 2, "imm": "0x8cd9eb96", "imm2": "0x98f33b24", "rot": 25, "bit": 1, "mask": 2}, - {"i": 117, "op": "add", "dst": 2, "src": 1, "src2": 7, "imm": "0x502138e2", "imm2": "0x47359729", "rot": 5, "bit": 22, "mask": 4}, - {"i": 118, "op": "mad", "dst": 3, "src": 2, "src2": 0, "imm": "0xd94a35c7", "imm2": "0xb83416c3", "rot": 11, "bit": 3, "mask": 4}, - {"i": 119, "op": "mul", "dst": 1, "src": 3, "src2": 6, "imm": "0x09b30cf7", "imm2": "0xef3b98e7", "rot": 17, "bit": 23, "mask": 8}, - {"i": 120, "op": "mad", "dst": 2, "src": 0, "src2": 4, "imm": "0x56416fe0", "imm2": "0x5e1dc8f3", "rot": 17, "bit": 14, "mask": 2}, - {"i": 121, "op": "mul", "dst": 0, "src": 3, "src2": 2, "imm": "0x2892697c", "imm2": "0x9cb3b14e", "rot": 29, "bit": 25, "mask": 2}, - {"i": 122, "op": "mulhi", "dst": 2, "src": 0, "src2": 3, "imm": "0xc33089a1", "imm2": "0xd6c5b530", "rot": 27, "bit": 13, "mask": 8}, - {"i": 123, "op": "mul", "dst": 5, "src": 6, "src2": 4, "imm": "0xdfe04e4a", "imm2": "0x3cc40160", "rot": 3, "bit": 14, "mask": 2}, - {"i": 124, "op": "mad", "dst": 4, "src": 2, "src2": 4, "imm": "0xb33dc1b7", "imm2": "0xab97743a", "rot": 7, "bit": 21, "mask": 4}, - {"i": 125, "op": "shfl", "dst": 4, "src": 2, "src2": 0, "imm": "0xfd469909", "imm2": "0xfc85dc55", "rot": 28, "bit": 24, "mask": 2}, - {"i": 126, "op": "xor", "dst": 2, "src": 0, "src2": 5, "imm": "0xa0094578", "imm2": "0xf1bee474", "rot": 31, "bit": 7, "mask": 2}, - {"i": 127, "op": "rotl", "dst": 1, "src": 7, "src2": 2, "imm": "0xb7ae00a0", "imm2": "0x86cce297", "rot": 7, "bit": 31, "mask": 8}, - {"i": 128, "op": "shfl", "dst": 7, "src": 2, "src2": 7, "imm": "0x019d1940", "imm2": "0x3fe9e7dd", "rot": 14, "bit": 4, "mask": 4}, - {"i": 129, "op": "rotl", "dst": 6, "src": 7, "src2": 7, "imm": "0x40a74cc4", "imm2": "0x09eb4adf", "rot": 17, "bit": 30, "mask": 1}, - {"i": 130, "op": "add", "dst": 2, "src": 5, "src2": 5, "imm": "0x33dff776", "imm2": "0x6c57e4e7", "rot": 27, "bit": 15, "mask": 4}, - {"i": 131, "op": "or", "dst": 0, "src": 3, "src2": 1, "imm": "0xe0c53bb9", "imm2": "0x124404f6", "rot": 4, "bit": 21, "mask": 16}, - {"i": 132, "op": "rotr", "dst": 5, "src": 0, "src2": 1, "imm": "0xe4353fce", "imm2": "0x559d0118", "rot": 2, "bit": 29, "mask": 4}, - {"i": 133, "op": "add", "dst": 2, "src": 3, "src2": 6, "imm": "0x5db25b34", "imm2": "0xe8212c0c", "rot": 21, "bit": 30, "mask": 1}, - {"i": 134, "op": "xor", "dst": 4, "src": 3, "src2": 6, "imm": "0x7366e50e", "imm2": "0x7cc1ffbd", "rot": 19, "bit": 18, "mask": 16}, - {"i": 135, "op": "xor", "dst": 6, "src": 1, "src2": 2, "imm": "0xa36decb7", "imm2": "0x79291734", "rot": 26, "bit": 0, "mask": 8}, - {"i": 136, "op": "sub", "dst": 1, "src": 7, "src2": 0, "imm": "0x225b03e4", "imm2": "0x7183e193", "rot": 16, "bit": 6, "mask": 2}, - {"i": 137, "op": "mad", "dst": 2, "src": 6, "src2": 2, "imm": "0x6f620d51", "imm2": "0x4e814e19", "rot": 6, "bit": 6, "mask": 2}, - {"i": 138, "op": "mulhi", "dst": 0, "src": 5, "src2": 6, "imm": "0x919d6bf3", "imm2": "0xc6240638", "rot": 17, "bit": 11, "mask": 16}, - {"i": 139, "op": "add", "dst": 2, "src": 7, "src2": 7, "imm": "0x218d4090", "imm2": "0xd44ff710", "rot": 1, "bit": 10, "mask": 16}, - {"i": 140, "op": "rotr", "dst": 1, "src": 4, "src2": 4, "imm": "0x4cbfa722", "imm2": "0x114e9564", "rot": 28, "bit": 9, "mask": 16}, - {"i": 141, "op": "shfl", "dst": 3, "src": 6, "src2": 2, "imm": "0xf702c6a1", "imm2": "0xf8f3c5d9", "rot": 7, "bit": 24, "mask": 1}, - {"i": 142, "op": "mad", "dst": 7, "src": 6, "src2": 2, "imm": "0xa2d285be", "imm2": "0x9cc94532", "rot": 15, "bit": 20, "mask": 8}, - {"i": 143, "op": "mul", "dst": 2, "src": 3, "src2": 7, "imm": "0x08b7ed80", "imm2": "0xa8abced4", "rot": 18, "bit": 10, "mask": 4}, - {"i": 144, "op": "add", "dst": 7, "src": 4, "src2": 2, "imm": "0xf4b1a8de", "imm2": "0xb98942fa", "rot": 18, "bit": 29, "mask": 8}, - {"i": 145, "op": "rotl", "dst": 7, "src": 6, "src2": 1, "imm": "0x64b6ba2d", "imm2": "0xf9e86793", "rot": 15, "bit": 24, "mask": 8}, - {"i": 146, "op": "xor", "dst": 7, "src": 5, "src2": 3, "imm": "0x41c42b5f", "imm2": "0x7c7e0e39", "rot": 8, "bit": 23, "mask": 2}, - {"i": 147, "op": "mad", "dst": 4, "src": 7, "src2": 4, "imm": "0x3caa807a", "imm2": "0x553a0cec", "rot": 11, "bit": 22, "mask": 8}, - {"i": 148, "op": "rotr", "dst": 6, "src": 5, "src2": 3, "imm": "0x3cb1289a", "imm2": "0x6b36f78a", "rot": 1, "bit": 9, "mask": 16}, - {"i": 149, "op": "mad", "dst": 1, "src": 2, "src2": 4, "imm": "0x95310ea8", "imm2": "0xc533aa6a", "rot": 16, "bit": 17, "mask": 1}, - {"i": 150, "op": "add", "dst": 1, "src": 7, "src2": 7, "imm": "0x2bef10f2", "imm2": "0x0d48ba42", "rot": 8, "bit": 17, "mask": 4}, - {"i": 151, "op": "xor", "dst": 5, "src": 4, "src2": 7, "imm": "0xda997ab2", "imm2": "0xae31d69e", "rot": 11, "bit": 31, "mask": 2}, - {"i": 152, "op": "add", "dst": 7, "src": 0, "src2": 6, "imm": "0xdc5cc080", "imm2": "0xc98dea9c", "rot": 16, "bit": 30, "mask": 2}, - {"i": 153, "op": "mul", "dst": 4, "src": 6, "src2": 7, "imm": "0xbfbf5f6c", "imm2": "0x61f611bb", "rot": 14, "bit": 22, "mask": 2}, - {"i": 154, "op": "mul", "dst": 0, "src": 2, "src2": 3, "imm": "0xa78008c3", "imm2": "0xbad5eeb1", "rot": 10, "bit": 28, "mask": 8}, - {"i": 155, "op": "xor", "dst": 6, "src": 5, "src2": 4, "imm": "0x1a73b866", "imm2": "0x1f5a62c9", "rot": 9, "bit": 27, "mask": 8}, - {"i": 156, "op": "rotr", "dst": 4, "src": 2, "src2": 0, "imm": "0x9c88500c", "imm2": "0xe25dccf9", "rot": 11, "bit": 2, "mask": 16}, - {"i": 157, "op": "rotl", "dst": 1, "src": 4, "src2": 4, "imm": "0xb05e7669", "imm2": "0x9db704b1", "rot": 11, "bit": 28, "mask": 4}, - {"i": 158, "op": "add", "dst": 5, "src": 0, "src2": 6, "imm": "0x18197438", "imm2": "0x6c752dcb", "rot": 11, "bit": 11, "mask": 2}, - {"i": 159, "op": "mad", "dst": 4, "src": 1, "src2": 5, "imm": "0x4796a65e", "imm2": "0x00e08c7a", "rot": 8, "bit": 19, "mask": 4}, - {"i": 160, "op": "rotl", "dst": 4, "src": 7, "src2": 5, "imm": "0x08a03052", "imm2": "0x0204f0ba", "rot": 26, "bit": 5, "mask": 2}, - {"i": 161, "op": "mad", "dst": 3, "src": 0, "src2": 7, "imm": "0xa0603b0e", "imm2": "0x7eee83d5", "rot": 28, "bit": 22, "mask": 16}, - {"i": 162, "op": "rotr", "dst": 3, "src": 1, "src2": 6, "imm": "0x78a3c69d", "imm2": "0x684693a0", "rot": 21, "bit": 23, "mask": 4}, - {"i": 163, "op": "add", "dst": 4, "src": 0, "src2": 7, "imm": "0xf1c46574", "imm2": "0x8e481727", "rot": 9, "bit": 3, "mask": 4}, - {"i": 164, "op": "mulhi", "dst": 0, "src": 4, "src2": 3, "imm": "0x04644afa", "imm2": "0x64bda2b5", "rot": 26, "bit": 16, "mask": 16}, - {"i": 165, "op": "add", "dst": 2, "src": 6, "src2": 2, "imm": "0xac578137", "imm2": "0x550ab406", "rot": 13, "bit": 20, "mask": 16}, - {"i": 166, "op": "rotl", "dst": 0, "src": 4, "src2": 5, "imm": "0x7f564760", "imm2": "0xb9a8b4f8", "rot": 13, "bit": 29, "mask": 1}, - {"i": 167, "op": "add", "dst": 3, "src": 1, "src2": 0, "imm": "0xa33e6706", "imm2": "0xaebb5966", "rot": 12, "bit": 14, "mask": 16}, - {"i": 168, "op": "or", "dst": 3, "src": 5, "src2": 3, "imm": "0x65b2f3eb", "imm2": "0xb1d00d20", "rot": 12, "bit": 2, "mask": 8}, - {"i": 169, "op": "rotr", "dst": 6, "src": 2, "src2": 3, "imm": "0x7f21faf5", "imm2": "0xbbf0d3f9", "rot": 17, "bit": 13, "mask": 8}, - {"i": 170, "op": "xor", "dst": 4, "src": 6, "src2": 2, "imm": "0x162c7140", "imm2": "0x90d404ad", "rot": 26, "bit": 1, "mask": 4}, - {"i": 171, "op": "sub", "dst": 6, "src": 1, "src2": 7, "imm": "0x6ef9c76e", "imm2": "0xfb7ba272", "rot": 31, "bit": 25, "mask": 1}, - {"i": 172, "op": "rotl", "dst": 7, "src": 5, "src2": 4, "imm": "0xde04eb3b", "imm2": "0xd56caa00", "rot": 22, "bit": 21, "mask": 4}, - {"i": 173, "op": "rotl", "dst": 5, "src": 3, "src2": 5, "imm": "0xa9eac934", "imm2": "0x2c338e51", "rot": 15, "bit": 22, "mask": 2}, - {"i": 174, "op": "shfl", "dst": 7, "src": 0, "src2": 3, "imm": "0x700be4e3", "imm2": "0x4bcfc732", "rot": 19, "bit": 6, "mask": 8}, - {"i": 175, "op": "xor", "dst": 0, "src": 5, "src2": 1, "imm": "0x02ccdba9", "imm2": "0xd0915be0", "rot": 15, "bit": 2, "mask": 16}, - {"i": 176, "op": "rotl", "dst": 7, "src": 4, "src2": 1, "imm": "0x489c8165", "imm2": "0xf24b5a4f", "rot": 6, "bit": 22, "mask": 1}, - {"i": 177, "op": "sub", "dst": 7, "src": 0, "src2": 6, "imm": "0x23ad9693", "imm2": "0x9a8c2f7b", "rot": 24, "bit": 2, "mask": 2}, - {"i": 178, "op": "rotl", "dst": 3, "src": 0, "src2": 6, "imm": "0xafa72a42", "imm2": "0x371d74ee", "rot": 30, "bit": 16, "mask": 1}, - {"i": 179, "op": "mad", "dst": 7, "src": 6, "src2": 1, "imm": "0x18a2a3f3", "imm2": "0xb811b951", "rot": 2, "bit": 9, "mask": 2}, - {"i": 180, "op": "rotl", "dst": 6, "src": 4, "src2": 1, "imm": "0xe6c69c0e", "imm2": "0xfc46a951", "rot": 9, "bit": 31, "mask": 4}, - {"i": 181, "op": "xor", "dst": 2, "src": 4, "src2": 7, "imm": "0xbd1b89e4", "imm2": "0xdf4bce5c", "rot": 15, "bit": 19, "mask": 4}, - {"i": 182, "op": "xor", "dst": 2, "src": 7, "src2": 5, "imm": "0x3dd12aed", "imm2": "0xd0756a69", "rot": 13, "bit": 16, "mask": 4}, - {"i": 183, "op": "xor", "dst": 7, "src": 2, "src2": 3, "imm": "0x77ce69d6", "imm2": "0x2b39bdf2", "rot": 8, "bit": 22, "mask": 16}, - {"i": 184, "op": "add", "dst": 1, "src": 2, "src2": 4, "imm": "0xd94d55ac", "imm2": "0x5bb7550f", "rot": 31, "bit": 21, "mask": 1}, - {"i": 185, "op": "or", "dst": 3, "src": 5, "src2": 6, "imm": "0x7b1ce846", "imm2": "0xcc3b8509", "rot": 28, "bit": 9, "mask": 4}, - {"i": 186, "op": "mulhi", "dst": 6, "src": 3, "src2": 0, "imm": "0xa5e24690", "imm2": "0x2200ba81", "rot": 26, "bit": 10, "mask": 16}, - {"i": 187, "op": "or", "dst": 4, "src": 0, "src2": 3, "imm": "0xf6efe759", "imm2": "0xae1f7118", "rot": 19, "bit": 20, "mask": 4}, - {"i": 188, "op": "shfl", "dst": 7, "src": 1, "src2": 5, "imm": "0xdb318b45", "imm2": "0xcec459c9", "rot": 20, "bit": 11, "mask": 2}, - {"i": 189, "op": "add", "dst": 6, "src": 5, "src2": 1, "imm": "0x89e747fe", "imm2": "0x2a354e2d", "rot": 22, "bit": 6, "mask": 1}, - {"i": 190, "op": "xor", "dst": 1, "src": 4, "src2": 2, "imm": "0xc379e617", "imm2": "0x75e9d63b", "rot": 23, "bit": 3, "mask": 2}, - {"i": 191, "op": "mulhi", "dst": 7, "src": 4, "src2": 3, "imm": "0x5a710287", "imm2": "0x7fe4ead6", "rot": 10, "bit": 25, "mask": 4}, - {"i": 192, "op": "rotl", "dst": 2, "src": 1, "src2": 2, "imm": "0x4d5e59a3", "imm2": "0x1e4fef28", "rot": 26, "bit": 0, "mask": 1}, - {"i": 193, "op": "rotl", "dst": 5, "src": 3, "src2": 7, "imm": "0x7444c47d", "imm2": "0xdad5f8be", "rot": 8, "bit": 30, "mask": 2}, - {"i": 194, "op": "shfl", "dst": 4, "src": 5, "src2": 7, "imm": "0x6a225bbb", "imm2": "0xd6532cd7", "rot": 13, "bit": 17, "mask": 16}, - {"i": 195, "op": "xor", "dst": 4, "src": 5, "src2": 3, "imm": "0x68344b9a", "imm2": "0xcb46a38b", "rot": 1, "bit": 27, "mask": 16}, - {"i": 196, "op": "mul", "dst": 1, "src": 3, "src2": 4, "imm": "0xbebf7359", "imm2": "0x3f0890ba", "rot": 18, "bit": 12, "mask": 4}, - {"i": 197, "op": "add", "dst": 5, "src": 2, "src2": 3, "imm": "0xea03e8e7", "imm2": "0x10cfdc71", "rot": 31, "bit": 7, "mask": 8}, - {"i": 198, "op": "add", "dst": 7, "src": 5, "src2": 1, "imm": "0xa7ee0102", "imm2": "0x66e148d3", "rot": 12, "bit": 15, "mask": 1}, - {"i": 199, "op": "sub", "dst": 5, "src": 2, "src2": 4, "imm": "0xc215584e", "imm2": "0x55fce30f", "rot": 30, "bit": 9, "mask": 2}, - {"i": 200, "op": "shfl", "dst": 5, "src": 1, "src2": 1, "imm": "0x308f8358", "imm2": "0x489f1f93", "rot": 13, "bit": 13, "mask": 2}, - {"i": 201, "op": "shfl", "dst": 7, "src": 1, "src2": 5, "imm": "0x713f1957", "imm2": "0x2239f757", "rot": 15, "bit": 7, "mask": 8}, - {"i": 202, "op": "rotr", "dst": 4, "src": 5, "src2": 1, "imm": "0xb037b6e7", "imm2": "0x49a8b528", "rot": 27, "bit": 13, "mask": 16}, - {"i": 203, "op": "xor", "dst": 7, "src": 5, "src2": 1, "imm": "0x56110241", "imm2": "0xefc8386d", "rot": 22, "bit": 20, "mask": 1}, - {"i": 204, "op": "xor", "dst": 7, "src": 0, "src2": 0, "imm": "0x0827fcc7", "imm2": "0xf4f5dd07", "rot": 13, "bit": 7, "mask": 2}, - {"i": 205, "op": "add", "dst": 6, "src": 2, "src2": 0, "imm": "0x8379a4de", "imm2": "0x8558b619", "rot": 26, "bit": 10, "mask": 1}, - {"i": 206, "op": "rotl", "dst": 4, "src": 3, "src2": 3, "imm": "0x091ceef0", "imm2": "0x69b0f72f", "rot": 13, "bit": 7, "mask": 1}, - {"i": 207, "op": "mulhi", "dst": 1, "src": 3, "src2": 4, "imm": "0x758edac1", "imm2": "0x98dd2f21", "rot": 15, "bit": 9, "mask": 1}, - {"i": 208, "op": "mad", "dst": 1, "src": 4, "src2": 4, "imm": "0x78871a1a", "imm2": "0x5e14e1c8", "rot": 19, "bit": 6, "mask": 2}, - {"i": 209, "op": "shfl", "dst": 2, "src": 0, "src2": 2, "imm": "0xf7e0b9aa", "imm2": "0xaecfd347", "rot": 21, "bit": 9, "mask": 4}, - {"i": 210, "op": "add", "dst": 7, "src": 0, "src2": 7, "imm": "0xaa8cb14e", "imm2": "0xf4049c4c", "rot": 24, "bit": 11, "mask": 8}, - {"i": 211, "op": "shfl", "dst": 7, "src": 1, "src2": 6, "imm": "0xc230d919", "imm2": "0xf76d08fb", "rot": 21, "bit": 13, "mask": 16}, - {"i": 212, "op": "xor", "dst": 1, "src": 0, "src2": 2, "imm": "0x31a5af90", "imm2": "0x7eef58ee", "rot": 9, "bit": 12, "mask": 4}, - {"i": 213, "op": "rotl", "dst": 7, "src": 1, "src2": 6, "imm": "0x0db26138", "imm2": "0x8e3c31f9", "rot": 3, "bit": 26, "mask": 2}, - {"i": 214, "op": "rotr", "dst": 4, "src": 7, "src2": 3, "imm": "0x29558100", "imm2": "0xe4b13ad6", "rot": 29, "bit": 24, "mask": 8}, - {"i": 215, "op": "shfl", "dst": 3, "src": 2, "src2": 6, "imm": "0x1b48c3d0", "imm2": "0x5c674ff6", "rot": 5, "bit": 9, "mask": 16}, - {"i": 216, "op": "or", "dst": 5, "src": 1, "src2": 0, "imm": "0xef768632", "imm2": "0x6de9d10d", "rot": 10, "bit": 4, "mask": 4}, - {"i": 217, "op": "sub", "dst": 1, "src": 2, "src2": 5, "imm": "0x88ca7f5a", "imm2": "0x24718a36", "rot": 18, "bit": 31, "mask": 1}, - {"i": 218, "op": "sub", "dst": 6, "src": 5, "src2": 0, "imm": "0xc4a06728", "imm2": "0xdc2a4fd8", "rot": 9, "bit": 9, "mask": 2}, - {"i": 219, "op": "rotl", "dst": 6, "src": 5, "src2": 7, "imm": "0xb7489e47", "imm2": "0xf13795c5", "rot": 4, "bit": 3, "mask": 8}, - {"i": 220, "op": "mulhi", "dst": 2, "src": 0, "src2": 2, "imm": "0x873cd31b", "imm2": "0x3dfbc55d", "rot": 12, "bit": 23, "mask": 8}, - {"i": 221, "op": "or", "dst": 2, "src": 0, "src2": 3, "imm": "0xdc21f099", "imm2": "0xee06f01e", "rot": 2, "bit": 17, "mask": 8}, - {"i": 222, "op": "shfl", "dst": 5, "src": 2, "src2": 6, "imm": "0x36def499", "imm2": "0xa2849d59", "rot": 23, "bit": 4, "mask": 8}, - {"i": 223, "op": "mulhi", "dst": 5, "src": 6, "src2": 7, "imm": "0xfb95fbca", "imm2": "0xc1aac427", "rot": 14, "bit": 11, "mask": 2}, - {"i": 224, "op": "sub", "dst": 0, "src": 6, "src2": 7, "imm": "0x3d9d29c4", "imm2": "0x34d0dcc0", "rot": 17, "bit": 6, "mask": 4}, - {"i": 225, "op": "rotl", "dst": 7, "src": 0, "src2": 5, "imm": "0x93b01b8e", "imm2": "0xfe1d75ac", "rot": 23, "bit": 15, "mask": 1}, - {"i": 226, "op": "or", "dst": 4, "src": 2, "src2": 4, "imm": "0xfed76e8e", "imm2": "0x1c24ecd8", "rot": 2, "bit": 6, "mask": 16}, - {"i": 227, "op": "mul", "dst": 2, "src": 4, "src2": 1, "imm": "0x5795f5b0", "imm2": "0x0566ea2a", "rot": 6, "bit": 28, "mask": 4}, - {"i": 228, "op": "rotl", "dst": 3, "src": 0, "src2": 2, "imm": "0x8c8486de", "imm2": "0xd066aa8f", "rot": 12, "bit": 20, "mask": 1}, - {"i": 229, "op": "rotr", "dst": 0, "src": 4, "src2": 0, "imm": "0x23a3e882", "imm2": "0xaca23902", "rot": 5, "bit": 22, "mask": 16}, - {"i": 230, "op": "add", "dst": 0, "src": 6, "src2": 4, "imm": "0x1d176220", "imm2": "0x2a7fecb2", "rot": 20, "bit": 16, "mask": 2}, - {"i": 231, "op": "shfl", "dst": 1, "src": 2, "src2": 0, "imm": "0x91172787", "imm2": "0xc5d7af28", "rot": 3, "bit": 24, "mask": 4}, - {"i": 232, "op": "mul", "dst": 2, "src": 1, "src2": 2, "imm": "0x0be68835", "imm2": "0xde692bdb", "rot": 30, "bit": 17, "mask": 1}, - {"i": 233, "op": "mad", "dst": 7, "src": 0, "src2": 1, "imm": "0xf90d2db5", "imm2": "0x96c4c175", "rot": 28, "bit": 7, "mask": 4}, - {"i": 234, "op": "rotl", "dst": 5, "src": 2, "src2": 1, "imm": "0x16379736", "imm2": "0x6973b905", "rot": 22, "bit": 17, "mask": 4}, - {"i": 235, "op": "rotr", "dst": 4, "src": 6, "src2": 2, "imm": "0x84292a13", "imm2": "0x0f897740", "rot": 12, "bit": 6, "mask": 8}, - {"i": 236, "op": "mad", "dst": 0, "src": 5, "src2": 1, "imm": "0xeb7de837", "imm2": "0x64f0c302", "rot": 4, "bit": 19, "mask": 1}, - {"i": 237, "op": "xor", "dst": 6, "src": 4, "src2": 4, "imm": "0x6ab4b683", "imm2": "0x20f17adb", "rot": 1, "bit": 0, "mask": 16}, - {"i": 238, "op": "xor", "dst": 4, "src": 6, "src2": 1, "imm": "0x5836b35c", "imm2": "0x3293cc4a", "rot": 16, "bit": 22, "mask": 16}, - {"i": 239, "op": "rotl", "dst": 6, "src": 1, "src2": 6, "imm": "0x769d8bc0", "imm2": "0xdc86c9cc", "rot": 18, "bit": 27, "mask": 16}, - {"i": 240, "op": "add", "dst": 4, "src": 7, "src2": 1, "imm": "0x17dafb4d", "imm2": "0xadce39f3", "rot": 12, "bit": 25, "mask": 8}, - {"i": 241, "op": "sub", "dst": 0, "src": 7, "src2": 4, "imm": "0xd4690bda", "imm2": "0xdab9b27b", "rot": 30, "bit": 21, "mask": 1}, - {"i": 242, "op": "rotr", "dst": 1, "src": 6, "src2": 2, "imm": "0x670a2d0a", "imm2": "0x0612e33c", "rot": 31, "bit": 25, "mask": 2}, - {"i": 243, "op": "add", "dst": 3, "src": 0, "src2": 2, "imm": "0xf2f77d26", "imm2": "0x0e7033b6", "rot": 27, "bit": 29, "mask": 1}, - {"i": 244, "op": "mad", "dst": 2, "src": 7, "src2": 4, "imm": "0xa9eefc9d", "imm2": "0x16166c85", "rot": 18, "bit": 23, "mask": 16}, - {"i": 245, "op": "mad", "dst": 4, "src": 0, "src2": 6, "imm": "0xb26f6c0f", "imm2": "0x0630e821", "rot": 23, "bit": 30, "mask": 4}, - {"i": 246, "op": "mul", "dst": 5, "src": 7, "src2": 2, "imm": "0x269ce4bf", "imm2": "0x1ce28d32", "rot": 30, "bit": 15, "mask": 16}, - {"i": 247, "op": "add", "dst": 3, "src": 5, "src2": 0, "imm": "0xfed2da4e", "imm2": "0x7b2ff6b7", "rot": 25, "bit": 31, "mask": 2}, - {"i": 248, "op": "add", "dst": 0, "src": 7, "src2": 5, "imm": "0x03b2891c", "imm2": "0xb5fad7b1", "rot": 2, "bit": 0, "mask": 4}, - {"i": 249, "op": "mulhi", "dst": 5, "src": 4, "src2": 6, "imm": "0xa69a1e71", "imm2": "0x15f0c0ea", "rot": 4, "bit": 1, "mask": 4}, - {"i": 250, "op": "add", "dst": 5, "src": 2, "src2": 3, "imm": "0x042cd6e3", "imm2": "0xa7e71c2f", "rot": 24, "bit": 11, "mask": 8}, - {"i": 251, "op": "shfl", "dst": 6, "src": 1, "src2": 2, "imm": "0x89c6b683", "imm2": "0x10bbf661", "rot": 24, "bit": 5, "mask": 1}, - {"i": 252, "op": "xor", "dst": 6, "src": 1, "src2": 3, "imm": "0x265c66d6", "imm2": "0xd4a689ed", "rot": 6, "bit": 14, "mask": 8}, - {"i": 253, "op": "mul", "dst": 2, "src": 5, "src2": 5, "imm": "0x890b8201", "imm2": "0x97c36bf3", "rot": 17, "bit": 22, "mask": 4}, - {"i": 254, "op": "add", "dst": 0, "src": 6, "src2": 0, "imm": "0x784a302b", "imm2": "0xb83d78de", "rot": 27, "bit": 16, "mask": 4}, - {"i": 255, "op": "sub", "dst": 2, "src": 4, "src2": 0, "imm": "0x817adb38", "imm2": "0xf3a3534b", "rot": 7, "bit": 22, "mask": 4} - ]}, - "shadow_placement": "per_load", - "instructions": [ - {"i": 0, "op": "mad", "dst": 2, "src": 3, "src2": 4, "imm": "0xbf7b174d", "imm2": "0x337b762e", "rot": 17, "bit": 2, "mask": 2, "width": 1}, - {"i": 1, "op": "mad", "dst": 2, "src": 1, "src2": 1, "imm": "0xdd04a5da", "imm2": "0x42da7657", "rot": 15, "bit": 30, "mask": 16, "width": 1}, - {"i": 2, "op": "mad", "dst": 2, "src": 3, "src2": 2, "imm": "0x734003fa", "imm2": "0x5bb67700", "rot": 3, "bit": 20, "mask": 1, "width": 1}, - {"i": 3, "op": "xor", "dst": 3, "src": 5, "src2": 5, "imm": "0xc55a1b1c", "imm2": "0xa19720f3", "rot": 7, "bit": 8, "mask": 1, "width": 1}, - {"i": 4, "op": "load", "dst": 7, "src": 2, "src2": 5, "imm": "0xad572dd7", "imm2": "0x9ceb3ea7", "rot": 18, "bit": 30, "mask": 2, "width": 1}, - {"i": 5, "op": "load", "dst": 5, "src": 7, "src2": 3, "imm": "0x769a53be", "imm2": "0x80f9067e", "rot": 12, "bit": 22, "mask": 1, "width": 1}, - {"i": 6, "op": "shfl", "dst": 1, "src": 4, "src2": 0, "imm": "0xd3613d88", "imm2": "0x262fb219", "rot": 10, "bit": 30, "mask": 8, "width": 1}, - {"i": 7, "op": "shfl", "dst": 7, "src": 3, "src2": 4, "imm": "0xce38e42f", "imm2": "0xb868b818", "rot": 11, "bit": 8, "mask": 8, "width": 1}, - {"i": 8, "op": "mulhi", "dst": 1, "src": 5, "src2": 2, "imm": "0xa5eebca5", "imm2": "0x5703a72b", "rot": 13, "bit": 13, "mask": 16, "width": 1}, - {"i": 9, "op": "rotr", "dst": 6, "src": 3, "src2": 4, "imm": "0x17a5a9c7", "imm2": "0xdcfb93a1", "rot": 20, "bit": 27, "mask": 2, "width": 1}, - {"i": 10, "op": "or", "dst": 3, "src": 4, "src2": 1, "imm": "0xccb7d785", "imm2": "0xc335364c", "rot": 14, "bit": 12, "mask": 4, "width": 1}, - {"i": 11, "op": "load", "dst": 4, "src": 3, "src2": 2, "imm": "0x88cb9af3", "imm2": "0x4e7dc10d", "rot": 24, "bit": 17, "mask": 4, "width": 1}, - {"i": 12, "op": "mulhi", "dst": 0, "src": 4, "src2": 4, "imm": "0x45374321", "imm2": "0x3cd91989", "rot": 11, "bit": 4, "mask": 2, "width": 1}, - {"i": 13, "op": "add", "dst": 5, "src": 1, "src2": 2, "imm": "0xc7934706", "imm2": "0xd3177981", "rot": 16, "bit": 30, "mask": 2, "width": 1}, - {"i": 14, "op": "load", "dst": 0, "src": 4, "src2": 3, "imm": "0xfd7f56bb", "imm2": "0x65e14f52", "rot": 13, "bit": 22, "mask": 2, "width": 1}, - {"i": 15, "op": "sub", "dst": 2, "src": 4, "src2": 4, "imm": "0x35a80b49", "imm2": "0x060f2d13", "rot": 16, "bit": 20, "mask": 16, "width": 1}, - {"i": 16, "op": "load", "dst": 2, "src": 0, "src2": 2, "imm": "0xae0a32c2", "imm2": "0x4c2a4cfe", "rot": 8, "bit": 31, "mask": 16, "width": 1}, - {"i": 17, "op": "load", "dst": 7, "src": 2, "src2": 6, "imm": "0x82a84cc3", "imm2": "0x21a38d68", "rot": 15, "bit": 21, "mask": 2, "width": 1}, - {"i": 18, "op": "shfl", "dst": 7, "src": 3, "src2": 3, "imm": "0xa3818806", "imm2": "0x8f66b5c8", "rot": 14, "bit": 6, "mask": 4, "width": 1}, - {"i": 19, "op": "mul", "dst": 5, "src": 0, "src2": 1, "imm": "0xa00de107", "imm2": "0x77bfcaa5", "rot": 3, "bit": 10, "mask": 2, "width": 1}, - {"i": 20, "op": "shfl", "dst": 3, "src": 4, "src2": 7, "imm": "0x1d2b8cab", "imm2": "0x80b4f9a2", "rot": 14, "bit": 25, "mask": 2, "width": 1}, - {"i": 21, "op": "shfl", "dst": 2, "src": 4, "src2": 5, "imm": "0x3ac915d2", "imm2": "0x5fba7bc2", "rot": 16, "bit": 1, "mask": 16, "width": 1}, - {"i": 22, "op": "mulhi", "dst": 6, "src": 2, "src2": 0, "imm": "0xdc3ec8fd", "imm2": "0x599e2fa3", "rot": 22, "bit": 3, "mask": 2, "width": 1}, - {"i": 23, "op": "load", "dst": 6, "src": 1, "src2": 5, "imm": "0x2a6b16d5", "imm2": "0xd73e396f", "rot": 28, "bit": 29, "mask": 2, "width": 1}, - {"i": 24, "op": "mul", "dst": 5, "src": 0, "src2": 7, "imm": "0x376d0223", "imm2": "0xe1c2169a", "rot": 4, "bit": 16, "mask": 16, "width": 1}, - {"i": 25, "op": "rotl", "dst": 5, "src": 7, "src2": 3, "imm": "0x78ad8c60", "imm2": "0x6f5b77d5", "rot": 19, "bit": 11, "mask": 16, "width": 1}, - {"i": 26, "op": "shfl", "dst": 7, "src": 6, "src2": 5, "imm": "0x93915b9f", "imm2": "0x1e61fb6b", "rot": 28, "bit": 23, "mask": 2, "width": 1}, - {"i": 27, "op": "xor", "dst": 0, "src": 5, "src2": 7, "imm": "0x6378fe15", "imm2": "0x66c78f42", "rot": 12, "bit": 31, "mask": 8, "width": 1}, - {"i": 28, "op": "xor", "dst": 0, "src": 4, "src2": 7, "imm": "0x20a57fda", "imm2": "0x088c848e", "rot": 16, "bit": 13, "mask": 4, "width": 1}, - {"i": 29, "op": "sub", "dst": 3, "src": 0, "src2": 2, "imm": "0x49d95fd5", "imm2": "0x1a5f946a", "rot": 6, "bit": 12, "mask": 1, "width": 1}, - {"i": 30, "op": "mul", "dst": 5, "src": 1, "src2": 7, "imm": "0x0a816217", "imm2": "0x405c4f73", "rot": 13, "bit": 27, "mask": 4, "width": 1}, - {"i": 31, "op": "load", "dst": 7, "src": 2, "src2": 2, "imm": "0x09ed045e", "imm2": "0xd69c4715", "rot": 5, "bit": 9, "mask": 2, "width": 1}, - {"i": 32, "op": "load", "dst": 1, "src": 0, "src2": 6, "imm": "0xeb79ea49", "imm2": "0xcc587f5a", "rot": 6, "bit": 8, "mask": 16, "width": 1}, - {"i": 33, "op": "xor", "dst": 5, "src": 6, "src2": 1, "imm": "0x3027401e", "imm2": "0x5f20c27e", "rot": 18, "bit": 9, "mask": 2, "width": 1}, - {"i": 34, "op": "load", "dst": 5, "src": 1, "src2": 3, "imm": "0x0e1cab07", "imm2": "0x09356c5b", "rot": 19, "bit": 31, "mask": 1, "width": 1}, - {"i": 35, "op": "mulhi", "dst": 0, "src": 5, "src2": 2, "imm": "0x90e31357", "imm2": "0xabd32484", "rot": 26, "bit": 5, "mask": 8, "width": 1}, - {"i": 36, "op": "shfl", "dst": 5, "src": 2, "src2": 5, "imm": "0xee9a955f", "imm2": "0x31b3faed", "rot": 8, "bit": 24, "mask": 4, "width": 1}, - {"i": 37, "op": "load", "dst": 7, "src": 0, "src2": 7, "imm": "0x3ba2f832", "imm2": "0x1160dcd3", "rot": 4, "bit": 29, "mask": 1, "width": 1}, - {"i": 38, "op": "add", "dst": 3, "src": 1, "src2": 7, "imm": "0x75ba2fad", "imm2": "0x230c005c", "rot": 4, "bit": 27, "mask": 1, "width": 1}, - {"i": 39, "op": "shfl", "dst": 1, "src": 5, "src2": 3, "imm": "0xdbf37e75", "imm2": "0xb5ac1969", "rot": 30, "bit": 13, "mask": 4, "width": 1}, - {"i": 40, "op": "xor", "dst": 2, "src": 5, "src2": 5, "imm": "0x47f136c5", "imm2": "0x06ce9153", "rot": 19, "bit": 10, "mask": 2, "width": 1}, - {"i": 41, "op": "mad", "dst": 3, "src": 6, "src2": 3, "imm": "0xce13eff8", "imm2": "0x04cc1d55", "rot": 3, "bit": 1, "mask": 4, "width": 1}, - {"i": 42, "op": "sub", "dst": 6, "src": 7, "src2": 1, "imm": "0x6a65ab71", "imm2": "0x8fbc1bcd", "rot": 4, "bit": 1, "mask": 8, "width": 1}, - {"i": 43, "op": "xor", "dst": 7, "src": 0, "src2": 7, "imm": "0xdaeb4928", "imm2": "0xc0423027", "rot": 24, "bit": 11, "mask": 8, "width": 1}, - {"i": 44, "op": "load", "dst": 1, "src": 7, "src2": 7, "imm": "0x778f01c9", "imm2": "0x28cedcea", "rot": 12, "bit": 4, "mask": 16, "width": 1}, - {"i": 45, "op": "mul", "dst": 2, "src": 3, "src2": 2, "imm": "0xf4264f1b", "imm2": "0x0f627d56", "rot": 5, "bit": 28, "mask": 8, "width": 1}, - {"i": 46, "op": "mulhi", "dst": 1, "src": 5, "src2": 1, "imm": "0xffb2147a", "imm2": "0xccde9b05", "rot": 13, "bit": 9, "mask": 2, "width": 1}, - {"i": 47, "op": "sub", "dst": 4, "src": 3, "src2": 5, "imm": "0x73b36234", "imm2": "0x3f5d5997", "rot": 7, "bit": 18, "mask": 2, "width": 1}, - {"i": 48, "op": "rotr", "dst": 2, "src": 6, "src2": 3, "imm": "0x3a4d9aa9", "imm2": "0x212bec7b", "rot": 4, "bit": 29, "mask": 16, "width": 1}, - {"i": 49, "op": "load", "dst": 3, "src": 5, "src2": 2, "imm": "0x626f11df", "imm2": "0x56cd5bfd", "rot": 7, "bit": 1, "mask": 1, "width": 1}, - {"i": 50, "op": "add", "dst": 1, "src": 5, "src2": 6, "imm": "0x81b8bc2c", "imm2": "0x1907970c", "rot": 28, "bit": 7, "mask": 4, "width": 1}, - {"i": 51, "op": "mul", "dst": 0, "src": 2, "src2": 2, "imm": "0xa8848b30", "imm2": "0xef6ac348", "rot": 9, "bit": 15, "mask": 8, "width": 1}, - {"i": 52, "op": "add", "dst": 0, "src": 2, "src2": 0, "imm": "0x4f92b968", "imm2": "0x699fd448", "rot": 22, "bit": 6, "mask": 4, "width": 1}, - {"i": 53, "op": "add", "dst": 1, "src": 0, "src2": 2, "imm": "0x2bb965af", "imm2": "0x77b1520d", "rot": 2, "bit": 12, "mask": 8, "width": 1}, - {"i": 54, "op": "rotl", "dst": 7, "src": 1, "src2": 0, "imm": "0x553e678b", "imm2": "0x3cc8eae0", "rot": 14, "bit": 20, "mask": 2, "width": 1}, - {"i": 55, "op": "add", "dst": 3, "src": 7, "src2": 2, "imm": "0x7b0fe07a", "imm2": "0xa54c55a0", "rot": 10, "bit": 1, "mask": 1, "width": 1}, - {"i": 56, "op": "load", "dst": 6, "src": 7, "src2": 4, "imm": "0x01eba9aa", "imm2": "0x2758c0f7", "rot": 14, "bit": 15, "mask": 4, "width": 1}, - {"i": 57, "op": "rotr", "dst": 1, "src": 5, "src2": 2, "imm": "0x1f5267b3", "imm2": "0x236f5a27", "rot": 2, "bit": 31, "mask": 16, "width": 1}, - {"i": 58, "op": "load", "dst": 5, "src": 4, "src2": 3, "imm": "0xa9954a9b", "imm2": "0x6a54d4e8", "rot": 11, "bit": 10, "mask": 16, "width": 1}, - {"i": 59, "op": "load", "dst": 6, "src": 2, "src2": 4, "imm": "0x9923ff88", "imm2": "0x9357254e", "rot": 16, "bit": 1, "mask": 16, "width": 1}, - {"i": 60, "op": "mad", "dst": 3, "src": 5, "src2": 0, "imm": "0xc0cc51a6", "imm2": "0x3fd7701b", "rot": 20, "bit": 1, "mask": 4, "width": 1}, - {"i": 61, "op": "add", "dst": 5, "src": 7, "src2": 7, "imm": "0xaf9dd72d", "imm2": "0xad7493e7", "rot": 7, "bit": 31, "mask": 16, "width": 1}, - {"i": 62, "op": "add", "dst": 4, "src": 6, "src2": 2, "imm": "0x89841d87", "imm2": "0x1e07c3d9", "rot": 6, "bit": 27, "mask": 1, "width": 1}, - {"i": 63, "op": "rotl", "dst": 5, "src": 4, "src2": 2, "imm": "0xae210f8d", "imm2": "0x8e499ba4", "rot": 19, "bit": 9, "mask": 1, "width": 1} - ] -} diff --git a/proto-cuda/packs-ca4/mx8_shl256x27/program.metal b/proto-cuda/packs-ca4/mx8_shl256x27/program.metal deleted file mode 100644 index 824172942..000000000 --- a/proto-cuda/packs-ca4/mx8_shl256x27/program.metal +++ /dev/null @@ -1,413 +0,0 @@ -#include -using namespace metal; - -#define MASK 0x0fffffffu -constant uint SEEDW[8] = { 0x67a9a7beu, 0x1a155b25u, 0xfddfb732u, 0x4b5af2e8u, 0xc55caf33u, 0xa27c13b7u, 0x06628a48u, 0x03852469u }; - -inline uint splitmix32(uint x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -inline uint rotl_imm(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 -inline uint rotr_var(uint x, uint n) { n &= 31u; return (x >> n) | (x << ((32u - n) & 31u)); } -inline uint ds_elem(uint i, uint d0, uint d1) { - uint x = i ^ d0; - x *= 0x9E3779B1u; x ^= x >> 15; - x += d1; - x *= 0x85EBCA77u; x ^= x >> 13; - x *= 0xC2B2AE3Du; x ^= x >> 16; - return x; -} - -kernel void igneum_hash(device const uint* dataset [[buffer(0)]], - device ulong* out [[buffer(1)]], - constant uint& baseNonce [[buffer(2)]], - uint gid [[thread_position_in_grid]]) { - uint nonce = baseNonce + gid; - uint r0, r1, r2, r3, r4, r5, r6, r7; - { uint x = nonce ^ SEEDW[0]; x += 0x9e3779b9u * 1u; x = splitmix32(x); r0 = x ^ SEEDW[1]; } - { uint x = nonce ^ SEEDW[1]; x += 0x9e3779b9u * 2u; x = splitmix32(x); r1 = x ^ SEEDW[2]; } - { uint x = nonce ^ SEEDW[2]; x += 0x9e3779b9u * 3u; x = splitmix32(x); r2 = x ^ SEEDW[3]; } - { uint x = nonce ^ SEEDW[3]; x += 0x9e3779b9u * 4u; x = splitmix32(x); r3 = x ^ SEEDW[4]; } - { uint x = nonce ^ SEEDW[4]; x += 0x9e3779b9u * 5u; x = splitmix32(x); r4 = x ^ SEEDW[5]; } - { uint x = nonce ^ SEEDW[5]; x += 0x9e3779b9u * 6u; x = splitmix32(x); r5 = x ^ SEEDW[6]; } - { uint x = nonce ^ SEEDW[6]; x += 0x9e3779b9u * 7u; x = splitmix32(x); r6 = x ^ SEEDW[7]; } - { uint x = nonce ^ SEEDW[7]; x += 0x9e3779b9u * 8u; x = splitmix32(x); r7 = x ^ SEEDW[0]; } - - for (uint it = 0u; it < 8u; ++it) { - uint sel = r0; - r2 = r3 * r4 + r2; // 0 - r2 = r1 * r1 + r2; // 1 - r2 = r3 * r2 + r2; // 2 - r3 = r3 ^ r5; // 3 - r7 = r7 ^ dataset[r2 & MASK]; // 4 - // per-load shadow sub-block 0 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 0 - for (uint sh0 = 0u; sh0 < 27u; ++sh0) { - r5 = r5 + r2 + select(0x92199f99u, 0x8bc12da9u, ((sel >> 18u) & 1u) != 0u); // s0 add - r0 = r0 + r7 + select(0x8d72d3adu, 0x63079e5au, ((sel >> 26u) & 1u) != 0u); // s1 add - r6 = r6 ^ simd_shuffle_xor(r3, (ushort)2); // s2 shfl - r4 = r4 - r2; // s3 sub - r7 = r7 + r0 + select(0xb21b4babu, 0x5d4c7a60u, ((sel >> 31u) & 1u) != 0u); // s4 add - r0 = rotl_imm(r0, 11u); // s5 rotl - r0 = r0 + r1 + select(0x2fe0e98bu, 0xc88e2942u, ((sel >> 16u) & 1u) != 0u); // s6 add - r7 = r7 ^ r1; // s7 xor - r1 = r6 * r5 + r1; // s8 mad - r6 = r6 ^ simd_shuffle_xor(r1, (ushort)4); // s9 shfl - r1 = r2 * r2 + r1; // s10 mad - r5 = r0 * r3 + r5; // s11 mad - r2 = r2 ^ simd_shuffle_xor(r6, (ushort)4); // s12 shfl - r1 = r1 + r5 + select(0xbdf8f9a5u, 0xbc48c63eu, ((sel >> 25u) & 1u) != 0u); // s13 add - r1 = rotl_imm(r1, 29u); // s14 rotl - r1 = r1 - r4; // s15 sub - } - r5 = r5 ^ dataset[r7 & MASK]; // 5 - // per-load shadow sub-block 1 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 1 - for (uint sh1 = 0u; sh1 < 27u; ++sh1) { - r7 = r7 | r1; // s16 or - r2 = r2 ^ simd_shuffle_xor(r4, (ushort)8); // s17 shfl - r7 = r7 ^ r4; // s18 xor - r6 = r6 * r1; // s19 mul - r5 = r6 * r0 + r5; // s20 mad - r3 = r3 - r1; // s21 sub - r6 = r6 * r0; // s22 mul - r2 = r2 + r0 + select(0x45c37cecu, 0x96e8f127u, ((sel >> 1u) & 1u) != 0u); // s23 add - r6 = r6 - r4; // s24 sub - r7 = r3 * r4 + r7; // s25 mad - r3 = rotl_imm(r3, 9u); // s26 rotl - r2 = r2 - r1; // s27 sub - r6 = mulhi(r6, r0); // s28 mulhi - r2 = r2 ^ simd_shuffle_xor(r4, (ushort)2); // s29 shfl - r1 = r1 ^ simd_shuffle_xor(r6, (ushort)1); // s30 shfl - r1 = r1 + r6 + select(0x37985632u, 0xb1cdb2abu, ((sel >> 1u) & 1u) != 0u); // s31 add - } - r1 = r1 ^ simd_shuffle_xor(r4, (ushort)8); // 6 - r7 = r7 ^ simd_shuffle_xor(r3, (ushort)8); // 7 - r1 = mulhi(r1, r5); // 8 - r6 = rotr_var(r6, r3); // 9 - r3 = r3 | r4; // 10 - r4 = r4 ^ dataset[r3 & MASK]; // 11 - // per-load shadow sub-block 2 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 2 - for (uint sh2 = 0u; sh2 < 27u; ++sh2) { - r0 = rotr_var(r0, r1); // s32 rotr - r3 = r3 ^ simd_shuffle_xor(r4, (ushort)2); // s33 shfl - r0 = r0 ^ simd_shuffle_xor(r3, (ushort)4); // s34 shfl - r7 = r7 * r0; // s35 mul - r3 = r3 + r1 + select(0x804e777eu, 0x856e0180u, ((sel >> 0u) & 1u) != 0u); // s36 add - r1 = r1 ^ r6; // s37 xor - r2 = r2 * r7; // s38 mul - r6 = r6 + r5 + select(0x0b06c8a7u, 0xe9bd0cc4u, ((sel >> 2u) & 1u) != 0u); // s39 add - r5 = r5 * r7; // s40 mul - r2 = mulhi(r2, r3); // s41 mulhi - r2 = r2 ^ r0; // s42 xor - r0 = rotl_imm(r0, 4u); // s43 rotl - r4 = mulhi(r4, r3); // s44 mulhi - r6 = r6 + r7 + select(0xee822e17u, 0xcdcb63f6u, ((sel >> 12u) & 1u) != 0u); // s45 add - r3 = r2 * r0 + r3; // s46 mad - r4 = r4 ^ simd_shuffle_xor(r3, (ushort)8); // s47 shfl - } - r0 = mulhi(r0, r4); // 12 - r5 = r5 + r1 + select(0xc7934706u, 0xd3177981u, ((sel >> 30u) & 1u) != 0u); // 13 - r0 = r0 ^ dataset[r4 & MASK]; // 14 - // per-load shadow sub-block 3 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 3 - for (uint sh3 = 0u; sh3 < 27u; ++sh3) { - r6 = mulhi(r6, r5); // s48 mulhi - r0 = r0 - r2; // s49 sub - r3 = r3 - r5; // s50 sub - r1 = rotr_var(r1, r4); // s51 rotr - r6 = r7 * r7 + r6; // s52 mad - r5 = r5 ^ r3; // s53 xor - r1 = r1 - r0; // s54 sub - r5 = r5 - r6; // s55 sub - r3 = r3 + r2 + select(0x3dfad1b6u, 0xd4758987u, ((sel >> 10u) & 1u) != 0u); // s56 add - r4 = r4 + r1 + select(0xfca75bc2u, 0x0602d6beu, ((sel >> 0u) & 1u) != 0u); // s57 add - r5 = mulhi(r5, r6); // s58 mulhi - r2 = r2 ^ simd_shuffle_xor(r1, (ushort)2); // s59 shfl - r2 = rotl_imm(r2, 9u); // s60 rotl - r4 = r4 | r6; // s61 or - r6 = rotr_var(r6, r4); // s62 rotr - r2 = r2 * r4; // s63 mul - } - r2 = r2 - r4; // 15 - r2 = r2 ^ dataset[r0 & MASK]; // 16 - // per-load shadow sub-block 4 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 4 - for (uint sh4 = 0u; sh4 < 27u; ++sh4) { - r0 = r0 ^ r5; // s64 xor - r2 = r2 ^ r7; // s65 xor - r2 = r2 + r1 + select(0xd71c02ffu, 0x8596687au, ((sel >> 6u) & 1u) != 0u); // s66 add - r1 = rotl_imm(r1, 21u); // s67 rotl - r3 = r3 * r2; // s68 mul - r7 = r7 ^ simd_shuffle_xor(r5, (ushort)2); // s69 shfl - r3 = r3 * r2; // s70 mul - r0 = r2 * r5 + r0; // s71 mad - r6 = r6 + r7 + select(0x3c6fe15du, 0x08ac5733u, ((sel >> 0u) & 1u) != 0u); // s72 add - r3 = r3 ^ simd_shuffle_xor(r2, (ushort)8); // s73 shfl - r3 = r3 * r6; // s74 mul - r6 = r6 ^ r7; // s75 xor - r3 = r3 | r0; // s76 or - r2 = r2 + r4 + select(0x295d5faeu, 0xc100b495u, ((sel >> 1u) & 1u) != 0u); // s77 add - r3 = mulhi(r3, r7); // s78 mulhi - r4 = r4 | r1; // s79 or - } - r7 = r7 ^ dataset[r2 & MASK]; // 17 - // per-load shadow sub-block 5 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 5 - for (uint sh5 = 0u; sh5 < 27u; ++sh5) { - r4 = rotr_var(r4, r3); // s80 rotr - r4 = r4 + r3 + select(0x274a9221u, 0x5cc59530u, ((sel >> 15u) & 1u) != 0u); // s81 add - r7 = r7 + r3 + select(0xc8651f8eu, 0x141479ecu, ((sel >> 5u) & 1u) != 0u); // s82 add - r3 = rotr_var(r3, r6); // s83 rotr - r2 = r4 * r7 + r2; // s84 mad - r2 = r2 ^ simd_shuffle_xor(r5, (ushort)16); // s85 shfl - r3 = r3 ^ r2; // s86 xor - r5 = r5 ^ simd_shuffle_xor(r7, (ushort)2); // s87 shfl - r0 = r0 ^ r4; // s88 xor - r3 = r3 + r2 + select(0x53f915c2u, 0x883c0c92u, ((sel >> 18u) & 1u) != 0u); // s89 add - r7 = rotr_var(r7, r5); // s90 rotr - r3 = r3 + r1 + select(0xe2be00a5u, 0x3cc5bd20u, ((sel >> 31u) & 1u) != 0u); // s91 add - r7 = r7 ^ simd_shuffle_xor(r2, (ushort)1); // s92 shfl - r6 = rotr_var(r6, r2); // s93 rotr - r7 = rotl_imm(r7, 14u); // s94 rotl - r4 = r5 * r5 + r4; // s95 mad - } - r7 = r7 ^ simd_shuffle_xor(r3, (ushort)4); // 18 - r5 = r5 * r0; // 19 - r3 = r3 ^ simd_shuffle_xor(r4, (ushort)2); // 20 - r2 = r2 ^ simd_shuffle_xor(r4, (ushort)16); // 21 - r6 = mulhi(r6, r2); // 22 - r6 = r6 ^ dataset[r1 & MASK]; // 23 - // per-load shadow sub-block 6 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 6 - for (uint sh6 = 0u; sh6 < 27u; ++sh6) { - r2 = r4 * r5 + r2; // s96 mad - r1 = r1 ^ r0; // s97 xor - r5 = r5 * r4; // s98 mul - r2 = r2 - r0; // s99 sub - r7 = rotl_imm(r7, 30u); // s100 rotl - r5 = r5 + r6 + select(0x423fd9c9u, 0xbfd646cbu, ((sel >> 17u) & 1u) != 0u); // s101 add - r7 = r7 + r6 + select(0x8d3c011du, 0x19b74a43u, ((sel >> 11u) & 1u) != 0u); // s102 add - r5 = rotl_imm(r5, 6u); // s103 rotl - r0 = r0 * r4; // s104 mul - r0 = r0 | r5; // s105 or - r0 = r0 + r1 + select(0x8ce14721u, 0x7dcb7e18u, ((sel >> 15u) & 1u) != 0u); // s106 add - r0 = rotl_imm(r0, 15u); // s107 rotl - r4 = r4 - r2; // s108 sub - r2 = mulhi(r2, r0); // s109 mulhi - r1 = mulhi(r1, r0); // s110 mulhi - r0 = r0 + r2 + select(0x2d9fc6b9u, 0x3c179ad8u, ((sel >> 28u) & 1u) != 0u); // s111 add - } - r5 = r5 * r0; // 24 - r5 = rotl_imm(r5, 19u); // 25 - r7 = r7 ^ simd_shuffle_xor(r6, (ushort)2); // 26 - r0 = r0 ^ r5; // 27 - r0 = r0 ^ r4; // 28 - r3 = r3 - r0; // 29 - r5 = r5 * r1; // 30 - r7 = r7 ^ dataset[r2 & MASK]; // 31 - // per-load shadow sub-block 7 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 7 - for (uint sh7 = 0u; sh7 < 27u; ++sh7) { - r2 = mulhi(r2, r0); // s112 mulhi - r6 = r6 - r3; // s113 sub - r7 = r6 * r3 + r7; // s114 mad - r5 = rotr_var(r5, r1); // s115 rotr - r6 = rotr_var(r6, r4); // s116 rotr - r2 = r2 + r1 + select(0x502138e2u, 0x47359729u, ((sel >> 22u) & 1u) != 0u); // s117 add - r3 = r2 * r0 + r3; // s118 mad - r1 = r1 * r3; // s119 mul - r2 = r0 * r4 + r2; // s120 mad - r0 = r0 * r3; // s121 mul - r2 = mulhi(r2, r0); // s122 mulhi - r5 = r5 * r6; // s123 mul - r4 = r2 * r4 + r4; // s124 mad - r4 = r4 ^ simd_shuffle_xor(r2, (ushort)2); // s125 shfl - r2 = r2 ^ r0; // s126 xor - r1 = rotl_imm(r1, 7u); // s127 rotl - } - r1 = r1 ^ dataset[r0 & MASK]; // 32 - // per-load shadow sub-block 8 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 8 - for (uint sh8 = 0u; sh8 < 27u; ++sh8) { - r7 = r7 ^ simd_shuffle_xor(r2, (ushort)4); // s128 shfl - r6 = rotl_imm(r6, 17u); // s129 rotl - r2 = r2 + r5 + select(0x33dff776u, 0x6c57e4e7u, ((sel >> 15u) & 1u) != 0u); // s130 add - r0 = r0 | r3; // s131 or - r5 = rotr_var(r5, r0); // s132 rotr - r2 = r2 + r3 + select(0x5db25b34u, 0xe8212c0cu, ((sel >> 30u) & 1u) != 0u); // s133 add - r4 = r4 ^ r3; // s134 xor - r6 = r6 ^ r1; // s135 xor - r1 = r1 - r7; // s136 sub - r2 = r6 * r2 + r2; // s137 mad - r0 = mulhi(r0, r5); // s138 mulhi - r2 = r2 + r7 + select(0x218d4090u, 0xd44ff710u, ((sel >> 10u) & 1u) != 0u); // s139 add - r1 = rotr_var(r1, r4); // s140 rotr - r3 = r3 ^ simd_shuffle_xor(r6, (ushort)1); // s141 shfl - r7 = r6 * r2 + r7; // s142 mad - r2 = r2 * r3; // s143 mul - } - r5 = r5 ^ r6; // 33 - r5 = r5 ^ dataset[r1 & MASK]; // 34 - // per-load shadow sub-block 9 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 9 - for (uint sh9 = 0u; sh9 < 27u; ++sh9) { - r7 = r7 + r4 + select(0xf4b1a8deu, 0xb98942fau, ((sel >> 29u) & 1u) != 0u); // s144 add - r7 = rotl_imm(r7, 15u); // s145 rotl - r7 = r7 ^ r5; // s146 xor - r4 = r7 * r4 + r4; // s147 mad - r6 = rotr_var(r6, r5); // s148 rotr - r1 = r2 * r4 + r1; // s149 mad - r1 = r1 + r7 + select(0x2bef10f2u, 0x0d48ba42u, ((sel >> 17u) & 1u) != 0u); // s150 add - r5 = r5 ^ r4; // s151 xor - r7 = r7 + r0 + select(0xdc5cc080u, 0xc98dea9cu, ((sel >> 30u) & 1u) != 0u); // s152 add - r4 = r4 * r6; // s153 mul - r0 = r0 * r2; // s154 mul - r6 = r6 ^ r5; // s155 xor - r4 = rotr_var(r4, r2); // s156 rotr - r1 = rotl_imm(r1, 11u); // s157 rotl - r5 = r5 + r0 + select(0x18197438u, 0x6c752dcbu, ((sel >> 11u) & 1u) != 0u); // s158 add - r4 = r1 * r5 + r4; // s159 mad - } - r0 = mulhi(r0, r5); // 35 - r5 = r5 ^ simd_shuffle_xor(r2, (ushort)4); // 36 - r7 = r7 ^ dataset[r0 & MASK]; // 37 - // per-load shadow sub-block 10 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 10 - for (uint sh10 = 0u; sh10 < 27u; ++sh10) { - r4 = rotl_imm(r4, 26u); // s160 rotl - r3 = r0 * r7 + r3; // s161 mad - r3 = rotr_var(r3, r1); // s162 rotr - r4 = r4 + r0 + select(0xf1c46574u, 0x8e481727u, ((sel >> 3u) & 1u) != 0u); // s163 add - r0 = mulhi(r0, r4); // s164 mulhi - r2 = r2 + r6 + select(0xac578137u, 0x550ab406u, ((sel >> 20u) & 1u) != 0u); // s165 add - r0 = rotl_imm(r0, 13u); // s166 rotl - r3 = r3 + r1 + select(0xa33e6706u, 0xaebb5966u, ((sel >> 14u) & 1u) != 0u); // s167 add - r3 = r3 | r5; // s168 or - r6 = rotr_var(r6, r2); // s169 rotr - r4 = r4 ^ r6; // s170 xor - r6 = r6 - r1; // s171 sub - r7 = rotl_imm(r7, 22u); // s172 rotl - r5 = rotl_imm(r5, 15u); // s173 rotl - r7 = r7 ^ simd_shuffle_xor(r0, (ushort)8); // s174 shfl - r0 = r0 ^ r5; // s175 xor - } - r3 = r3 + r1 + select(0x75ba2fadu, 0x230c005cu, ((sel >> 27u) & 1u) != 0u); // 38 - r1 = r1 ^ simd_shuffle_xor(r5, (ushort)4); // 39 - r2 = r2 ^ r5; // 40 - r3 = r6 * r3 + r3; // 41 - r6 = r6 - r7; // 42 - r7 = r7 ^ r0; // 43 - r1 = r1 ^ dataset[r7 & MASK]; // 44 - // per-load shadow sub-block 11 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 11 - for (uint sh11 = 0u; sh11 < 27u; ++sh11) { - r7 = rotl_imm(r7, 6u); // s176 rotl - r7 = r7 - r0; // s177 sub - r3 = rotl_imm(r3, 30u); // s178 rotl - r7 = r6 * r1 + r7; // s179 mad - r6 = rotl_imm(r6, 9u); // s180 rotl - r2 = r2 ^ r4; // s181 xor - r2 = r2 ^ r7; // s182 xor - r7 = r7 ^ r2; // s183 xor - r1 = r1 + r2 + select(0xd94d55acu, 0x5bb7550fu, ((sel >> 21u) & 1u) != 0u); // s184 add - r3 = r3 | r5; // s185 or - r6 = mulhi(r6, r3); // s186 mulhi - r4 = r4 | r0; // s187 or - r7 = r7 ^ simd_shuffle_xor(r1, (ushort)2); // s188 shfl - r6 = r6 + r5 + select(0x89e747feu, 0x2a354e2du, ((sel >> 6u) & 1u) != 0u); // s189 add - r1 = r1 ^ r4; // s190 xor - r7 = mulhi(r7, r4); // s191 mulhi - } - r2 = r2 * r3; // 45 - r1 = mulhi(r1, r5); // 46 - r4 = r4 - r3; // 47 - r2 = rotr_var(r2, r6); // 48 - r3 = r3 ^ dataset[r5 & MASK]; // 49 - // per-load shadow sub-block 12 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 12 - for (uint sh12 = 0u; sh12 < 27u; ++sh12) { - r2 = rotl_imm(r2, 26u); // s192 rotl - r5 = rotl_imm(r5, 8u); // s193 rotl - r4 = r4 ^ simd_shuffle_xor(r5, (ushort)16); // s194 shfl - r4 = r4 ^ r5; // s195 xor - r1 = r1 * r3; // s196 mul - r5 = r5 + r2 + select(0xea03e8e7u, 0x10cfdc71u, ((sel >> 7u) & 1u) != 0u); // s197 add - r7 = r7 + r5 + select(0xa7ee0102u, 0x66e148d3u, ((sel >> 15u) & 1u) != 0u); // s198 add - r5 = r5 - r2; // s199 sub - r5 = r5 ^ simd_shuffle_xor(r1, (ushort)2); // s200 shfl - r7 = r7 ^ simd_shuffle_xor(r1, (ushort)8); // s201 shfl - r4 = rotr_var(r4, r5); // s202 rotr - r7 = r7 ^ r5; // s203 xor - r7 = r7 ^ r0; // s204 xor - r6 = r6 + r2 + select(0x8379a4deu, 0x8558b619u, ((sel >> 10u) & 1u) != 0u); // s205 add - r4 = rotl_imm(r4, 13u); // s206 rotl - r1 = mulhi(r1, r3); // s207 mulhi - } - r1 = r1 + r5 + select(0x81b8bc2cu, 0x1907970cu, ((sel >> 7u) & 1u) != 0u); // 50 - r0 = r0 * r2; // 51 - r0 = r0 + r2 + select(0x4f92b968u, 0x699fd448u, ((sel >> 6u) & 1u) != 0u); // 52 - r1 = r1 + r0 + select(0x2bb965afu, 0x77b1520du, ((sel >> 12u) & 1u) != 0u); // 53 - r7 = rotl_imm(r7, 14u); // 54 - r3 = r3 + r7 + select(0x7b0fe07au, 0xa54c55a0u, ((sel >> 1u) & 1u) != 0u); // 55 - r6 = r6 ^ dataset[r7 & MASK]; // 56 - // per-load shadow sub-block 13 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 13 - for (uint sh13 = 0u; sh13 < 27u; ++sh13) { - r1 = r4 * r4 + r1; // s208 mad - r2 = r2 ^ simd_shuffle_xor(r0, (ushort)4); // s209 shfl - r7 = r7 + r0 + select(0xaa8cb14eu, 0xf4049c4cu, ((sel >> 11u) & 1u) != 0u); // s210 add - r7 = r7 ^ simd_shuffle_xor(r1, (ushort)16); // s211 shfl - r1 = r1 ^ r0; // s212 xor - r7 = rotl_imm(r7, 3u); // s213 rotl - r4 = rotr_var(r4, r7); // s214 rotr - r3 = r3 ^ simd_shuffle_xor(r2, (ushort)16); // s215 shfl - r5 = r5 | r1; // s216 or - r1 = r1 - r2; // s217 sub - r6 = r6 - r5; // s218 sub - r6 = rotl_imm(r6, 4u); // s219 rotl - r2 = mulhi(r2, r0); // s220 mulhi - r2 = r2 | r0; // s221 or - r5 = r5 ^ simd_shuffle_xor(r2, (ushort)8); // s222 shfl - r5 = mulhi(r5, r6); // s223 mulhi - } - r1 = rotr_var(r1, r5); // 57 - r5 = r5 ^ dataset[r4 & MASK]; // 58 - // per-load shadow sub-block 14 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 14 - for (uint sh14 = 0u; sh14 < 27u; ++sh14) { - r0 = r0 - r6; // s224 sub - r7 = rotl_imm(r7, 23u); // s225 rotl - r4 = r4 | r2; // s226 or - r2 = r2 * r4; // s227 mul - r3 = rotl_imm(r3, 12u); // s228 rotl - r0 = rotr_var(r0, r4); // s229 rotr - r0 = r0 + r6 + select(0x1d176220u, 0x2a7fecb2u, ((sel >> 16u) & 1u) != 0u); // s230 add - r1 = r1 ^ simd_shuffle_xor(r2, (ushort)4); // s231 shfl - r2 = r2 * r1; // s232 mul - r7 = r0 * r1 + r7; // s233 mad - r5 = rotl_imm(r5, 22u); // s234 rotl - r4 = rotr_var(r4, r6); // s235 rotr - r0 = r5 * r1 + r0; // s236 mad - r6 = r6 ^ r4; // s237 xor - r4 = r4 ^ r6; // s238 xor - r6 = rotl_imm(r6, 18u); // s239 rotl - } - r6 = r6 ^ dataset[r2 & MASK]; // 59 - // per-load shadow sub-block 15 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 15 - for (uint sh15 = 0u; sh15 < 27u; ++sh15) { - r4 = r4 + r7 + select(0x17dafb4du, 0xadce39f3u, ((sel >> 25u) & 1u) != 0u); // s240 add - r0 = r0 - r7; // s241 sub - r1 = rotr_var(r1, r6); // s242 rotr - r3 = r3 + r0 + select(0xf2f77d26u, 0x0e7033b6u, ((sel >> 29u) & 1u) != 0u); // s243 add - r2 = r7 * r4 + r2; // s244 mad - r4 = r0 * r6 + r4; // s245 mad - r5 = r5 * r7; // s246 mul - r3 = r3 + r5 + select(0xfed2da4eu, 0x7b2ff6b7u, ((sel >> 31u) & 1u) != 0u); // s247 add - r0 = r0 + r7 + select(0x03b2891cu, 0xb5fad7b1u, ((sel >> 0u) & 1u) != 0u); // s248 add - r5 = mulhi(r5, r4); // s249 mulhi - r5 = r5 + r2 + select(0x042cd6e3u, 0xa7e71c2fu, ((sel >> 11u) & 1u) != 0u); // s250 add - r6 = r6 ^ simd_shuffle_xor(r1, (ushort)1); // s251 shfl - r6 = r6 ^ r1; // s252 xor - r2 = r2 * r5; // s253 mul - r0 = r0 + r6 + select(0x784a302bu, 0xb83d78deu, ((sel >> 16u) & 1u) != 0u); // s254 add - r2 = r2 - r4; // s255 sub - } - r3 = r5 * r0 + r3; // 60 - r5 = r5 + r7 + select(0xaf9dd72du, 0xad7493e7u, ((sel >> 31u) & 1u) != 0u); // 61 - r4 = r4 + r6 + select(0x89841d87u, 0x1e07c3d9u, ((sel >> 27u) & 1u) != 0u); // 62 - r5 = rotl_imm(r5, 19u); // 63 - } - uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((ulong)hi << 32) | (ulong)lo; -} diff --git a/proto-cuda/packs-ca4/mx8_shl256x27/program_bound.metal b/proto-cuda/packs-ca4/mx8_shl256x27/program_bound.metal deleted file mode 100644 index 6a79a2753..000000000 --- a/proto-cuda/packs-ca4/mx8_shl256x27/program_bound.metal +++ /dev/null @@ -1,415 +0,0 @@ -#include -using namespace metal; - -#define MASK 0x0fffffffu -constant uint SEEDW[8] = { 0x67a9a7beu, 0x1a155b25u, 0xfddfb732u, 0x4b5af2e8u, 0xc55caf33u, 0xa27c13b7u, 0x06628a48u, 0x03852469u }; - -inline uint splitmix32(uint x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -inline uint rotl_imm(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 -inline uint rotr_var(uint x, uint n) { n &= 31u; return (x >> n) | (x << ((32u - n) & 31u)); } -inline uint ds_elem(uint i, uint d0, uint d1) { - uint x = i ^ d0; - x *= 0x9E3779B1u; x ^= x >> 15; - x += d1; - x *= 0x85EBCA77u; x ^= x >> 13; - x *= 0xC2B2AE3Du; x ^= x >> 16; - return x; -} - -// Header-bound variant: the init words come from buffer 3 (bind.rs), not from SEEDW. -kernel void igneum_hash_bound(device const uint* dataset [[buffer(0)]], - device ulong* out [[buffer(1)]], - constant uint& baseNonce [[buffer(2)]], - constant uint* initw [[buffer(3)]], - uint gid [[thread_position_in_grid]]) { - uint nonce = baseNonce + gid; - uint r0, r1, r2, r3, r4, r5, r6, r7; - { uint x = nonce ^ initw[0]; x += 0x9e3779b9u * 1u; x = splitmix32(x); r0 = x ^ initw[1]; } - { uint x = nonce ^ initw[1]; x += 0x9e3779b9u * 2u; x = splitmix32(x); r1 = x ^ initw[2]; } - { uint x = nonce ^ initw[2]; x += 0x9e3779b9u * 3u; x = splitmix32(x); r2 = x ^ initw[3]; } - { uint x = nonce ^ initw[3]; x += 0x9e3779b9u * 4u; x = splitmix32(x); r3 = x ^ initw[4]; } - { uint x = nonce ^ initw[4]; x += 0x9e3779b9u * 5u; x = splitmix32(x); r4 = x ^ initw[5]; } - { uint x = nonce ^ initw[5]; x += 0x9e3779b9u * 6u; x = splitmix32(x); r5 = x ^ initw[6]; } - { uint x = nonce ^ initw[6]; x += 0x9e3779b9u * 7u; x = splitmix32(x); r6 = x ^ initw[7]; } - { uint x = nonce ^ initw[7]; x += 0x9e3779b9u * 8u; x = splitmix32(x); r7 = x ^ initw[0]; } - - for (uint it = 0u; it < 8u; ++it) { - uint sel = r0; - r2 = r3 * r4 + r2; // 0 - r2 = r1 * r1 + r2; // 1 - r2 = r3 * r2 + r2; // 2 - r3 = r3 ^ r5; // 3 - r7 = r7 ^ dataset[r2 & MASK]; // 4 - // per-load shadow sub-block 0 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 0 - for (uint sh0 = 0u; sh0 < 27u; ++sh0) { - r5 = r5 + r2 + select(0x92199f99u, 0x8bc12da9u, ((sel >> 18u) & 1u) != 0u); // s0 add - r0 = r0 + r7 + select(0x8d72d3adu, 0x63079e5au, ((sel >> 26u) & 1u) != 0u); // s1 add - r6 = r6 ^ simd_shuffle_xor(r3, (ushort)2); // s2 shfl - r4 = r4 - r2; // s3 sub - r7 = r7 + r0 + select(0xb21b4babu, 0x5d4c7a60u, ((sel >> 31u) & 1u) != 0u); // s4 add - r0 = rotl_imm(r0, 11u); // s5 rotl - r0 = r0 + r1 + select(0x2fe0e98bu, 0xc88e2942u, ((sel >> 16u) & 1u) != 0u); // s6 add - r7 = r7 ^ r1; // s7 xor - r1 = r6 * r5 + r1; // s8 mad - r6 = r6 ^ simd_shuffle_xor(r1, (ushort)4); // s9 shfl - r1 = r2 * r2 + r1; // s10 mad - r5 = r0 * r3 + r5; // s11 mad - r2 = r2 ^ simd_shuffle_xor(r6, (ushort)4); // s12 shfl - r1 = r1 + r5 + select(0xbdf8f9a5u, 0xbc48c63eu, ((sel >> 25u) & 1u) != 0u); // s13 add - r1 = rotl_imm(r1, 29u); // s14 rotl - r1 = r1 - r4; // s15 sub - } - r5 = r5 ^ dataset[r7 & MASK]; // 5 - // per-load shadow sub-block 1 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 1 - for (uint sh1 = 0u; sh1 < 27u; ++sh1) { - r7 = r7 | r1; // s16 or - r2 = r2 ^ simd_shuffle_xor(r4, (ushort)8); // s17 shfl - r7 = r7 ^ r4; // s18 xor - r6 = r6 * r1; // s19 mul - r5 = r6 * r0 + r5; // s20 mad - r3 = r3 - r1; // s21 sub - r6 = r6 * r0; // s22 mul - r2 = r2 + r0 + select(0x45c37cecu, 0x96e8f127u, ((sel >> 1u) & 1u) != 0u); // s23 add - r6 = r6 - r4; // s24 sub - r7 = r3 * r4 + r7; // s25 mad - r3 = rotl_imm(r3, 9u); // s26 rotl - r2 = r2 - r1; // s27 sub - r6 = mulhi(r6, r0); // s28 mulhi - r2 = r2 ^ simd_shuffle_xor(r4, (ushort)2); // s29 shfl - r1 = r1 ^ simd_shuffle_xor(r6, (ushort)1); // s30 shfl - r1 = r1 + r6 + select(0x37985632u, 0xb1cdb2abu, ((sel >> 1u) & 1u) != 0u); // s31 add - } - r1 = r1 ^ simd_shuffle_xor(r4, (ushort)8); // 6 - r7 = r7 ^ simd_shuffle_xor(r3, (ushort)8); // 7 - r1 = mulhi(r1, r5); // 8 - r6 = rotr_var(r6, r3); // 9 - r3 = r3 | r4; // 10 - r4 = r4 ^ dataset[r3 & MASK]; // 11 - // per-load shadow sub-block 2 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 2 - for (uint sh2 = 0u; sh2 < 27u; ++sh2) { - r0 = rotr_var(r0, r1); // s32 rotr - r3 = r3 ^ simd_shuffle_xor(r4, (ushort)2); // s33 shfl - r0 = r0 ^ simd_shuffle_xor(r3, (ushort)4); // s34 shfl - r7 = r7 * r0; // s35 mul - r3 = r3 + r1 + select(0x804e777eu, 0x856e0180u, ((sel >> 0u) & 1u) != 0u); // s36 add - r1 = r1 ^ r6; // s37 xor - r2 = r2 * r7; // s38 mul - r6 = r6 + r5 + select(0x0b06c8a7u, 0xe9bd0cc4u, ((sel >> 2u) & 1u) != 0u); // s39 add - r5 = r5 * r7; // s40 mul - r2 = mulhi(r2, r3); // s41 mulhi - r2 = r2 ^ r0; // s42 xor - r0 = rotl_imm(r0, 4u); // s43 rotl - r4 = mulhi(r4, r3); // s44 mulhi - r6 = r6 + r7 + select(0xee822e17u, 0xcdcb63f6u, ((sel >> 12u) & 1u) != 0u); // s45 add - r3 = r2 * r0 + r3; // s46 mad - r4 = r4 ^ simd_shuffle_xor(r3, (ushort)8); // s47 shfl - } - r0 = mulhi(r0, r4); // 12 - r5 = r5 + r1 + select(0xc7934706u, 0xd3177981u, ((sel >> 30u) & 1u) != 0u); // 13 - r0 = r0 ^ dataset[r4 & MASK]; // 14 - // per-load shadow sub-block 3 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 3 - for (uint sh3 = 0u; sh3 < 27u; ++sh3) { - r6 = mulhi(r6, r5); // s48 mulhi - r0 = r0 - r2; // s49 sub - r3 = r3 - r5; // s50 sub - r1 = rotr_var(r1, r4); // s51 rotr - r6 = r7 * r7 + r6; // s52 mad - r5 = r5 ^ r3; // s53 xor - r1 = r1 - r0; // s54 sub - r5 = r5 - r6; // s55 sub - r3 = r3 + r2 + select(0x3dfad1b6u, 0xd4758987u, ((sel >> 10u) & 1u) != 0u); // s56 add - r4 = r4 + r1 + select(0xfca75bc2u, 0x0602d6beu, ((sel >> 0u) & 1u) != 0u); // s57 add - r5 = mulhi(r5, r6); // s58 mulhi - r2 = r2 ^ simd_shuffle_xor(r1, (ushort)2); // s59 shfl - r2 = rotl_imm(r2, 9u); // s60 rotl - r4 = r4 | r6; // s61 or - r6 = rotr_var(r6, r4); // s62 rotr - r2 = r2 * r4; // s63 mul - } - r2 = r2 - r4; // 15 - r2 = r2 ^ dataset[r0 & MASK]; // 16 - // per-load shadow sub-block 4 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 4 - for (uint sh4 = 0u; sh4 < 27u; ++sh4) { - r0 = r0 ^ r5; // s64 xor - r2 = r2 ^ r7; // s65 xor - r2 = r2 + r1 + select(0xd71c02ffu, 0x8596687au, ((sel >> 6u) & 1u) != 0u); // s66 add - r1 = rotl_imm(r1, 21u); // s67 rotl - r3 = r3 * r2; // s68 mul - r7 = r7 ^ simd_shuffle_xor(r5, (ushort)2); // s69 shfl - r3 = r3 * r2; // s70 mul - r0 = r2 * r5 + r0; // s71 mad - r6 = r6 + r7 + select(0x3c6fe15du, 0x08ac5733u, ((sel >> 0u) & 1u) != 0u); // s72 add - r3 = r3 ^ simd_shuffle_xor(r2, (ushort)8); // s73 shfl - r3 = r3 * r6; // s74 mul - r6 = r6 ^ r7; // s75 xor - r3 = r3 | r0; // s76 or - r2 = r2 + r4 + select(0x295d5faeu, 0xc100b495u, ((sel >> 1u) & 1u) != 0u); // s77 add - r3 = mulhi(r3, r7); // s78 mulhi - r4 = r4 | r1; // s79 or - } - r7 = r7 ^ dataset[r2 & MASK]; // 17 - // per-load shadow sub-block 5 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 5 - for (uint sh5 = 0u; sh5 < 27u; ++sh5) { - r4 = rotr_var(r4, r3); // s80 rotr - r4 = r4 + r3 + select(0x274a9221u, 0x5cc59530u, ((sel >> 15u) & 1u) != 0u); // s81 add - r7 = r7 + r3 + select(0xc8651f8eu, 0x141479ecu, ((sel >> 5u) & 1u) != 0u); // s82 add - r3 = rotr_var(r3, r6); // s83 rotr - r2 = r4 * r7 + r2; // s84 mad - r2 = r2 ^ simd_shuffle_xor(r5, (ushort)16); // s85 shfl - r3 = r3 ^ r2; // s86 xor - r5 = r5 ^ simd_shuffle_xor(r7, (ushort)2); // s87 shfl - r0 = r0 ^ r4; // s88 xor - r3 = r3 + r2 + select(0x53f915c2u, 0x883c0c92u, ((sel >> 18u) & 1u) != 0u); // s89 add - r7 = rotr_var(r7, r5); // s90 rotr - r3 = r3 + r1 + select(0xe2be00a5u, 0x3cc5bd20u, ((sel >> 31u) & 1u) != 0u); // s91 add - r7 = r7 ^ simd_shuffle_xor(r2, (ushort)1); // s92 shfl - r6 = rotr_var(r6, r2); // s93 rotr - r7 = rotl_imm(r7, 14u); // s94 rotl - r4 = r5 * r5 + r4; // s95 mad - } - r7 = r7 ^ simd_shuffle_xor(r3, (ushort)4); // 18 - r5 = r5 * r0; // 19 - r3 = r3 ^ simd_shuffle_xor(r4, (ushort)2); // 20 - r2 = r2 ^ simd_shuffle_xor(r4, (ushort)16); // 21 - r6 = mulhi(r6, r2); // 22 - r6 = r6 ^ dataset[r1 & MASK]; // 23 - // per-load shadow sub-block 6 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 6 - for (uint sh6 = 0u; sh6 < 27u; ++sh6) { - r2 = r4 * r5 + r2; // s96 mad - r1 = r1 ^ r0; // s97 xor - r5 = r5 * r4; // s98 mul - r2 = r2 - r0; // s99 sub - r7 = rotl_imm(r7, 30u); // s100 rotl - r5 = r5 + r6 + select(0x423fd9c9u, 0xbfd646cbu, ((sel >> 17u) & 1u) != 0u); // s101 add - r7 = r7 + r6 + select(0x8d3c011du, 0x19b74a43u, ((sel >> 11u) & 1u) != 0u); // s102 add - r5 = rotl_imm(r5, 6u); // s103 rotl - r0 = r0 * r4; // s104 mul - r0 = r0 | r5; // s105 or - r0 = r0 + r1 + select(0x8ce14721u, 0x7dcb7e18u, ((sel >> 15u) & 1u) != 0u); // s106 add - r0 = rotl_imm(r0, 15u); // s107 rotl - r4 = r4 - r2; // s108 sub - r2 = mulhi(r2, r0); // s109 mulhi - r1 = mulhi(r1, r0); // s110 mulhi - r0 = r0 + r2 + select(0x2d9fc6b9u, 0x3c179ad8u, ((sel >> 28u) & 1u) != 0u); // s111 add - } - r5 = r5 * r0; // 24 - r5 = rotl_imm(r5, 19u); // 25 - r7 = r7 ^ simd_shuffle_xor(r6, (ushort)2); // 26 - r0 = r0 ^ r5; // 27 - r0 = r0 ^ r4; // 28 - r3 = r3 - r0; // 29 - r5 = r5 * r1; // 30 - r7 = r7 ^ dataset[r2 & MASK]; // 31 - // per-load shadow sub-block 7 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 7 - for (uint sh7 = 0u; sh7 < 27u; ++sh7) { - r2 = mulhi(r2, r0); // s112 mulhi - r6 = r6 - r3; // s113 sub - r7 = r6 * r3 + r7; // s114 mad - r5 = rotr_var(r5, r1); // s115 rotr - r6 = rotr_var(r6, r4); // s116 rotr - r2 = r2 + r1 + select(0x502138e2u, 0x47359729u, ((sel >> 22u) & 1u) != 0u); // s117 add - r3 = r2 * r0 + r3; // s118 mad - r1 = r1 * r3; // s119 mul - r2 = r0 * r4 + r2; // s120 mad - r0 = r0 * r3; // s121 mul - r2 = mulhi(r2, r0); // s122 mulhi - r5 = r5 * r6; // s123 mul - r4 = r2 * r4 + r4; // s124 mad - r4 = r4 ^ simd_shuffle_xor(r2, (ushort)2); // s125 shfl - r2 = r2 ^ r0; // s126 xor - r1 = rotl_imm(r1, 7u); // s127 rotl - } - r1 = r1 ^ dataset[r0 & MASK]; // 32 - // per-load shadow sub-block 8 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 8 - for (uint sh8 = 0u; sh8 < 27u; ++sh8) { - r7 = r7 ^ simd_shuffle_xor(r2, (ushort)4); // s128 shfl - r6 = rotl_imm(r6, 17u); // s129 rotl - r2 = r2 + r5 + select(0x33dff776u, 0x6c57e4e7u, ((sel >> 15u) & 1u) != 0u); // s130 add - r0 = r0 | r3; // s131 or - r5 = rotr_var(r5, r0); // s132 rotr - r2 = r2 + r3 + select(0x5db25b34u, 0xe8212c0cu, ((sel >> 30u) & 1u) != 0u); // s133 add - r4 = r4 ^ r3; // s134 xor - r6 = r6 ^ r1; // s135 xor - r1 = r1 - r7; // s136 sub - r2 = r6 * r2 + r2; // s137 mad - r0 = mulhi(r0, r5); // s138 mulhi - r2 = r2 + r7 + select(0x218d4090u, 0xd44ff710u, ((sel >> 10u) & 1u) != 0u); // s139 add - r1 = rotr_var(r1, r4); // s140 rotr - r3 = r3 ^ simd_shuffle_xor(r6, (ushort)1); // s141 shfl - r7 = r6 * r2 + r7; // s142 mad - r2 = r2 * r3; // s143 mul - } - r5 = r5 ^ r6; // 33 - r5 = r5 ^ dataset[r1 & MASK]; // 34 - // per-load shadow sub-block 9 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 9 - for (uint sh9 = 0u; sh9 < 27u; ++sh9) { - r7 = r7 + r4 + select(0xf4b1a8deu, 0xb98942fau, ((sel >> 29u) & 1u) != 0u); // s144 add - r7 = rotl_imm(r7, 15u); // s145 rotl - r7 = r7 ^ r5; // s146 xor - r4 = r7 * r4 + r4; // s147 mad - r6 = rotr_var(r6, r5); // s148 rotr - r1 = r2 * r4 + r1; // s149 mad - r1 = r1 + r7 + select(0x2bef10f2u, 0x0d48ba42u, ((sel >> 17u) & 1u) != 0u); // s150 add - r5 = r5 ^ r4; // s151 xor - r7 = r7 + r0 + select(0xdc5cc080u, 0xc98dea9cu, ((sel >> 30u) & 1u) != 0u); // s152 add - r4 = r4 * r6; // s153 mul - r0 = r0 * r2; // s154 mul - r6 = r6 ^ r5; // s155 xor - r4 = rotr_var(r4, r2); // s156 rotr - r1 = rotl_imm(r1, 11u); // s157 rotl - r5 = r5 + r0 + select(0x18197438u, 0x6c752dcbu, ((sel >> 11u) & 1u) != 0u); // s158 add - r4 = r1 * r5 + r4; // s159 mad - } - r0 = mulhi(r0, r5); // 35 - r5 = r5 ^ simd_shuffle_xor(r2, (ushort)4); // 36 - r7 = r7 ^ dataset[r0 & MASK]; // 37 - // per-load shadow sub-block 10 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 10 - for (uint sh10 = 0u; sh10 < 27u; ++sh10) { - r4 = rotl_imm(r4, 26u); // s160 rotl - r3 = r0 * r7 + r3; // s161 mad - r3 = rotr_var(r3, r1); // s162 rotr - r4 = r4 + r0 + select(0xf1c46574u, 0x8e481727u, ((sel >> 3u) & 1u) != 0u); // s163 add - r0 = mulhi(r0, r4); // s164 mulhi - r2 = r2 + r6 + select(0xac578137u, 0x550ab406u, ((sel >> 20u) & 1u) != 0u); // s165 add - r0 = rotl_imm(r0, 13u); // s166 rotl - r3 = r3 + r1 + select(0xa33e6706u, 0xaebb5966u, ((sel >> 14u) & 1u) != 0u); // s167 add - r3 = r3 | r5; // s168 or - r6 = rotr_var(r6, r2); // s169 rotr - r4 = r4 ^ r6; // s170 xor - r6 = r6 - r1; // s171 sub - r7 = rotl_imm(r7, 22u); // s172 rotl - r5 = rotl_imm(r5, 15u); // s173 rotl - r7 = r7 ^ simd_shuffle_xor(r0, (ushort)8); // s174 shfl - r0 = r0 ^ r5; // s175 xor - } - r3 = r3 + r1 + select(0x75ba2fadu, 0x230c005cu, ((sel >> 27u) & 1u) != 0u); // 38 - r1 = r1 ^ simd_shuffle_xor(r5, (ushort)4); // 39 - r2 = r2 ^ r5; // 40 - r3 = r6 * r3 + r3; // 41 - r6 = r6 - r7; // 42 - r7 = r7 ^ r0; // 43 - r1 = r1 ^ dataset[r7 & MASK]; // 44 - // per-load shadow sub-block 11 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 11 - for (uint sh11 = 0u; sh11 < 27u; ++sh11) { - r7 = rotl_imm(r7, 6u); // s176 rotl - r7 = r7 - r0; // s177 sub - r3 = rotl_imm(r3, 30u); // s178 rotl - r7 = r6 * r1 + r7; // s179 mad - r6 = rotl_imm(r6, 9u); // s180 rotl - r2 = r2 ^ r4; // s181 xor - r2 = r2 ^ r7; // s182 xor - r7 = r7 ^ r2; // s183 xor - r1 = r1 + r2 + select(0xd94d55acu, 0x5bb7550fu, ((sel >> 21u) & 1u) != 0u); // s184 add - r3 = r3 | r5; // s185 or - r6 = mulhi(r6, r3); // s186 mulhi - r4 = r4 | r0; // s187 or - r7 = r7 ^ simd_shuffle_xor(r1, (ushort)2); // s188 shfl - r6 = r6 + r5 + select(0x89e747feu, 0x2a354e2du, ((sel >> 6u) & 1u) != 0u); // s189 add - r1 = r1 ^ r4; // s190 xor - r7 = mulhi(r7, r4); // s191 mulhi - } - r2 = r2 * r3; // 45 - r1 = mulhi(r1, r5); // 46 - r4 = r4 - r3; // 47 - r2 = rotr_var(r2, r6); // 48 - r3 = r3 ^ dataset[r5 & MASK]; // 49 - // per-load shadow sub-block 12 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 12 - for (uint sh12 = 0u; sh12 < 27u; ++sh12) { - r2 = rotl_imm(r2, 26u); // s192 rotl - r5 = rotl_imm(r5, 8u); // s193 rotl - r4 = r4 ^ simd_shuffle_xor(r5, (ushort)16); // s194 shfl - r4 = r4 ^ r5; // s195 xor - r1 = r1 * r3; // s196 mul - r5 = r5 + r2 + select(0xea03e8e7u, 0x10cfdc71u, ((sel >> 7u) & 1u) != 0u); // s197 add - r7 = r7 + r5 + select(0xa7ee0102u, 0x66e148d3u, ((sel >> 15u) & 1u) != 0u); // s198 add - r5 = r5 - r2; // s199 sub - r5 = r5 ^ simd_shuffle_xor(r1, (ushort)2); // s200 shfl - r7 = r7 ^ simd_shuffle_xor(r1, (ushort)8); // s201 shfl - r4 = rotr_var(r4, r5); // s202 rotr - r7 = r7 ^ r5; // s203 xor - r7 = r7 ^ r0; // s204 xor - r6 = r6 + r2 + select(0x8379a4deu, 0x8558b619u, ((sel >> 10u) & 1u) != 0u); // s205 add - r4 = rotl_imm(r4, 13u); // s206 rotl - r1 = mulhi(r1, r3); // s207 mulhi - } - r1 = r1 + r5 + select(0x81b8bc2cu, 0x1907970cu, ((sel >> 7u) & 1u) != 0u); // 50 - r0 = r0 * r2; // 51 - r0 = r0 + r2 + select(0x4f92b968u, 0x699fd448u, ((sel >> 6u) & 1u) != 0u); // 52 - r1 = r1 + r0 + select(0x2bb965afu, 0x77b1520du, ((sel >> 12u) & 1u) != 0u); // 53 - r7 = rotl_imm(r7, 14u); // 54 - r3 = r3 + r7 + select(0x7b0fe07au, 0xa54c55a0u, ((sel >> 1u) & 1u) != 0u); // 55 - r6 = r6 ^ dataset[r7 & MASK]; // 56 - // per-load shadow sub-block 13 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 13 - for (uint sh13 = 0u; sh13 < 27u; ++sh13) { - r1 = r4 * r4 + r1; // s208 mad - r2 = r2 ^ simd_shuffle_xor(r0, (ushort)4); // s209 shfl - r7 = r7 + r0 + select(0xaa8cb14eu, 0xf4049c4cu, ((sel >> 11u) & 1u) != 0u); // s210 add - r7 = r7 ^ simd_shuffle_xor(r1, (ushort)16); // s211 shfl - r1 = r1 ^ r0; // s212 xor - r7 = rotl_imm(r7, 3u); // s213 rotl - r4 = rotr_var(r4, r7); // s214 rotr - r3 = r3 ^ simd_shuffle_xor(r2, (ushort)16); // s215 shfl - r5 = r5 | r1; // s216 or - r1 = r1 - r2; // s217 sub - r6 = r6 - r5; // s218 sub - r6 = rotl_imm(r6, 4u); // s219 rotl - r2 = mulhi(r2, r0); // s220 mulhi - r2 = r2 | r0; // s221 or - r5 = r5 ^ simd_shuffle_xor(r2, (ushort)8); // s222 shfl - r5 = mulhi(r5, r6); // s223 mulhi - } - r1 = rotr_var(r1, r5); // 57 - r5 = r5 ^ dataset[r4 & MASK]; // 58 - // per-load shadow sub-block 14 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 14 - for (uint sh14 = 0u; sh14 < 27u; ++sh14) { - r0 = r0 - r6; // s224 sub - r7 = rotl_imm(r7, 23u); // s225 rotl - r4 = r4 | r2; // s226 or - r2 = r2 * r4; // s227 mul - r3 = rotl_imm(r3, 12u); // s228 rotl - r0 = rotr_var(r0, r4); // s229 rotr - r0 = r0 + r6 + select(0x1d176220u, 0x2a7fecb2u, ((sel >> 16u) & 1u) != 0u); // s230 add - r1 = r1 ^ simd_shuffle_xor(r2, (ushort)4); // s231 shfl - r2 = r2 * r1; // s232 mul - r7 = r0 * r1 + r7; // s233 mad - r5 = rotl_imm(r5, 22u); // s234 rotl - r4 = rotr_var(r4, r6); // s235 rotr - r0 = r5 * r1 + r0; // s236 mad - r6 = r6 ^ r4; // s237 xor - r4 = r4 ^ r6; // s238 xor - r6 = rotl_imm(r6, 18u); // s239 rotl - } - r6 = r6 ^ dataset[r2 & MASK]; // 59 - // per-load shadow sub-block 15 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 15 - for (uint sh15 = 0u; sh15 < 27u; ++sh15) { - r4 = r4 + r7 + select(0x17dafb4du, 0xadce39f3u, ((sel >> 25u) & 1u) != 0u); // s240 add - r0 = r0 - r7; // s241 sub - r1 = rotr_var(r1, r6); // s242 rotr - r3 = r3 + r0 + select(0xf2f77d26u, 0x0e7033b6u, ((sel >> 29u) & 1u) != 0u); // s243 add - r2 = r7 * r4 + r2; // s244 mad - r4 = r0 * r6 + r4; // s245 mad - r5 = r5 * r7; // s246 mul - r3 = r3 + r5 + select(0xfed2da4eu, 0x7b2ff6b7u, ((sel >> 31u) & 1u) != 0u); // s247 add - r0 = r0 + r7 + select(0x03b2891cu, 0xb5fad7b1u, ((sel >> 0u) & 1u) != 0u); // s248 add - r5 = mulhi(r5, r4); // s249 mulhi - r5 = r5 + r2 + select(0x042cd6e3u, 0xa7e71c2fu, ((sel >> 11u) & 1u) != 0u); // s250 add - r6 = r6 ^ simd_shuffle_xor(r1, (ushort)1); // s251 shfl - r6 = r6 ^ r1; // s252 xor - r2 = r2 * r5; // s253 mul - r0 = r0 + r6 + select(0x784a302bu, 0xb83d78deu, ((sel >> 16u) & 1u) != 0u); // s254 add - r2 = r2 - r4; // s255 sub - } - r3 = r5 * r0 + r3; // 60 - r5 = r5 + r7 + select(0xaf9dd72du, 0xad7493e7u, ((sel >> 31u) & 1u) != 0u); // 61 - r4 = r4 + r6 + select(0x89841d87u, 0x1e07c3d9u, ((sel >> 27u) & 1u) != 0u); // 62 - r5 = rotl_imm(r5, 19u); // 63 - } - uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((ulong)hi << 32) | (ulong)lo; -} diff --git a/proto-cuda/packs-ca4/mx8_shl256x27/vectors.h b/proto-cuda/packs-ca4/mx8_shl256x27/vectors.h deleted file mode 100644 index 9172a34a0..000000000 --- a/proto-cuda/packs-ca4/mx8_shl256x27/vectors.h +++ /dev/null @@ -1,57 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// Expected outputs: igneum-pow (Rust) CPU interpreter, generator v2, memory-hard dataset -#pragma once -#ifdef __cplusplus -#include -#else -#include -#endif - -#define IGNEUM_VEC_WARPS 3 -static const uint32_t IGNEUM_VEC_BASE[IGNEUM_VEC_WARPS] = { 0u, 4096u, 1000000u }; -static const uint64_t IGNEUM_VEC_OUT[IGNEUM_VEC_WARPS][32] = { - { // base nonce 0 - 0x0a008d554d35429aull, 0xfb2681fdf5fc3293ull, 0x4526ec961092c4e4ull, 0x873267c98e5472cdull, 0xf576cf510f0b7015ull, 0xa7f4cdcfa91ba331ull, 0xa6881b77d915ed90ull, 0x7338e75fbdbd7f1eull, - 0xeec2f5b6580d07a7ull, 0xb5c7c1473819047dull, 0x3387a9ed8baf8711ull, 0xf446b831fa7b0595ull, 0xc49d4bd5f91c9555ull, 0xf186c878e0409cdbull, 0xbd93b92dd92e7681ull, 0x6b944e1145f76494ull, - 0x0bc18f251a45b475ull, 0x4f80f40a6e5d5150ull, 0x4d93a82ad0b3950bull, 0x3ea105c6c0f67533ull, 0x77db54bed6b58c77ull, 0x845d40f2699874edull, 0x79d2a0ba16fc489cull, 0x08cb4f6c2a078aacull, - 0xa01a53746f7a80fdull, 0x3b020b846570d9b5ull, 0xc93259df5bf323feull, 0xb5298b181a3a8b90ull, 0x764f8989fd118ed7ull, 0xd52ce83b540e69e7ull, 0xe114c6fdd36e7948ull, 0x2a7026992394c8f4ull - }, - { // base nonce 4096 - 0x181fc20625d777abull, 0x281c9b9bb3d2977bull, 0x1333a00594efc268ull, 0x002527da1c9a24bcull, 0xc124153842c78fbcull, 0x9904df37c80a0a8full, 0x82e480d0394587ceull, 0x3b8d36a65320a4ddull, - 0x2dfcc68e33b11dd1ull, 0x7d10b7c0b27b6b3full, 0x3d53db2f9bea5331ull, 0xeca210125b3ff139ull, 0x64d61fde830b3cdbull, 0x43ae5fed38e403faull, 0xace2a1ca730d129eull, 0xe3936ae1257fd154ull, - 0x5bf902aca786ee28ull, 0x39729472e23da36eull, 0xfc1abe26d12ee3eeull, 0x110aefe5a71baaa3ull, 0xa249ce6b0e53aa25ull, 0x10309737a2078f9eull, 0x4d6290a077e3700aull, 0xbe79096e287a4e5full, - 0x939900e342e34625ull, 0xd7300eb3ad23d58eull, 0x2414d887d6e3f195ull, 0x7d912e1eedd77aa3ull, 0xbeb6de75bdbbac36ull, 0xab1492af36df3568ull, 0x8efe5c666763648aull, 0x2dcfbe7cabf15656ull - }, - { // base nonce 1000000 - 0x9fded10a8c47e785ull, 0x945270db04af2013ull, 0xc7de12c1de45b88full, 0x8ea5c78cd1ebacdaull, 0xaec9ec7f1762c90full, 0x837d3e08fdae21a7ull, 0xd239e3e6845de2e8ull, 0xfc5204f6f61979f4ull, - 0x317db1642da77b55ull, 0xfecb12ccfcdf4068ull, 0x9cfb42154e45363bull, 0x64fd5e545960f93aull, 0x2f2b6a58ca36cfc1ull, 0xf8ff9b1150da80a5ull, 0xdf5f8386f019f7f9ull, 0xabdde56f5ae17efdull, - 0xf1076c8a92c91247ull, 0x5846ede66ab1cdf4ull, 0x3262ee2574f429c6ull, 0x00a0d76c16bbd34full, 0x3113163208d19833ull, 0xc74867fe9fd6b996ull, 0x689f2c59617de8ffull, 0x0be51f8944058adbull, - 0x90fad0ad13d3043dull, 0x9df977fac983c5fdull, 0x8c2da8911af1b94cull, 0xab614ff463ec4932ull, 0xd65687efae91acd1ull, 0xb4129eadf85178a0ull, 0x0ba3c9ec2acc25b8ull, 0xf2a041b8dfe31089ull - } -}; - -// Dataset self-test: dataset[0..15] and dataset[IGNEUM_MASK] (268435455). -static const uint32_t IGNEUM_DS_HEAD[16] = { - 0xfdad4319u, 0x1a7b68e1u, 0xde6db608u, 0x13d73892u, 0xd17f447au, 0xb2221ccfu, 0x9db004bdu, 0x57d7d367u, - 0xdbc4cf34u, 0x697c009au, 0xc43af1d4u, 0x97f12b2eu, 0x74c37cd0u, 0xc651ea15u, 0x665a6d29u, 0x22330a2du -}; -static const uint32_t IGNEUM_DS_LAST_INDEX = 268435455u; -static const uint32_t IGNEUM_DS_LAST = 0xa83e7aa6u; -// 64 sampled dataset words (index, value) computed on the Mac. -#define IGNEUM_DS_SAMPLES 64 -static const uint32_t IGNEUM_DS_SAMPLE_INDEX[IGNEUM_DS_SAMPLES] = { - 59471966u, 217795994u, 208353206u, 42483309u, 172547758u, 148076330u, 183853158u, 214389424u, 267488061u, 169781097u, 184093494u, 153880993u, 84977930u, 46426879u, 3093825u, 225364072u, 44593546u, 260713159u, 168250303u, 52384140u, 223401610u, 45554030u, 95410555u, 175039924u, 79171087u, 267580473u, 24168642u, 37981670u, 171551130u, 195559979u, 204611762u, 140997658u, 138925853u, 86637313u, 20736778u, 219665210u, 160430336u, 264654675u, 8013395u, 228945585u, 213884386u, 104419827u, 44185464u, 142737231u, 99284897u, 132475900u, 61861762u, 132056166u, 262388043u, 91878046u, 117353561u, 124768597u, 71352993u, 190698941u, 46055428u, 55281366u, 165145231u, 106810753u, 171985651u, 232085256u, 159510492u, 40072060u, 209107596u, 39023794u -}; -static const uint32_t IGNEUM_DS_SAMPLE_VALUE[IGNEUM_DS_SAMPLES] = { - 0x3230bc7bu, 0x7fbfe2c9u, 0xb2690991u, 0x1745c7c5u, 0x0ab0ccafu, 0x1bf87d6bu, 0x160139fdu, 0x719817acu, 0x0155df4bu, 0xbe1e86c3u, 0x680bcd6cu, 0x79c3dc6cu, 0x181e7e5fu, 0x0713a109u, 0xc705dd9fu, 0x3933b7a8u, 0xdd1c0431u, 0x50522b30u, 0xa0020b38u, 0xbff39e96u, 0x21b67e18u, 0x740f8db3u, 0x2baba568u, 0x2c9bef83u, 0x0ad9b671u, 0xc4327869u, 0x7b4fd7d0u, 0x2c29965fu, 0xec56f15fu, 0x61111746u, 0x303a1d6eu, 0xbddcfd1au, 0xf829a355u, 0x6d5df2a9u, 0x01ab8e44u, 0x06d13507u, 0xda8dcfc6u, 0x01a703e1u, 0xafe7d2c1u, 0xc091c3a2u, 0xac1814feu, 0x6e6ff62au, 0x8fdf01bau, 0xdd3f7159u, 0xdfa0d75cu, 0x26684c35u, 0x7f441e63u, 0x88df2570u, 0x8aa4d5ebu, 0xcc816c05u, 0x434df890u, 0xcd392ad6u, 0x1ab4cb63u, 0x595926fau, 0x7cd76b41u, 0x20cb95c4u, 0x13cf823fu, 0xf9daf901u, 0xff9af40au, 0x2c7dfa51u, 0x871206dbu, 0x938c116cu, 0xb64bf199u, 0x5751f874u -}; -// Cache self-test (memory-hard mode): cache[0..15], the last 16 words, and FNV-1a 64 over all 2^26 words. -static const uint32_t IGNEUM_CACHE_HEAD[16] = { - 0x355a86d2u, 0x7957db1cu, 0xd21772afu, 0x6fc1e09bu, 0xd55ce61du, 0x6e6a278bu, 0xd3f543ceu, 0x223d8e82u, - 0x143ab337u, 0x2e9f05bdu, 0x2eb389bfu, 0x0c6e449eu, 0x5cfa4222u, 0xba6560feu, 0x8e3e1aa4u, 0xdbcc1d53u -}; -static const uint32_t IGNEUM_CACHE_LAST[16] = { - 0x41190d91u, 0xbd277957u, 0x22ddbb49u, 0x6986f207u, 0xdf69a4d6u, 0x26401a3au, 0x818230fbu, 0xc417122du, - 0x3597b211u, 0xb553ce55u, 0xcf39cc0du, 0x3b7fc43au, 0x3fd43b00u, 0x67e1c80eu, 0xffa7ea7du, 0xca2960abu -}; -static const uint64_t IGNEUM_CACHE_FNV64 = 0x48c4f5bf24166b2eull; diff --git a/proto-cuda/packs-ca4/mx8_shl256x27/vectors.json b/proto-cuda/packs-ca4/mx8_shl256x27/vectors.json deleted file mode 100644 index 671e33cb7..000000000 --- a/proto-cuda/packs-ca4/mx8_shl256x27/vectors.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "seed": "igneum-genesis", - "day": "2026-10-03", - "dataset_mode": "memory-hard", - "dataset_log2_words": 28, - "mask": "0x0fffffff", - "lanes": 32, - "source": "igneum-pow (Rust) CPU interpreter, generator v2, memory-hard dataset", - "warps": [ - {"base_nonce": 0, "expected": [ - "0x0a008d554d35429a", "0xfb2681fdf5fc3293", "0x4526ec961092c4e4", "0x873267c98e5472cd", "0xf576cf510f0b7015", "0xa7f4cdcfa91ba331", "0xa6881b77d915ed90", "0x7338e75fbdbd7f1e", - "0xeec2f5b6580d07a7", "0xb5c7c1473819047d", "0x3387a9ed8baf8711", "0xf446b831fa7b0595", "0xc49d4bd5f91c9555", "0xf186c878e0409cdb", "0xbd93b92dd92e7681", "0x6b944e1145f76494", - "0x0bc18f251a45b475", "0x4f80f40a6e5d5150", "0x4d93a82ad0b3950b", "0x3ea105c6c0f67533", "0x77db54bed6b58c77", "0x845d40f2699874ed", "0x79d2a0ba16fc489c", "0x08cb4f6c2a078aac", - "0xa01a53746f7a80fd", "0x3b020b846570d9b5", "0xc93259df5bf323fe", "0xb5298b181a3a8b90", "0x764f8989fd118ed7", "0xd52ce83b540e69e7", "0xe114c6fdd36e7948", "0x2a7026992394c8f4" - ]}, - {"base_nonce": 4096, "expected": [ - "0x181fc20625d777ab", "0x281c9b9bb3d2977b", "0x1333a00594efc268", "0x002527da1c9a24bc", "0xc124153842c78fbc", "0x9904df37c80a0a8f", "0x82e480d0394587ce", "0x3b8d36a65320a4dd", - "0x2dfcc68e33b11dd1", "0x7d10b7c0b27b6b3f", "0x3d53db2f9bea5331", "0xeca210125b3ff139", "0x64d61fde830b3cdb", "0x43ae5fed38e403fa", "0xace2a1ca730d129e", "0xe3936ae1257fd154", - "0x5bf902aca786ee28", "0x39729472e23da36e", "0xfc1abe26d12ee3ee", "0x110aefe5a71baaa3", "0xa249ce6b0e53aa25", "0x10309737a2078f9e", "0x4d6290a077e3700a", "0xbe79096e287a4e5f", - "0x939900e342e34625", "0xd7300eb3ad23d58e", "0x2414d887d6e3f195", "0x7d912e1eedd77aa3", "0xbeb6de75bdbbac36", "0xab1492af36df3568", "0x8efe5c666763648a", "0x2dcfbe7cabf15656" - ]}, - {"base_nonce": 1000000, "expected": [ - "0x9fded10a8c47e785", "0x945270db04af2013", "0xc7de12c1de45b88f", "0x8ea5c78cd1ebacda", "0xaec9ec7f1762c90f", "0x837d3e08fdae21a7", "0xd239e3e6845de2e8", "0xfc5204f6f61979f4", - "0x317db1642da77b55", "0xfecb12ccfcdf4068", "0x9cfb42154e45363b", "0x64fd5e545960f93a", "0x2f2b6a58ca36cfc1", "0xf8ff9b1150da80a5", "0xdf5f8386f019f7f9", "0xabdde56f5ae17efd", - "0xf1076c8a92c91247", "0x5846ede66ab1cdf4", "0x3262ee2574f429c6", "0x00a0d76c16bbd34f", "0x3113163208d19833", "0xc74867fe9fd6b996", "0x689f2c59617de8ff", "0x0be51f8944058adb", - "0x90fad0ad13d3043d", "0x9df977fac983c5fd", "0x8c2da8911af1b94c", "0xab614ff463ec4932", "0xd65687efae91acd1", "0xb4129eadf85178a0", "0x0ba3c9ec2acc25b8", "0xf2a041b8dfe31089" - ]} - ], - "dataset_head": ["0xfdad4319", "0x1a7b68e1", "0xde6db608", "0x13d73892", "0xd17f447a", "0xb2221ccf", "0x9db004bd", "0x57d7d367", "0xdbc4cf34", "0x697c009a", "0xc43af1d4", "0x97f12b2e", "0x74c37cd0", "0xc651ea15", "0x665a6d29", "0x22330a2d"], - "dataset_last_index": 268435455, - "dataset_last": "0xa83e7aa6", - "dataset_samples": [{"index": 59471966, "value": "0x3230bc7b"}, {"index": 217795994, "value": "0x7fbfe2c9"}, {"index": 208353206, "value": "0xb2690991"}, {"index": 42483309, "value": "0x1745c7c5"}, {"index": 172547758, "value": "0x0ab0ccaf"}, {"index": 148076330, "value": "0x1bf87d6b"}, {"index": 183853158, "value": "0x160139fd"}, {"index": 214389424, "value": "0x719817ac"}, {"index": 267488061, "value": "0x0155df4b"}, {"index": 169781097, "value": "0xbe1e86c3"}, {"index": 184093494, "value": "0x680bcd6c"}, {"index": 153880993, "value": "0x79c3dc6c"}, {"index": 84977930, "value": "0x181e7e5f"}, {"index": 46426879, "value": "0x0713a109"}, {"index": 3093825, "value": "0xc705dd9f"}, {"index": 225364072, "value": "0x3933b7a8"}, {"index": 44593546, "value": "0xdd1c0431"}, {"index": 260713159, "value": "0x50522b30"}, {"index": 168250303, "value": "0xa0020b38"}, {"index": 52384140, "value": "0xbff39e96"}, {"index": 223401610, "value": "0x21b67e18"}, {"index": 45554030, "value": "0x740f8db3"}, {"index": 95410555, "value": "0x2baba568"}, {"index": 175039924, "value": "0x2c9bef83"}, {"index": 79171087, "value": "0x0ad9b671"}, {"index": 267580473, "value": "0xc4327869"}, {"index": 24168642, "value": "0x7b4fd7d0"}, {"index": 37981670, "value": "0x2c29965f"}, {"index": 171551130, "value": "0xec56f15f"}, {"index": 195559979, "value": "0x61111746"}, {"index": 204611762, "value": "0x303a1d6e"}, {"index": 140997658, "value": "0xbddcfd1a"}, {"index": 138925853, "value": "0xf829a355"}, {"index": 86637313, "value": "0x6d5df2a9"}, {"index": 20736778, "value": "0x01ab8e44"}, {"index": 219665210, "value": "0x06d13507"}, {"index": 160430336, "value": "0xda8dcfc6"}, {"index": 264654675, "value": "0x01a703e1"}, {"index": 8013395, "value": "0xafe7d2c1"}, {"index": 228945585, "value": "0xc091c3a2"}, {"index": 213884386, "value": "0xac1814fe"}, {"index": 104419827, "value": "0x6e6ff62a"}, {"index": 44185464, "value": "0x8fdf01ba"}, {"index": 142737231, "value": "0xdd3f7159"}, {"index": 99284897, "value": "0xdfa0d75c"}, {"index": 132475900, "value": "0x26684c35"}, {"index": 61861762, "value": "0x7f441e63"}, {"index": 132056166, "value": "0x88df2570"}, {"index": 262388043, "value": "0x8aa4d5eb"}, {"index": 91878046, "value": "0xcc816c05"}, {"index": 117353561, "value": "0x434df890"}, {"index": 124768597, "value": "0xcd392ad6"}, {"index": 71352993, "value": "0x1ab4cb63"}, {"index": 190698941, "value": "0x595926fa"}, {"index": 46055428, "value": "0x7cd76b41"}, {"index": 55281366, "value": "0x20cb95c4"}, {"index": 165145231, "value": "0x13cf823f"}, {"index": 106810753, "value": "0xf9daf901"}, {"index": 171985651, "value": "0xff9af40a"}, {"index": 232085256, "value": "0x2c7dfa51"}, {"index": 159510492, "value": "0x871206db"}, {"index": 40072060, "value": "0x938c116c"}, {"index": 209107596, "value": "0xb64bf199"}, {"index": 39023794, "value": "0x5751f874"}], - "cache_head": ["0x355a86d2", "0x7957db1c", "0xd21772af", "0x6fc1e09b", "0xd55ce61d", "0x6e6a278b", "0xd3f543ce", "0x223d8e82", "0x143ab337", "0x2e9f05bd", "0x2eb389bf", "0x0c6e449e", "0x5cfa4222", "0xba6560fe", "0x8e3e1aa4", "0xdbcc1d53"], - "cache_last_line": ["0x41190d91", "0xbd277957", "0x22ddbb49", "0x6986f207", "0xdf69a4d6", "0x26401a3a", "0x818230fb", "0xc417122d", "0x3597b211", "0xb553ce55", "0xcf39cc0d", "0x3b7fc43a", "0x3fd43b00", "0x67e1c80e", "0xffa7ea7d", "0xca2960ab"], - "cache_fnv1a64": "0x48c4f5bf24166b2e" -} diff --git a/proto-cuda/packs-ca4/mx8_shl256x27_v2/kernel.cl b/proto-cuda/packs-ca4/mx8_shl256x27_v2/kernel.cl deleted file mode 100644 index 72dd03640..000000000 --- a/proto-cuda/packs-ca4/mx8_shl256x27_v2/kernel.cl +++ /dev/null @@ -1,583 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// OpenCL C twin of the Metal kernel for the same seed (see proto-opencl/README.md, WAVEFRONT.md and program.metal). -// Built from source at runtime by proto-opencl/host.c, which passes these defines: -// IGNEUM_GROUP work-group size of igneum_hash, a multiple of 32 (default 32: one work-group = one 32-lane unit) -// IGNEUM_EXCHANGE 0 = local-memory exchange with a barrier (any device, any wave width; the default) -// 1 = sub_group_shuffle_xor (cl_khr_subgroup_shuffle), only with IGNEUM_GROUP 32 and a sub-group size of exactly 32 -// 2 = intel_sub_group_shuffle_xor (cl_intel_subgroups), same condition -// The verification unit is always 32 lanes. A 64-wide hardware wave (AMD GCN/CDNA, RDNA in wave64) runs two units; -// the exchange masks are 1, 2, 4, 8, 16, so every partner lane lies inside the lane's own aligned run of 32. -#ifndef IGNEUM_GROUP -#define IGNEUM_GROUP 32 -#endif -#ifndef IGNEUM_EXCHANGE -#define IGNEUM_EXCHANGE 0 -#endif -#ifdef __OPENCL_VERSION__ -#define IGNEUM_KERNEL_HASH __kernel __attribute__((reqd_work_group_size(IGNEUM_GROUP, 1, 1))) -#define IGNEUM_LOCAL_WORDS(name, n) __local uint name[n] -#if IGNEUM_EXCHANGE == 1 -#ifdef cl_khr_subgroups -#pragma OPENCL EXTENSION cl_khr_subgroups : enable -#endif -#ifdef cl_khr_subgroup_shuffle -#pragma OPENCL EXTENSION cl_khr_subgroup_shuffle : enable -#endif -#elif IGNEUM_EXCHANGE == 2 -#pragma OPENCL EXTENSION cl_intel_subgroups : enable -#endif -#else -// Not an OpenCL compiler: proto-opencl/emu compiles this file as C++ and supplies the built-ins and these two macros. -#include "emu_opencl.h" -#endif - -#if IGNEUM_EXCHANGE == 1 -#define IGNEUM_SHFL_XOR(dst, a, m) dst = sub_group_shuffle_xor((a), (uint)(m)) -#define IGNEUM_BCAST0(dst, a) dst = sub_group_broadcast((a), 0u) -#elif IGNEUM_EXCHANGE == 2 -#define IGNEUM_SHFL_XOR(dst, a, m) dst = intel_sub_group_shuffle_xor((a), (uint)(m)) -#define IGNEUM_BCAST0(dst, a) dst = sub_group_broadcast((a), 0u) -#else -// Local-memory exchange. Two buffers of IGNEUM_GROUP words alternate (xk counts exchanges), so one barrier per -// exchange is enough: a lane can only overwrite buffer b at exchange k+2 after passing barrier k+1, and every lane -// reaches barrier k+1 only after its read of buffer b at exchange k. The partner lid ^ m stays inside the lane's -// aligned run of 32 because m < 32. Control flow is uniform, so every work-item reaches every barrier. -#define IGNEUM_SHFL_XOR(dst, a, m) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid ^ (uint)(m))]; xk += 1u; } -#define IGNEUM_BCAST0(dst, a) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid & ~31u)]; xk += 1u; } -#endif - -static inline uint splitmix32(uint x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -// n is a literal in 1..31 at every call site. OpenCL rotate() rotates left by n modulo 32. -static inline uint rotl_imm(uint x, uint n) { return rotate(x, n); } -// Right rotation by n modulo 32 as a left rotation by (32 - n) modulo 32; n == 0 gives x. -static inline uint rotr_var(uint x, uint n) { return rotate(x, (0u - n) & 31u); } -static inline uint ds_elem(uint i, uint d0, uint d1) { - uint x = i ^ d0; - x *= 0x9E3779B1u; x ^= x >> 15; - x += d1; - x *= 0x85EBCA77u; x ^= x >> 13; - x *= 0xC2B2AE3Du; x ^= x >> 16; - return x; -} - -// Memory-hard dataset core (MEMHARD.md). Cache: 2^26 words in 2^16 segments of 64 chained ChaCha12 lines. -// Item: 8 rounds of 8 x seed-parameterised mixer + one 64-byte cache read, then 8 x final mixer (class v3, mixer multiplier 8, -// docs/plans/mixer-x4.md: the round key of application j of round r is 0x9E3779B9 * (r * m + j + 1)). All parameters are literals. -#define MH_CACHE_LINE_MASK 0x003fffffu -#define MH_SEGMENT_LINES 64u -#define MH_QR(a, b, c, d, r1, r2, r3, r4) { a += b; d ^= a; d = mh_rotl(d, r1); c += d; b ^= c; b = mh_rotl(b, r2); a += b; d ^= a; d = mh_rotl(d, r3); c += d; b ^= c; b = mh_rotl(b, r4); } -static inline uint mh_rotl(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 at every call site - -// y = ChaCha12 core(x) + x -static inline void mh_chacha_block(const uint* x, uint* y) { - for (uint i = 0u; i < 16u; ++i) y[i] = x[i]; - for (uint r = 0u; r < 6u; ++r) { - MH_QR(y[0], y[4], y[8], y[12], 16u, 12u, 8u, 7u) MH_QR(y[1], y[5], y[9], y[13], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[6], y[10], y[14], 16u, 12u, 8u, 7u) MH_QR(y[3], y[7], y[11], y[15], 16u, 12u, 8u, 7u) - MH_QR(y[0], y[5], y[10], y[15], 16u, 12u, 8u, 7u) MH_QR(y[1], y[6], y[11], y[12], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[7], y[8], y[13], 16u, 12u, 8u, 7u) MH_QR(y[3], y[4], y[9], y[14], 16u, 12u, 8u, 7u) - } - for (uint i = 0u; i < 16u; ++i) y[i] += x[i]; -} - -// One cache segment: 64 chained lines written at cache[seg * 1024]. in_j = prev ^ (sigma || K || seg || j || tag), prev_0 = 0. -static inline void mh_cache_segment(__global uint* cache, uint seg) { - uint prev[16]; uint x[16]; uint y[16]; - for (uint i = 0u; i < 16u; ++i) prev[i] = 0u; - for (uint j = 0u; j < MH_SEGMENT_LINES; ++j) { - x[0] = 0x61707865u ^ prev[0]; x[1] = 0x3320646eu ^ prev[1]; x[2] = 0x79622d32u ^ prev[2]; x[3] = 0x6b206574u ^ prev[3]; - x[4] = 0x3067619fu ^ prev[4]; - x[5] = 0x3c269176u ^ prev[5]; - x[6] = 0x84a03b03u ^ prev[6]; - x[7] = 0xf8c63294u ^ prev[7]; - x[8] = 0xff977c5bu ^ prev[8]; - x[9] = 0xe60def3eu ^ prev[9]; - x[10] = 0x63630141u ^ prev[10]; - x[11] = 0xb8fbcb58u ^ prev[11]; - x[12] = seg ^ prev[12]; x[13] = j ^ prev[13]; x[14] = 0x49676e65u ^ prev[14]; x[15] = 0x756d4d48u ^ prev[15]; - mh_chacha_block(x, y); - __global uint* line = cache + ((seg * MH_SEGMENT_LINES + j) * 16u); - for (uint i = 0u; i < 16u; ++i) { line[i] = y[i]; prev[i] = y[i]; } - } -} - -// M_r: per word (s ^ (RC + rk)) * MUL, then a column round and a diagonal round with the seed-drawn rotations. -static inline void mh_mixer(uint* s, uint rk) { - s[0] = (s[0] ^ (0xbab68293u + rk)) * 0x42146205u; - s[1] = (s[1] ^ (0xcc162340u + rk)) * 0x52cbe0fbu; - s[2] = (s[2] ^ (0x6ce151ccu + rk)) * 0x7ecf4a03u; - s[3] = (s[3] ^ (0xe62b8997u + rk)) * 0x6728907fu; - s[4] = (s[4] ^ (0xc9c80297u + rk)) * 0xd81d9751u; - s[5] = (s[5] ^ (0xf74a1654u + rk)) * 0x132952c3u; - s[6] = (s[6] ^ (0x3d704af5u + rk)) * 0xf60de277u; - s[7] = (s[7] ^ (0x3cf522b7u + rk)) * 0x05358035u; - s[8] = (s[8] ^ (0x2b9cac04u + rk)) * 0xbaf6499du; - s[9] = (s[9] ^ (0xa880ac10u + rk)) * 0xe4db9667u; - s[10] = (s[10] ^ (0x13e5dd1du + rk)) * 0x3e98f45du; - s[11] = (s[11] ^ (0x6fc3e233u + rk)) * 0xd0004eddu; - s[12] = (s[12] ^ (0x2d83eeacu + rk)) * 0x2691630du; - s[13] = (s[13] ^ (0x9006e8bfu + rk)) * 0x9beb3bcfu; - s[14] = (s[14] ^ (0x2c4b5362u + rk)) * 0xab310379u; - s[15] = (s[15] ^ (0x31b49ee2u + rk)) * 0x99cfb423u; - MH_QR(s[0], s[4], s[8], s[12], 20u, 20u, 19u, 4u) MH_QR(s[1], s[5], s[9], s[13], 20u, 20u, 19u, 4u) - MH_QR(s[2], s[6], s[10], s[14], 20u, 20u, 19u, 4u) MH_QR(s[3], s[7], s[11], s[15], 20u, 20u, 19u, 4u) - MH_QR(s[0], s[5], s[10], s[15], 26u, 3u, 3u, 27u) MH_QR(s[1], s[6], s[11], s[12], 26u, 3u, 3u, 27u) - MH_QR(s[2], s[7], s[8], s[13], 26u, 3u, 3u, 27u) MH_QR(s[3], s[4], s[9], s[14], 26u, 3u, 3u, 27u) -} - -// Item t: 16 words. s = (K, t * MUL[i] + RC[i]); 8 rounds of 8 x mixer + cache line s[0] & mask; 8 x final mixer. -static inline void mh_item(__global const uint* cache, uint t, uint* s) { - s[0] = 0x3067619fu; - s[1] = 0x3c269176u; - s[2] = 0x84a03b03u; - s[3] = 0xf8c63294u; - s[4] = 0xff977c5bu; - s[5] = 0xe60def3eu; - s[6] = 0x63630141u; - s[7] = 0xb8fbcb58u; - s[8] = t * 0x42146205u + 0xbab68293u; - s[9] = t * 0x52cbe0fbu + 0xcc162340u; - s[10] = t * 0x7ecf4a03u + 0x6ce151ccu; - s[11] = t * 0x6728907fu + 0xe62b8997u; - s[12] = t * 0xd81d9751u + 0xc9c80297u; - s[13] = t * 0x132952c3u + 0xf74a1654u; - s[14] = t * 0xf60de277u + 0x3d704af5u; - s[15] = t * 0x05358035u + 0x3cf522b7u; - for (uint r = 0u; r < 8u; ++r) { - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (r * 8u + j + 1u)); - __global const uint* line = cache + ((s[0] & MH_CACHE_LINE_MASK) * 16u); - for (uint i = 0u; i < 16u; ++i) s[i] ^= line[i]; - } - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (64u + j + 1u)); -} -// dataset[w] without the dataset: derive item w >> 4 and take word w & 15. -static inline uint mh_word(__global const uint* cache, uint w) { uint s[16]; mh_item(cache, w >> 4u, s); return s[w & 15u]; } - -// Memory-hard dataset (MEMHARD.md). One work-item per cache segment; one work-item per 64-byte dataset item. -// The same constants as memhard.h in this pack (one emitter, three dialects). -__kernel void igneum_cache_fill(__global uint* cache, uint nSegments) { - uint seg = (uint)get_global_id(0); - if (seg < nSegments) mh_cache_segment(cache, seg); -} -__kernel void igneum_build(__global uint* ds, __global const uint* cache, uint nItems) { - uint t = (uint)get_global_id(0); - if (t < nItems) { - uint s[16]; - mh_item(cache, t, s); - __global uint* d = ds + ((ulong)t * 16u); - for (uint i = 0u; i < 16u; ++i) d[i] = s[i]; - } -} - -// One hash per work-item. IGNEUM_GROUP is a multiple of 32; lane = lid & 31 and every exchange stays inside the -// lane's own aligned run of 32 work-items, exactly like simd_shuffle_xor inside a 32-wide Metal SIMD group and -// __shfl_xor_sync inside a CUDA warp. Control flow is uniform (no branches at all). -IGNEUM_KERNEL_HASH void igneum_hash(__global const uint* ds, __global ulong* out, uint baseNonce, uint mask) { - uint gid = (uint)get_global_id(0); - uint lid = (uint)get_local_id(0); - uint nonce = baseNonce + gid; - uint r0, r1, r2, r3, r4, r5, r6, r7; -#if IGNEUM_EXCHANGE == 0 - IGNEUM_LOCAL_WORDS(xch, 2 * IGNEUM_GROUP); - uint xk = 0u; -#else - (void)lid; -#endif - { uint x = nonce ^ 0xf71aee9fu; x += 0x9e3779b9u; x = splitmix32(x); r0 = x ^ 0xad930c88u; } // SEEDW[0], 0x9e3779b9u * 1u, SEEDW[1] - { uint x = nonce ^ 0xad930c88u; x += 0x3c6ef372u; x = splitmix32(x); r1 = x ^ 0x7f982573u; } // SEEDW[1], 0x9e3779b9u * 2u, SEEDW[2] - { uint x = nonce ^ 0x7f982573u; x += 0xdaa66d2bu; x = splitmix32(x); r2 = x ^ 0xa41f9137u; } // SEEDW[2], 0x9e3779b9u * 3u, SEEDW[3] - { uint x = nonce ^ 0xa41f9137u; x += 0x78dde6e4u; x = splitmix32(x); r3 = x ^ 0x76d803d6u; } // SEEDW[3], 0x9e3779b9u * 4u, SEEDW[4] - { uint x = nonce ^ 0x76d803d6u; x += 0x1715609du; x = splitmix32(x); r4 = x ^ 0x37b4a534u; } // SEEDW[4], 0x9e3779b9u * 5u, SEEDW[5] - { uint x = nonce ^ 0x37b4a534u; x += 0xb54cda56u; x = splitmix32(x); r5 = x ^ 0x4d3fb826u; } // SEEDW[5], 0x9e3779b9u * 6u, SEEDW[6] - { uint x = nonce ^ 0x4d3fb826u; x += 0x5384540fu; x = splitmix32(x); r6 = x ^ 0xff614dcbu; } // SEEDW[6], 0x9e3779b9u * 7u, SEEDW[7] - { uint x = nonce ^ 0xff614dcbu; x += 0xf1bbcdc8u; x = splitmix32(x); r7 = x ^ 0xf71aee9fu; } // SEEDW[7], 0x9e3779b9u * 8u, SEEDW[0] - - for (uint it = 0u; it < 8u; ++it) { - uint sel = r0; - r2 = r2 + r5 + ((((sel >> 2u) & 1u) != 0u) ? 0x894e457du : 0xe3e2ed7du); // 0 add - r7 = r4 * r0 + r7; // 1 mad - r3 = r3 - r6; // 2 sub - r5 = rotr_var(r5, r1); // 3 rotr - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 1u); r6 = r6 ^ t_; } // 4 shfl - r0 = r0 | r3; // 5 or - r0 = r0 * r1; // 6 mul - r7 = r7 * r6; // 7 mul - r3 = r3 ^ ds[r0 & mask]; // 8 load - // per-load shadow sub-block 0 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 0 - for (uint sh0 = 0u; sh0 < 27u; ++sh0) { - r5 = rotr_var(r5, r4); // s0 rotr - r6 = r6 - r5; // s1 sub - r7 = r7 - r6; // s2 sub - r5 = r5 * r4; // s3 mul - r7 = r7 ^ r6; // s4 xor - r2 = rotl_imm(r2, 23u); // s5 rotl - r7 = r7 ^ r4; // s6 xor - r0 = r0 + r7 + ((((sel >> 26u) & 1u) != 0u) ? 0x3051c491u : 0xdaef8862u); // s7 add - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 4u); r6 = r6 ^ t_; } // s8 shfl - r5 = r5 + r1 + ((((sel >> 26u) & 1u) != 0u) ? 0x5170d0b3u : 0xc20e7045u); // s9 add - r0 = r0 | r3; // s10 or - r5 = mul_hi(r5, r4); // s11 mulhi - r7 = r7 ^ r0; // s12 xor - r5 = r5 - r2; // s13 sub - r0 = r0 + r6 + ((((sel >> 0u) & 1u) != 0u) ? 0x2ead087fu : 0xc86d98a4u); // s14 add - r5 = r5 ^ r7; // s15 xor - } - r4 = r4 + r3 + ((((sel >> 22u) & 1u) != 0u) ? 0xd26d3573u : 0xce9bea84u); // 9 add - r2 = r2 * r4; // 10 mul - r3 = r3 ^ ds[r2 & mask]; // 11 load - // per-load shadow sub-block 1 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 1 - for (uint sh1 = 0u; sh1 < 27u; ++sh1) { - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 8u); r7 = r7 ^ t_; } // s16 shfl - r5 = r5 - r7; // s17 sub - r0 = r0 | r6; // s18 or - r0 = r0 + r5 + ((((sel >> 13u) & 1u) != 0u) ? 0xc47c70a8u : 0x4248b651u); // s19 add - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 16u); r4 = r4 ^ t_; } // s20 shfl - r0 = r0 + r4 + ((((sel >> 21u) & 1u) != 0u) ? 0x718c1008u : 0x1e35684fu); // s21 add - r0 = mul_hi(r0, r3); // s22 mulhi - r4 = r4 ^ r2; // s23 xor - r0 = r0 - r2; // s24 sub - r5 = rotl_imm(r5, 13u); // s25 rotl - r7 = r7 - r0; // s26 sub - r2 = r2 ^ r3; // s27 xor - r2 = r3 * r2 + r2; // s28 mad - r2 = r2 ^ r1; // s29 xor - r4 = mul_hi(r4, r0); // s30 mulhi - r6 = r6 ^ r4; // s31 xor - } - r4 = mul_hi(r4, r1); // 12 mulhi - r7 = mul_hi(r7, r3); // 13 mulhi - r3 = r3 + r2 + ((((sel >> 8u) & 1u) != 0u) ? 0xc2828a42u : 0x514f9ff4u); // 14 add - r1 = r3 * r2 + r1; // 15 mad - r2 = rotl_imm(r2, 7u); // 16 rotl - r1 = r1 ^ ds[r2 & mask]; // 17 load - // per-load shadow sub-block 2 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 2 - for (uint sh2 = 0u; sh2 < 27u; ++sh2) { - r4 = r4 ^ r2; // s32 xor - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 1u); r1 = r1 ^ t_; } // s33 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 1u); r5 = r5 ^ t_; } // s34 shfl - r4 = r4 - r0; // s35 sub - r6 = r6 - r3; // s36 sub - r2 = r2 | r6; // s37 or - r2 = rotl_imm(r2, 30u); // s38 rotl - r4 = rotr_var(r4, r7); // s39 rotr - r0 = r0 | r7; // s40 or - r2 = rotr_var(r2, r5); // s41 rotr - r1 = r3 * r3 + r1; // s42 mad - r6 = r5 * r5 + r6; // s43 mad - r1 = mul_hi(r1, r0); // s44 mulhi - r1 = r1 * r3; // s45 mul - r0 = r2 * r0 + r0; // s46 mad - r1 = r1 - r5; // s47 sub - } - r3 = r3 * r2; // 18 mul - r6 = rotl_imm(r6, 24u); // 19 rotl - r6 = r6 ^ ds[r3 & mask]; // 20 load - // per-load shadow sub-block 3 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 3 - for (uint sh3 = 0u; sh3 < 27u; ++sh3) { - r4 = r4 ^ r0; // s48 xor - r2 = rotr_var(r2, r0); // s49 rotr - r0 = mul_hi(r0, r5); // s50 mulhi - r7 = r7 | r5; // s51 or - r4 = r0 * r3 + r4; // s52 mad - r0 = rotr_var(r0, r4); // s53 rotr - r6 = r3 * r3 + r6; // s54 mad - r6 = r4 * r2 + r6; // s55 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r0, 2u); r5 = r5 ^ t_; } // s56 shfl - r7 = rotl_imm(r7, 21u); // s57 rotl - r3 = r3 ^ r7; // s58 xor - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 1u); r0 = r0 ^ t_; } // s59 shfl - r5 = r5 ^ r6; // s60 xor - r4 = rotr_var(r4, r7); // s61 rotr - r1 = r1 + r6 + ((((sel >> 10u) & 1u) != 0u) ? 0x31f87d74u : 0xf7ccf1e9u); // s62 add - r5 = r5 + r2 + ((((sel >> 15u) & 1u) != 0u) ? 0x9c2847f4u : 0x2f386099u); // s63 add - } - r1 = r1 ^ ds[r6 & mask]; // 21 load - // per-load shadow sub-block 4 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 4 - for (uint sh4 = 0u; sh4 < 27u; ++sh4) { - r7 = r7 + r0 + ((((sel >> 19u) & 1u) != 0u) ? 0xd3241188u : 0x1b30ce7au); // s64 add - r6 = r6 + r7 + ((((sel >> 13u) & 1u) != 0u) ? 0x9b42c3deu : 0x02dc8349u); // s65 add - r0 = rotl_imm(r0, 16u); // s66 rotl - r2 = rotl_imm(r2, 9u); // s67 rotl - r2 = r5 * r6 + r2; // s68 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 8u); r6 = r6 ^ t_; } // s69 shfl - r2 = r2 ^ r5; // s70 xor - r5 = mul_hi(r5, r1); // s71 mulhi - r6 = rotl_imm(r6, 29u); // s72 rotl - r0 = r0 - r7; // s73 sub - r5 = r2 * r2 + r5; // s74 mad - r0 = rotr_var(r0, r5); // s75 rotr - r7 = r7 + r2 + ((((sel >> 9u) & 1u) != 0u) ? 0x0da1ce17u : 0x05d36679u); // s76 add - r7 = r7 - r2; // s77 sub - r7 = rotr_var(r7, r0); // s78 rotr - r1 = r5 * r5 + r1; // s79 mad - } - r5 = r3 * r3 + r5; // 22 mad - r4 = r4 ^ ds[r7 & mask]; // 23 load - // per-load shadow sub-block 5 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 5 - for (uint sh5 = 0u; sh5 < 27u; ++sh5) { - r3 = rotr_var(r3, r0); // s80 rotr - r6 = r6 + r5 + ((((sel >> 2u) & 1u) != 0u) ? 0xadf5ef88u : 0xf14547bdu); // s81 add - r0 = r0 | r6; // s82 or - r1 = mul_hi(r1, r0); // s83 mulhi - r7 = r6 * r7 + r7; // s84 mad - r5 = rotl_imm(r5, 29u); // s85 rotl - r2 = r2 ^ r6; // s86 xor - r5 = r5 + r4 + ((((sel >> 24u) & 1u) != 0u) ? 0x35ff14aeu : 0xba4947c2u); // s87 add - r3 = r3 + r6 + ((((sel >> 3u) & 1u) != 0u) ? 0xf3900fc1u : 0xf6878beeu); // s88 add - r0 = r0 + r2 + ((((sel >> 12u) & 1u) != 0u) ? 0xf7e8f59fu : 0x926f3607u); // s89 add - r2 = r2 ^ r6; // s90 xor - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 2u); r3 = r3 ^ t_; } // s91 shfl - r6 = r6 - r1; // s92 sub - r5 = r5 + r1 + ((((sel >> 13u) & 1u) != 0u) ? 0x0e376f9cu : 0xdab36c29u); // s93 add - r6 = r1 * r1 + r6; // s94 mad - r3 = r3 ^ r4; // s95 xor - } - r6 = r6 ^ ds[r4 & mask]; // 24 load - // per-load shadow sub-block 6 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 6 - for (uint sh6 = 0u; sh6 < 27u; ++sh6) { - r6 = r5 * r2 + r6; // s96 mad - r7 = r7 + r3 + ((((sel >> 23u) & 1u) != 0u) ? 0x9f917747u : 0x204129e9u); // s97 add - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 16u); r5 = r5 ^ t_; } // s98 shfl - r2 = r2 * r6; // s99 mul - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 8u); r7 = r7 ^ t_; } // s100 shfl - r6 = r6 + r1 + ((((sel >> 10u) & 1u) != 0u) ? 0xe42fe974u : 0xe3b596a0u); // s101 add - r5 = rotl_imm(r5, 3u); // s102 rotl - r5 = r5 ^ r1; // s103 xor - r4 = r4 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0xc3d77ffbu : 0x705c3f94u); // s104 add - r0 = r0 + r6 + ((((sel >> 25u) & 1u) != 0u) ? 0xea02a4cau : 0xbc6fb42bu); // s105 add - r3 = r3 | r5; // s106 or - r0 = r0 - r6; // s107 sub - r0 = rotr_var(r0, r3); // s108 rotr - r5 = r5 + r4 + ((((sel >> 4u) & 1u) != 0u) ? 0x0357e63eu : 0xcb37ea87u); // s109 add - r7 = rotl_imm(r7, 5u); // s110 rotl - r4 = r4 ^ r1; // s111 xor - } - r5 = r5 * r7; // 25 mul - r0 = r0 * r3; // 26 mul - r0 = r0 | r3; // 27 or - r1 = r3 * r4 + r1; // 28 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 8u); r0 = r0 ^ t_; } // 29 shfl - r7 = r7 - r3; // 30 sub - r4 = r4 ^ r1; // 31 xor - r4 = r4 | r5; // 32 or - r3 = rotr_var(r3, r5); // 33 rotr - r4 = r4 ^ ds[r5 & mask]; // 34 load - // per-load shadow sub-block 7 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 7 - for (uint sh7 = 0u; sh7 < 27u; ++sh7) { - r6 = r6 ^ r5; // s112 xor - r1 = r1 | r3; // s113 or - r0 = r0 + r5 + ((((sel >> 3u) & 1u) != 0u) ? 0x7f8b8cd2u : 0x9aab0297u); // s114 add - r3 = r3 - r5; // s115 sub - r4 = rotr_var(r4, r2); // s116 rotr - r4 = r4 + r6 + ((((sel >> 6u) & 1u) != 0u) ? 0x59400b57u : 0xc511183fu); // s117 add - r6 = r6 - r1; // s118 sub - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 4u); r3 = r3 ^ t_; } // s119 shfl - r6 = r4 * r4 + r6; // s120 mad - r2 = rotl_imm(r2, 8u); // s121 rotl - r5 = r0 * r3 + r5; // s122 mad - r7 = r7 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x27c70dc0u : 0x5200c242u); // s123 add - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 4u); r3 = r3 ^ t_; } // s124 shfl - r0 = r0 + r1 + ((((sel >> 17u) & 1u) != 0u) ? 0xec2f5997u : 0x2d5028ceu); // s125 add - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 16u); r3 = r3 ^ t_; } // s126 shfl - r5 = r5 ^ r0; // s127 xor - } - r0 = r5 * r3 + r0; // 35 mad - r6 = r6 ^ ds[r3 & mask]; // 36 load - // per-load shadow sub-block 8 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 8 - for (uint sh8 = 0u; sh8 < 27u; ++sh8) { - r7 = r7 + r1 + ((((sel >> 10u) & 1u) != 0u) ? 0x09e8ede2u : 0x432f6c0du); // s128 add - r2 = r2 ^ r4; // s129 xor - r0 = r0 * r2; // s130 mul - r4 = r4 | r3; // s131 or - r3 = r3 | r7; // s132 or - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 1u); r7 = r7 ^ t_; } // s133 shfl - r5 = r5 + r2 + ((((sel >> 11u) & 1u) != 0u) ? 0xce4fdf8eu : 0xcf5ecc75u); // s134 add - r3 = r2 * r3 + r3; // s135 mad - r1 = rotl_imm(r1, 11u); // s136 rotl - r2 = r2 | r0; // s137 or - r3 = r3 ^ r4; // s138 xor - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 1u); r2 = r2 ^ t_; } // s139 shfl - r2 = rotl_imm(r2, 6u); // s140 rotl - r0 = r0 * r3; // s141 mul - r1 = r1 * r7; // s142 mul - r0 = r0 ^ r1; // s143 xor - } - r5 = r5 * r3; // 37 mul - r5 = r5 ^ r4; // 38 xor - r6 = r6 ^ r0; // 39 xor - r4 = rotr_var(r4, r0); // 40 rotr - r7 = r7 ^ r6; // 41 xor - r1 = r1 ^ ds[r7 & mask]; // 42 load - // per-load shadow sub-block 9 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 9 - for (uint sh9 = 0u; sh9 < 27u; ++sh9) { - r6 = r2 * r0 + r6; // s144 mad - r1 = rotl_imm(r1, 15u); // s145 rotl - r1 = rotl_imm(r1, 30u); // s146 rotl - r6 = r6 ^ r0; // s147 xor - r3 = r3 - r2; // s148 sub - r3 = r3 + r0 + ((((sel >> 5u) & 1u) != 0u) ? 0x11bbdeefu : 0xaa002f15u); // s149 add - r6 = r6 + r5 + ((((sel >> 9u) & 1u) != 0u) ? 0xff9d4b0eu : 0xbe64b2e2u); // s150 add - r1 = r1 * r4; // s151 mul - r5 = rotl_imm(r5, 3u); // s152 rotl - r5 = r5 * r2; // s153 mul - r2 = rotl_imm(r2, 21u); // s154 rotl - r5 = rotl_imm(r5, 18u); // s155 rotl - r1 = r1 ^ r5; // s156 xor - r1 = mul_hi(r1, r6); // s157 mulhi - r4 = r7 * r3 + r4; // s158 mad - r4 = r4 - r7; // s159 sub - } - r2 = r2 - r4; // 43 sub - r6 = mul_hi(r6, r7); // 44 mulhi - r3 = rotl_imm(r3, 13u); // 45 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 16u); r7 = r7 ^ t_; } // 46 shfl - r2 = rotl_imm(r2, 26u); // 47 rotl - r6 = r6 * r2; // 48 mul - r2 = r2 ^ ds[r3 & mask]; // 49 load - // per-load shadow sub-block 10 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 10 - for (uint sh10 = 0u; sh10 < 27u; ++sh10) { - r3 = r3 ^ r2; // s160 xor - r3 = mul_hi(r3, r4); // s161 mulhi - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 1u); r3 = r3 ^ t_; } // s162 shfl - r0 = r0 - r1; // s163 sub - { uint t_; IGNEUM_SHFL_XOR(t_, r0, 8u); r6 = r6 ^ t_; } // s164 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 2u); r4 = r4 ^ t_; } // s165 shfl - r3 = r3 | r6; // s166 or - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 4u); r6 = r6 ^ t_; } // s167 shfl - r2 = r2 ^ r1; // s168 xor - r5 = r5 ^ r1; // s169 xor - r5 = r5 ^ r0; // s170 xor - r3 = r6 * r0 + r3; // s171 mad - r3 = r3 - r0; // s172 sub - r6 = r6 + r0 + ((((sel >> 27u) & 1u) != 0u) ? 0xc72dc2a0u : 0x3b90694bu); // s173 add - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 8u); r7 = r7 ^ t_; } // s174 shfl - r5 = mul_hi(r5, r2); // s175 mulhi - } - r5 = r5 ^ ds[r1 & mask]; // 50 load - // per-load shadow sub-block 11 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 11 - for (uint sh11 = 0u; sh11 < 27u; ++sh11) { - r3 = r3 + r6 + ((((sel >> 5u) & 1u) != 0u) ? 0x4eb75843u : 0x9e65cebdu); // s176 add - r0 = r4 * r1 + r0; // s177 mad - r6 = rotr_var(r6, r7); // s178 rotr - r0 = r0 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0x7001d036u : 0x01e5b250u); // s179 add - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 16u); r5 = r5 ^ t_; } // s180 shfl - r3 = r3 * r0; // s181 mul - r0 = rotl_imm(r0, 23u); // s182 rotl - r7 = mul_hi(r7, r0); // s183 mulhi - r0 = r0 * r4; // s184 mul - r1 = r1 + r4 + ((((sel >> 19u) & 1u) != 0u) ? 0x91736711u : 0xbef14988u); // s185 add - r7 = r7 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x15e0cdf3u : 0xf5d741beu); // s186 add - r1 = r1 ^ r7; // s187 xor - r1 = r1 + r3 + ((((sel >> 21u) & 1u) != 0u) ? 0x7549bc3eu : 0x32be33c6u); // s188 add - r1 = r0 * r6 + r1; // s189 mad - r0 = rotr_var(r0, r5); // s190 rotr - r4 = r0 * r1 + r4; // s191 mad - } - r0 = rotl_imm(r0, 20u); // 51 rotl - r1 = r1 ^ r2; // 52 xor - r2 = r2 ^ ds[r1 & mask]; // 53 load - // per-load shadow sub-block 12 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 12 - for (uint sh12 = 0u; sh12 < 27u; ++sh12) { - r3 = r6 * r1 + r3; // s192 mad - r6 = rotl_imm(r6, 31u); // s193 rotl - r5 = rotl_imm(r5, 28u); // s194 rotl - r4 = r4 * r3; // s195 mul - r2 = r2 + r3 + ((((sel >> 4u) & 1u) != 0u) ? 0x7c3d6253u : 0x9d7aebbbu); // s196 add - r3 = r3 | r5; // s197 or - r6 = r6 - r0; // s198 sub - r7 = r7 * r4; // s199 mul - r3 = rotr_var(r3, r2); // s200 rotr - { uint t_; IGNEUM_SHFL_XOR(t_, r0, 16u); r4 = r4 ^ t_; } // s201 shfl - r7 = r7 ^ r3; // s202 xor - r4 = r4 | r3; // s203 or - r3 = rotr_var(r3, r6); // s204 rotr - r0 = rotr_var(r0, r1); // s205 rotr - r2 = r5 * r2 + r2; // s206 mad - r1 = mul_hi(r1, r5); // s207 mulhi - } - r4 = rotl_imm(r4, 20u); // 54 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 2u); r3 = r3 ^ t_; } // 55 shfl - r1 = r1 ^ ds[r0 & mask]; // 56 load - // per-load shadow sub-block 13 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 13 - for (uint sh13 = 0u; sh13 < 27u; ++sh13) { - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 1u); r4 = r4 ^ t_; } // s208 shfl - r7 = r5 * r7 + r7; // s209 mad - r6 = r6 ^ r3; // s210 xor - r3 = r3 ^ r2; // s211 xor - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 16u); r5 = r5 ^ t_; } // s212 shfl - r4 = rotl_imm(r4, 10u); // s213 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 16u); r5 = r5 ^ t_; } // s214 shfl - r3 = r3 | r0; // s215 or - r4 = r4 * r5; // s216 mul - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 16u); r0 = r0 ^ t_; } // s217 shfl - r0 = r0 * r5; // s218 mul - r0 = rotl_imm(r0, 13u); // s219 rotl - r1 = mul_hi(r1, r0); // s220 mulhi - r4 = r4 ^ r1; // s221 xor - r2 = r2 | r3; // s222 or - r0 = r0 ^ r4; // s223 xor - } - r3 = r3 ^ ds[r5 & mask]; // 57 load - // per-load shadow sub-block 14 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 14 - for (uint sh14 = 0u; sh14 < 27u; ++sh14) { - r5 = r6 * r1 + r5; // s224 mad - r4 = r4 ^ r3; // s225 xor - r5 = r5 + r0 + ((((sel >> 3u) & 1u) != 0u) ? 0xe4e51c75u : 0xfe960971u); // s226 add - r3 = mul_hi(r3, r2); // s227 mulhi - r2 = r2 ^ r6; // s228 xor - r1 = mul_hi(r1, r5); // s229 mulhi - r3 = r3 + r4 + ((((sel >> 25u) & 1u) != 0u) ? 0x4d597c08u : 0x48b3ce0au); // s230 add - r2 = rotr_var(r2, r3); // s231 rotr - r2 = r2 - r7; // s232 sub - r6 = r6 | r2; // s233 or - r0 = rotr_var(r0, r3); // s234 rotr - r4 = r4 + r3 + ((((sel >> 13u) & 1u) != 0u) ? 0xc9f1d54cu : 0x2ed8c878u); // s235 add - r0 = r3 * r7 + r0; // s236 mad - r3 = r3 + r5 + ((((sel >> 29u) & 1u) != 0u) ? 0xc572bd00u : 0x22e8b90au); // s237 add - r0 = mul_hi(r0, r4); // s238 mulhi - r6 = r6 * r5; // s239 mul - } - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 2u); r1 = r1 ^ t_; } // 58 shfl - r6 = r6 ^ ds[r4 & mask]; // 59 load - // per-load shadow sub-block 15 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 15 - for (uint sh15 = 0u; sh15 < 27u; ++sh15) { - r5 = r5 + r2 + ((((sel >> 20u) & 1u) != 0u) ? 0xb233f94fu : 0xc6b790e6u); // s240 add - r0 = r0 - r3; // s241 sub - r4 = r4 + r7 + ((((sel >> 4u) & 1u) != 0u) ? 0x534d924bu : 0x0f918d3bu); // s242 add - r5 = rotr_var(r5, r0); // s243 rotr - r1 = rotr_var(r1, r7); // s244 rotr - r2 = r2 | r0; // s245 or - r4 = r4 * r7; // s246 mul - r1 = r2 * r5 + r1; // s247 mad - r2 = r2 * r5; // s248 mul - r4 = r4 ^ r2; // s249 xor - r3 = r3 ^ r1; // s250 xor - r5 = rotr_var(r5, r7); // s251 rotr - r2 = r7 * r7 + r2; // s252 mad - r7 = rotr_var(r7, r3); // s253 rotr - r6 = r6 + r1 + ((((sel >> 21u) & 1u) != 0u) ? 0x86d27169u : 0xb8a27d95u); // s254 add - r6 = r6 | r7; // s255 or - } - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 4u); r7 = r7 ^ t_; } // 60 shfl - r5 = rotr_var(r5, r0); // 61 rotr - r0 = r0 + r6 + ((((sel >> 14u) & 1u) != 0u) ? 0xb13a5391u : 0x8c2e5c24u); // 62 add - r2 = r2 + r1 + ((((sel >> 21u) & 1u) != 0u) ? 0xf82fc8b5u : 0xb225b762u); // 63 add - } - uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((ulong)hi << 32) | (ulong)lo; -} - -#if IGNEUM_EXCHANGE != 0 -// Reports the sub-group size this device uses for a work-group of IGNEUM_GROUP items. host.c runs it only when the -// per-kernel query (clGetKernelSubGroupInfoKHR on igneum_hash) is unavailable; that query is preferred because a -// compiler may pick a different wave width per kernel (RDNA: wave32 or wave64). See WAVEFRONT.md. -IGNEUM_KERNEL_HASH void igneum_probe_subgroup(__global uint* out) { - if (get_local_id(0) == 0u) { out[0] = get_sub_group_size(); out[1] = get_num_sub_groups(); } -} -#endif diff --git a/proto-cuda/packs-ca4/mx8_shl256x27_v2/kernel.cu b/proto-cuda/packs-ca4/mx8_shl256x27_v2/kernel.cu deleted file mode 100644 index c68c11857..000000000 --- a/proto-cuda/packs-ca4/mx8_shl256x27_v2/kernel.cu +++ /dev/null @@ -1,468 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// Bit-exact twin of the Metal kernel for the same seed (see proto-cuda/CHECKLIST.md and program.metal). -// Compiled ahead of time by nvcc together with proto-cuda/host.cu. No NVRTC. -#include -#include -#include "program.h" -#include "memhard.h" - -__device__ __forceinline__ uint32_t splitmix32(uint32_t x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -// n is a literal in 1..31 at every call site, so both shift amounts are in 1..31. -__device__ __forceinline__ uint32_t rotl_imm(uint32_t x, uint32_t n) { return (x << n) | (x >> (32u - n)); } -// n is masked to 0..31; the second shift amount is masked too, so n == 0 gives x. -__device__ __forceinline__ uint32_t rotr_var(uint32_t x, uint32_t n) { n &= 31u; return (x >> n) | (x << ((32u - n) & 31u)); } -__device__ __forceinline__ uint32_t ds_elem(uint32_t i, uint32_t d0, uint32_t d1) { - uint32_t x = i ^ d0; - x *= 0x9E3779B1u; x ^= x >> 15; - x += d1; - x *= 0x85EBCA77u; x ^= x >> 13; - x *= 0xC2B2AE3Du; x ^= x >> 16; - return x; -} - -// Memory-hard dataset (MEMHARD.md). One thread per cache segment; one thread per 64-byte dataset item. -// The core functions (mh_cache_segment, mh_item) are in memhard.h and are also compiled for the host. -__global__ void igneum_cache_fill(uint32_t* cache, uint32_t nSegments) { - uint32_t seg = blockIdx.x * blockDim.x + threadIdx.x; - if (seg < nSegments) mh_cache_segment(cache, seg); -} -__global__ void igneum_build(uint32_t* ds, const uint32_t* cache, uint32_t nItems) { - uint32_t t = blockIdx.x * blockDim.x + threadIdx.x; - if (t < nItems) { - uint32_t s[16]; - mh_item(cache, t, s); - uint32_t* d = ds + (size_t)t * 16u; - for (uint32_t i = 0u; i < 16u; ++i) d[i] = s[i]; - } -} - -// One hash per thread. blockDim.x is a multiple of 32; lane = threadIdx.x & 31 and every -// __shfl_xor_sync stays inside the lane's own warp, exactly like simd_shuffle_xor inside a -// 32-wide Metal SIMD group. Control flow is uniform, so the full 0xffffffff member mask is valid. -__global__ void igneum_hash(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask) { - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t nonce = baseNonce + gid; - uint32_t r0, r1, r2, r3, r4, r5, r6, r7; - { uint32_t x = nonce ^ 0xf71aee9fu; x += 0x9e3779b9u; x = splitmix32(x); r0 = x ^ 0xad930c88u; } // SEEDW[0], 0x9e3779b9u * 1u, SEEDW[1] - { uint32_t x = nonce ^ 0xad930c88u; x += 0x3c6ef372u; x = splitmix32(x); r1 = x ^ 0x7f982573u; } // SEEDW[1], 0x9e3779b9u * 2u, SEEDW[2] - { uint32_t x = nonce ^ 0x7f982573u; x += 0xdaa66d2bu; x = splitmix32(x); r2 = x ^ 0xa41f9137u; } // SEEDW[2], 0x9e3779b9u * 3u, SEEDW[3] - { uint32_t x = nonce ^ 0xa41f9137u; x += 0x78dde6e4u; x = splitmix32(x); r3 = x ^ 0x76d803d6u; } // SEEDW[3], 0x9e3779b9u * 4u, SEEDW[4] - { uint32_t x = nonce ^ 0x76d803d6u; x += 0x1715609du; x = splitmix32(x); r4 = x ^ 0x37b4a534u; } // SEEDW[4], 0x9e3779b9u * 5u, SEEDW[5] - { uint32_t x = nonce ^ 0x37b4a534u; x += 0xb54cda56u; x = splitmix32(x); r5 = x ^ 0x4d3fb826u; } // SEEDW[5], 0x9e3779b9u * 6u, SEEDW[6] - { uint32_t x = nonce ^ 0x4d3fb826u; x += 0x5384540fu; x = splitmix32(x); r6 = x ^ 0xff614dcbu; } // SEEDW[6], 0x9e3779b9u * 7u, SEEDW[7] - { uint32_t x = nonce ^ 0xff614dcbu; x += 0xf1bbcdc8u; x = splitmix32(x); r7 = x ^ 0xf71aee9fu; } // SEEDW[7], 0x9e3779b9u * 8u, SEEDW[0] - - for (uint32_t it = 0u; it < 8u; ++it) { - uint32_t sel = r0; - r2 = r2 + r5 + ((((sel >> 2u) & 1u) != 0u) ? 0x894e457du : 0xe3e2ed7du); // 0 add - r7 = r4 * r0 + r7; // 1 mad - r3 = r3 - r6; // 2 sub - r5 = rotr_var(r5, r1); // 3 rotr - r6 = r6 ^ __shfl_xor_sync(0xffffffffu, r2, 1); // 4 shfl - r0 = r0 | r3; // 5 or - r0 = r0 * r1; // 6 mul - r7 = r7 * r6; // 7 mul - r3 = r3 ^ ds[r0 & mask]; // 8 load - // per-load shadow sub-block 0 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 0 - for (uint32_t sh0 = 0u; sh0 < 27u; ++sh0) { - r5 = rotr_var(r5, r4); // s0 rotr - r6 = r6 - r5; // s1 sub - r7 = r7 - r6; // s2 sub - r5 = r5 * r4; // s3 mul - r7 = r7 ^ r6; // s4 xor - r2 = rotl_imm(r2, 23u); // s5 rotl - r7 = r7 ^ r4; // s6 xor - r0 = r0 + r7 + ((((sel >> 26u) & 1u) != 0u) ? 0x3051c491u : 0xdaef8862u); // s7 add - r6 = r6 ^ __shfl_xor_sync(0xffffffffu, r1, 4); // s8 shfl - r5 = r5 + r1 + ((((sel >> 26u) & 1u) != 0u) ? 0x5170d0b3u : 0xc20e7045u); // s9 add - r0 = r0 | r3; // s10 or - r5 = __umulhi(r5, r4); // s11 mulhi - r7 = r7 ^ r0; // s12 xor - r5 = r5 - r2; // s13 sub - r0 = r0 + r6 + ((((sel >> 0u) & 1u) != 0u) ? 0x2ead087fu : 0xc86d98a4u); // s14 add - r5 = r5 ^ r7; // s15 xor - } - r4 = r4 + r3 + ((((sel >> 22u) & 1u) != 0u) ? 0xd26d3573u : 0xce9bea84u); // 9 add - r2 = r2 * r4; // 10 mul - r3 = r3 ^ ds[r2 & mask]; // 11 load - // per-load shadow sub-block 1 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 1 - for (uint32_t sh1 = 0u; sh1 < 27u; ++sh1) { - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r6, 8); // s16 shfl - r5 = r5 - r7; // s17 sub - r0 = r0 | r6; // s18 or - r0 = r0 + r5 + ((((sel >> 13u) & 1u) != 0u) ? 0xc47c70a8u : 0x4248b651u); // s19 add - r4 = r4 ^ __shfl_xor_sync(0xffffffffu, r5, 16); // s20 shfl - r0 = r0 + r4 + ((((sel >> 21u) & 1u) != 0u) ? 0x718c1008u : 0x1e35684fu); // s21 add - r0 = __umulhi(r0, r3); // s22 mulhi - r4 = r4 ^ r2; // s23 xor - r0 = r0 - r2; // s24 sub - r5 = rotl_imm(r5, 13u); // s25 rotl - r7 = r7 - r0; // s26 sub - r2 = r2 ^ r3; // s27 xor - r2 = r3 * r2 + r2; // s28 mad - r2 = r2 ^ r1; // s29 xor - r4 = __umulhi(r4, r0); // s30 mulhi - r6 = r6 ^ r4; // s31 xor - } - r4 = __umulhi(r4, r1); // 12 mulhi - r7 = __umulhi(r7, r3); // 13 mulhi - r3 = r3 + r2 + ((((sel >> 8u) & 1u) != 0u) ? 0xc2828a42u : 0x514f9ff4u); // 14 add - r1 = r3 * r2 + r1; // 15 mad - r2 = rotl_imm(r2, 7u); // 16 rotl - r1 = r1 ^ ds[r2 & mask]; // 17 load - // per-load shadow sub-block 2 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 2 - for (uint32_t sh2 = 0u; sh2 < 27u; ++sh2) { - r4 = r4 ^ r2; // s32 xor - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r2, 1); // s33 shfl - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r6, 1); // s34 shfl - r4 = r4 - r0; // s35 sub - r6 = r6 - r3; // s36 sub - r2 = r2 | r6; // s37 or - r2 = rotl_imm(r2, 30u); // s38 rotl - r4 = rotr_var(r4, r7); // s39 rotr - r0 = r0 | r7; // s40 or - r2 = rotr_var(r2, r5); // s41 rotr - r1 = r3 * r3 + r1; // s42 mad - r6 = r5 * r5 + r6; // s43 mad - r1 = __umulhi(r1, r0); // s44 mulhi - r1 = r1 * r3; // s45 mul - r0 = r2 * r0 + r0; // s46 mad - r1 = r1 - r5; // s47 sub - } - r3 = r3 * r2; // 18 mul - r6 = rotl_imm(r6, 24u); // 19 rotl - r6 = r6 ^ ds[r3 & mask]; // 20 load - // per-load shadow sub-block 3 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 3 - for (uint32_t sh3 = 0u; sh3 < 27u; ++sh3) { - r4 = r4 ^ r0; // s48 xor - r2 = rotr_var(r2, r0); // s49 rotr - r0 = __umulhi(r0, r5); // s50 mulhi - r7 = r7 | r5; // s51 or - r4 = r0 * r3 + r4; // s52 mad - r0 = rotr_var(r0, r4); // s53 rotr - r6 = r3 * r3 + r6; // s54 mad - r6 = r4 * r2 + r6; // s55 mad - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r0, 2); // s56 shfl - r7 = rotl_imm(r7, 21u); // s57 rotl - r3 = r3 ^ r7; // s58 xor - r0 = r0 ^ __shfl_xor_sync(0xffffffffu, r4, 1); // s59 shfl - r5 = r5 ^ r6; // s60 xor - r4 = rotr_var(r4, r7); // s61 rotr - r1 = r1 + r6 + ((((sel >> 10u) & 1u) != 0u) ? 0x31f87d74u : 0xf7ccf1e9u); // s62 add - r5 = r5 + r2 + ((((sel >> 15u) & 1u) != 0u) ? 0x9c2847f4u : 0x2f386099u); // s63 add - } - r1 = r1 ^ ds[r6 & mask]; // 21 load - // per-load shadow sub-block 4 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 4 - for (uint32_t sh4 = 0u; sh4 < 27u; ++sh4) { - r7 = r7 + r0 + ((((sel >> 19u) & 1u) != 0u) ? 0xd3241188u : 0x1b30ce7au); // s64 add - r6 = r6 + r7 + ((((sel >> 13u) & 1u) != 0u) ? 0x9b42c3deu : 0x02dc8349u); // s65 add - r0 = rotl_imm(r0, 16u); // s66 rotl - r2 = rotl_imm(r2, 9u); // s67 rotl - r2 = r5 * r6 + r2; // s68 mad - r6 = r6 ^ __shfl_xor_sync(0xffffffffu, r1, 8); // s69 shfl - r2 = r2 ^ r5; // s70 xor - r5 = __umulhi(r5, r1); // s71 mulhi - r6 = rotl_imm(r6, 29u); // s72 rotl - r0 = r0 - r7; // s73 sub - r5 = r2 * r2 + r5; // s74 mad - r0 = rotr_var(r0, r5); // s75 rotr - r7 = r7 + r2 + ((((sel >> 9u) & 1u) != 0u) ? 0x0da1ce17u : 0x05d36679u); // s76 add - r7 = r7 - r2; // s77 sub - r7 = rotr_var(r7, r0); // s78 rotr - r1 = r5 * r5 + r1; // s79 mad - } - r5 = r3 * r3 + r5; // 22 mad - r4 = r4 ^ ds[r7 & mask]; // 23 load - // per-load shadow sub-block 5 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 5 - for (uint32_t sh5 = 0u; sh5 < 27u; ++sh5) { - r3 = rotr_var(r3, r0); // s80 rotr - r6 = r6 + r5 + ((((sel >> 2u) & 1u) != 0u) ? 0xadf5ef88u : 0xf14547bdu); // s81 add - r0 = r0 | r6; // s82 or - r1 = __umulhi(r1, r0); // s83 mulhi - r7 = r6 * r7 + r7; // s84 mad - r5 = rotl_imm(r5, 29u); // s85 rotl - r2 = r2 ^ r6; // s86 xor - r5 = r5 + r4 + ((((sel >> 24u) & 1u) != 0u) ? 0x35ff14aeu : 0xba4947c2u); // s87 add - r3 = r3 + r6 + ((((sel >> 3u) & 1u) != 0u) ? 0xf3900fc1u : 0xf6878beeu); // s88 add - r0 = r0 + r2 + ((((sel >> 12u) & 1u) != 0u) ? 0xf7e8f59fu : 0x926f3607u); // s89 add - r2 = r2 ^ r6; // s90 xor - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r2, 2); // s91 shfl - r6 = r6 - r1; // s92 sub - r5 = r5 + r1 + ((((sel >> 13u) & 1u) != 0u) ? 0x0e376f9cu : 0xdab36c29u); // s93 add - r6 = r1 * r1 + r6; // s94 mad - r3 = r3 ^ r4; // s95 xor - } - r6 = r6 ^ ds[r4 & mask]; // 24 load - // per-load shadow sub-block 6 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 6 - for (uint32_t sh6 = 0u; sh6 < 27u; ++sh6) { - r6 = r5 * r2 + r6; // s96 mad - r7 = r7 + r3 + ((((sel >> 23u) & 1u) != 0u) ? 0x9f917747u : 0x204129e9u); // s97 add - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r1, 16); // s98 shfl - r2 = r2 * r6; // s99 mul - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r3, 8); // s100 shfl - r6 = r6 + r1 + ((((sel >> 10u) & 1u) != 0u) ? 0xe42fe974u : 0xe3b596a0u); // s101 add - r5 = rotl_imm(r5, 3u); // s102 rotl - r5 = r5 ^ r1; // s103 xor - r4 = r4 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0xc3d77ffbu : 0x705c3f94u); // s104 add - r0 = r0 + r6 + ((((sel >> 25u) & 1u) != 0u) ? 0xea02a4cau : 0xbc6fb42bu); // s105 add - r3 = r3 | r5; // s106 or - r0 = r0 - r6; // s107 sub - r0 = rotr_var(r0, r3); // s108 rotr - r5 = r5 + r4 + ((((sel >> 4u) & 1u) != 0u) ? 0x0357e63eu : 0xcb37ea87u); // s109 add - r7 = rotl_imm(r7, 5u); // s110 rotl - r4 = r4 ^ r1; // s111 xor - } - r5 = r5 * r7; // 25 mul - r0 = r0 * r3; // 26 mul - r0 = r0 | r3; // 27 or - r1 = r3 * r4 + r1; // 28 mad - r0 = r0 ^ __shfl_xor_sync(0xffffffffu, r2, 8); // 29 shfl - r7 = r7 - r3; // 30 sub - r4 = r4 ^ r1; // 31 xor - r4 = r4 | r5; // 32 or - r3 = rotr_var(r3, r5); // 33 rotr - r4 = r4 ^ ds[r5 & mask]; // 34 load - // per-load shadow sub-block 7 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 7 - for (uint32_t sh7 = 0u; sh7 < 27u; ++sh7) { - r6 = r6 ^ r5; // s112 xor - r1 = r1 | r3; // s113 or - r0 = r0 + r5 + ((((sel >> 3u) & 1u) != 0u) ? 0x7f8b8cd2u : 0x9aab0297u); // s114 add - r3 = r3 - r5; // s115 sub - r4 = rotr_var(r4, r2); // s116 rotr - r4 = r4 + r6 + ((((sel >> 6u) & 1u) != 0u) ? 0x59400b57u : 0xc511183fu); // s117 add - r6 = r6 - r1; // s118 sub - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r1, 4); // s119 shfl - r6 = r4 * r4 + r6; // s120 mad - r2 = rotl_imm(r2, 8u); // s121 rotl - r5 = r0 * r3 + r5; // s122 mad - r7 = r7 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x27c70dc0u : 0x5200c242u); // s123 add - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r5, 4); // s124 shfl - r0 = r0 + r1 + ((((sel >> 17u) & 1u) != 0u) ? 0xec2f5997u : 0x2d5028ceu); // s125 add - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r6, 16); // s126 shfl - r5 = r5 ^ r0; // s127 xor - } - r0 = r5 * r3 + r0; // 35 mad - r6 = r6 ^ ds[r3 & mask]; // 36 load - // per-load shadow sub-block 8 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 8 - for (uint32_t sh8 = 0u; sh8 < 27u; ++sh8) { - r7 = r7 + r1 + ((((sel >> 10u) & 1u) != 0u) ? 0x09e8ede2u : 0x432f6c0du); // s128 add - r2 = r2 ^ r4; // s129 xor - r0 = r0 * r2; // s130 mul - r4 = r4 | r3; // s131 or - r3 = r3 | r7; // s132 or - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r3, 1); // s133 shfl - r5 = r5 + r2 + ((((sel >> 11u) & 1u) != 0u) ? 0xce4fdf8eu : 0xcf5ecc75u); // s134 add - r3 = r2 * r3 + r3; // s135 mad - r1 = rotl_imm(r1, 11u); // s136 rotl - r2 = r2 | r0; // s137 or - r3 = r3 ^ r4; // s138 xor - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r4, 1); // s139 shfl - r2 = rotl_imm(r2, 6u); // s140 rotl - r0 = r0 * r3; // s141 mul - r1 = r1 * r7; // s142 mul - r0 = r0 ^ r1; // s143 xor - } - r5 = r5 * r3; // 37 mul - r5 = r5 ^ r4; // 38 xor - r6 = r6 ^ r0; // 39 xor - r4 = rotr_var(r4, r0); // 40 rotr - r7 = r7 ^ r6; // 41 xor - r1 = r1 ^ ds[r7 & mask]; // 42 load - // per-load shadow sub-block 9 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 9 - for (uint32_t sh9 = 0u; sh9 < 27u; ++sh9) { - r6 = r2 * r0 + r6; // s144 mad - r1 = rotl_imm(r1, 15u); // s145 rotl - r1 = rotl_imm(r1, 30u); // s146 rotl - r6 = r6 ^ r0; // s147 xor - r3 = r3 - r2; // s148 sub - r3 = r3 + r0 + ((((sel >> 5u) & 1u) != 0u) ? 0x11bbdeefu : 0xaa002f15u); // s149 add - r6 = r6 + r5 + ((((sel >> 9u) & 1u) != 0u) ? 0xff9d4b0eu : 0xbe64b2e2u); // s150 add - r1 = r1 * r4; // s151 mul - r5 = rotl_imm(r5, 3u); // s152 rotl - r5 = r5 * r2; // s153 mul - r2 = rotl_imm(r2, 21u); // s154 rotl - r5 = rotl_imm(r5, 18u); // s155 rotl - r1 = r1 ^ r5; // s156 xor - r1 = __umulhi(r1, r6); // s157 mulhi - r4 = r7 * r3 + r4; // s158 mad - r4 = r4 - r7; // s159 sub - } - r2 = r2 - r4; // 43 sub - r6 = __umulhi(r6, r7); // 44 mulhi - r3 = rotl_imm(r3, 13u); // 45 rotl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r6, 16); // 46 shfl - r2 = rotl_imm(r2, 26u); // 47 rotl - r6 = r6 * r2; // 48 mul - r2 = r2 ^ ds[r3 & mask]; // 49 load - // per-load shadow sub-block 10 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 10 - for (uint32_t sh10 = 0u; sh10 < 27u; ++sh10) { - r3 = r3 ^ r2; // s160 xor - r3 = __umulhi(r3, r4); // s161 mulhi - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r2, 1); // s162 shfl - r0 = r0 - r1; // s163 sub - r6 = r6 ^ __shfl_xor_sync(0xffffffffu, r0, 8); // s164 shfl - r4 = r4 ^ __shfl_xor_sync(0xffffffffu, r2, 2); // s165 shfl - r3 = r3 | r6; // s166 or - r6 = r6 ^ __shfl_xor_sync(0xffffffffu, r3, 4); // s167 shfl - r2 = r2 ^ r1; // s168 xor - r5 = r5 ^ r1; // s169 xor - r5 = r5 ^ r0; // s170 xor - r3 = r6 * r0 + r3; // s171 mad - r3 = r3 - r0; // s172 sub - r6 = r6 + r0 + ((((sel >> 27u) & 1u) != 0u) ? 0xc72dc2a0u : 0x3b90694bu); // s173 add - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r2, 8); // s174 shfl - r5 = __umulhi(r5, r2); // s175 mulhi - } - r5 = r5 ^ ds[r1 & mask]; // 50 load - // per-load shadow sub-block 11 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 11 - for (uint32_t sh11 = 0u; sh11 < 27u; ++sh11) { - r3 = r3 + r6 + ((((sel >> 5u) & 1u) != 0u) ? 0x4eb75843u : 0x9e65cebdu); // s176 add - r0 = r4 * r1 + r0; // s177 mad - r6 = rotr_var(r6, r7); // s178 rotr - r0 = r0 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0x7001d036u : 0x01e5b250u); // s179 add - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r2, 16); // s180 shfl - r3 = r3 * r0; // s181 mul - r0 = rotl_imm(r0, 23u); // s182 rotl - r7 = __umulhi(r7, r0); // s183 mulhi - r0 = r0 * r4; // s184 mul - r1 = r1 + r4 + ((((sel >> 19u) & 1u) != 0u) ? 0x91736711u : 0xbef14988u); // s185 add - r7 = r7 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x15e0cdf3u : 0xf5d741beu); // s186 add - r1 = r1 ^ r7; // s187 xor - r1 = r1 + r3 + ((((sel >> 21u) & 1u) != 0u) ? 0x7549bc3eu : 0x32be33c6u); // s188 add - r1 = r0 * r6 + r1; // s189 mad - r0 = rotr_var(r0, r5); // s190 rotr - r4 = r0 * r1 + r4; // s191 mad - } - r0 = rotl_imm(r0, 20u); // 51 rotl - r1 = r1 ^ r2; // 52 xor - r2 = r2 ^ ds[r1 & mask]; // 53 load - // per-load shadow sub-block 12 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 12 - for (uint32_t sh12 = 0u; sh12 < 27u; ++sh12) { - r3 = r6 * r1 + r3; // s192 mad - r6 = rotl_imm(r6, 31u); // s193 rotl - r5 = rotl_imm(r5, 28u); // s194 rotl - r4 = r4 * r3; // s195 mul - r2 = r2 + r3 + ((((sel >> 4u) & 1u) != 0u) ? 0x7c3d6253u : 0x9d7aebbbu); // s196 add - r3 = r3 | r5; // s197 or - r6 = r6 - r0; // s198 sub - r7 = r7 * r4; // s199 mul - r3 = rotr_var(r3, r2); // s200 rotr - r4 = r4 ^ __shfl_xor_sync(0xffffffffu, r0, 16); // s201 shfl - r7 = r7 ^ r3; // s202 xor - r4 = r4 | r3; // s203 or - r3 = rotr_var(r3, r6); // s204 rotr - r0 = rotr_var(r0, r1); // s205 rotr - r2 = r5 * r2 + r2; // s206 mad - r1 = __umulhi(r1, r5); // s207 mulhi - } - r4 = rotl_imm(r4, 20u); // 54 rotl - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r1, 2); // 55 shfl - r1 = r1 ^ ds[r0 & mask]; // 56 load - // per-load shadow sub-block 13 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 13 - for (uint32_t sh13 = 0u; sh13 < 27u; ++sh13) { - r4 = r4 ^ __shfl_xor_sync(0xffffffffu, r3, 1); // s208 shfl - r7 = r5 * r7 + r7; // s209 mad - r6 = r6 ^ r3; // s210 xor - r3 = r3 ^ r2; // s211 xor - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r6, 16); // s212 shfl - r4 = rotl_imm(r4, 10u); // s213 rotl - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r6, 16); // s214 shfl - r3 = r3 | r0; // s215 or - r4 = r4 * r5; // s216 mul - r0 = r0 ^ __shfl_xor_sync(0xffffffffu, r3, 16); // s217 shfl - r0 = r0 * r5; // s218 mul - r0 = rotl_imm(r0, 13u); // s219 rotl - r1 = __umulhi(r1, r0); // s220 mulhi - r4 = r4 ^ r1; // s221 xor - r2 = r2 | r3; // s222 or - r0 = r0 ^ r4; // s223 xor - } - r3 = r3 ^ ds[r5 & mask]; // 57 load - // per-load shadow sub-block 14 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 14 - for (uint32_t sh14 = 0u; sh14 < 27u; ++sh14) { - r5 = r6 * r1 + r5; // s224 mad - r4 = r4 ^ r3; // s225 xor - r5 = r5 + r0 + ((((sel >> 3u) & 1u) != 0u) ? 0xe4e51c75u : 0xfe960971u); // s226 add - r3 = __umulhi(r3, r2); // s227 mulhi - r2 = r2 ^ r6; // s228 xor - r1 = __umulhi(r1, r5); // s229 mulhi - r3 = r3 + r4 + ((((sel >> 25u) & 1u) != 0u) ? 0x4d597c08u : 0x48b3ce0au); // s230 add - r2 = rotr_var(r2, r3); // s231 rotr - r2 = r2 - r7; // s232 sub - r6 = r6 | r2; // s233 or - r0 = rotr_var(r0, r3); // s234 rotr - r4 = r4 + r3 + ((((sel >> 13u) & 1u) != 0u) ? 0xc9f1d54cu : 0x2ed8c878u); // s235 add - r0 = r3 * r7 + r0; // s236 mad - r3 = r3 + r5 + ((((sel >> 29u) & 1u) != 0u) ? 0xc572bd00u : 0x22e8b90au); // s237 add - r0 = __umulhi(r0, r4); // s238 mulhi - r6 = r6 * r5; // s239 mul - } - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r2, 2); // 58 shfl - r6 = r6 ^ ds[r4 & mask]; // 59 load - // per-load shadow sub-block 15 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 15 - for (uint32_t sh15 = 0u; sh15 < 27u; ++sh15) { - r5 = r5 + r2 + ((((sel >> 20u) & 1u) != 0u) ? 0xb233f94fu : 0xc6b790e6u); // s240 add - r0 = r0 - r3; // s241 sub - r4 = r4 + r7 + ((((sel >> 4u) & 1u) != 0u) ? 0x534d924bu : 0x0f918d3bu); // s242 add - r5 = rotr_var(r5, r0); // s243 rotr - r1 = rotr_var(r1, r7); // s244 rotr - r2 = r2 | r0; // s245 or - r4 = r4 * r7; // s246 mul - r1 = r2 * r5 + r1; // s247 mad - r2 = r2 * r5; // s248 mul - r4 = r4 ^ r2; // s249 xor - r3 = r3 ^ r1; // s250 xor - r5 = rotr_var(r5, r7); // s251 rotr - r2 = r7 * r7 + r2; // s252 mad - r7 = rotr_var(r7, r3); // s253 rotr - r6 = r6 + r1 + ((((sel >> 21u) & 1u) != 0u) ? 0x86d27169u : 0xb8a27d95u); // s254 add - r6 = r6 | r7; // s255 or - } - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r3, 4); // 60 shfl - r5 = rotr_var(r5, r0); // 61 rotr - r0 = r0 + r6 + ((((sel >> 14u) & 1u) != 0u) ? 0xb13a5391u : 0x8c2e5c24u); // 62 add - r2 = r2 + r1 + ((((sel >> 21u) & 1u) != 0u) ? 0xf82fc8b5u : 0xb225b762u); // 63 add - } - uint32_t lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint32_t hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((uint64_t)hi << 32) | (uint64_t)lo; -} - -// Host-side launch wrappers. Declared in program.h, called from host.cu. -cudaError_t igneum_launch_cache_fill(uint32_t* cache, uint32_t nSegments) { - if (nSegments == 0u) return cudaErrorInvalidValue; - uint32_t block = 256u; - uint32_t grid = (nSegments + block - 1u) / block; - igneum_cache_fill<<>>(cache, nSegments); - return cudaGetLastError(); -} - -cudaError_t igneum_launch_build(uint32_t* ds, const uint32_t* cache, uint32_t nItems) { - if (nItems == 0u) return cudaErrorInvalidValue; - uint32_t block = 256u; - uint32_t grid = (nItems + block - 1u) / block; - igneum_build<<>>(ds, cache, nItems); - return cudaGetLastError(); -} - -cudaError_t igneum_launch_hash(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask, - uint32_t nonces, uint32_t blockWarps) { - if (blockWarps == 0u || blockWarps > 32u) return cudaErrorInvalidValue; - uint32_t block = 32u * blockWarps; - if (nonces == 0u || (nonces % block) != 0u) return cudaErrorInvalidValue; - igneum_hash<<>>(ds, out, baseNonce, mask); - return cudaGetLastError(); -} - -cudaError_t igneum_hash_info(int* numRegs, int* blocksPerSM, uint32_t blockWarps) { - cudaFuncAttributes attr; - cudaError_t e = cudaFuncGetAttributes(&attr, igneum_hash); - if (e != cudaSuccess) return e; - *numRegs = attr.numRegs; - return cudaOccupancyMaxActiveBlocksPerMultiprocessor(blocksPerSM, igneum_hash, (int)(32u * blockWarps), 0); -} diff --git a/proto-cuda/packs-ca4/mx8_shl256x27_v2/kernel_bound.cl b/proto-cuda/packs-ca4/mx8_shl256x27_v2/kernel_bound.cl deleted file mode 100644 index d3808563f..000000000 --- a/proto-cuda/packs-ca4/mx8_shl256x27_v2/kernel_bound.cl +++ /dev/null @@ -1,981 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// OpenCL C twin of the Metal kernel for the same seed (see proto-opencl/README.md, WAVEFRONT.md and program.metal). -// Built from source at runtime by proto-opencl/host.c, which passes these defines: -// IGNEUM_GROUP work-group size of igneum_hash, a multiple of 32 (default 32: one work-group = one 32-lane unit) -// IGNEUM_EXCHANGE 0 = local-memory exchange with a barrier (any device, any wave width; the default) -// 1 = sub_group_shuffle_xor (cl_khr_subgroup_shuffle), only with IGNEUM_GROUP 32 and a sub-group size of exactly 32 -// 2 = intel_sub_group_shuffle_xor (cl_intel_subgroups), same condition -// The verification unit is always 32 lanes. A 64-wide hardware wave (AMD GCN/CDNA, RDNA in wave64) runs two units; -// the exchange masks are 1, 2, 4, 8, 16, so every partner lane lies inside the lane's own aligned run of 32. -#ifndef IGNEUM_GROUP -#define IGNEUM_GROUP 32 -#endif -#ifndef IGNEUM_EXCHANGE -#define IGNEUM_EXCHANGE 0 -#endif -#ifdef __OPENCL_VERSION__ -#define IGNEUM_KERNEL_HASH __kernel __attribute__((reqd_work_group_size(IGNEUM_GROUP, 1, 1))) -#define IGNEUM_LOCAL_WORDS(name, n) __local uint name[n] -#if IGNEUM_EXCHANGE == 1 -#ifdef cl_khr_subgroups -#pragma OPENCL EXTENSION cl_khr_subgroups : enable -#endif -#ifdef cl_khr_subgroup_shuffle -#pragma OPENCL EXTENSION cl_khr_subgroup_shuffle : enable -#endif -#elif IGNEUM_EXCHANGE == 2 -#pragma OPENCL EXTENSION cl_intel_subgroups : enable -#endif -#else -// Not an OpenCL compiler: proto-opencl/emu compiles this file as C++ and supplies the built-ins and these two macros. -#include "emu_opencl.h" -#endif - -#if IGNEUM_EXCHANGE == 1 -#define IGNEUM_SHFL_XOR(dst, a, m) dst = sub_group_shuffle_xor((a), (uint)(m)) -#define IGNEUM_BCAST0(dst, a) dst = sub_group_broadcast((a), 0u) -#elif IGNEUM_EXCHANGE == 2 -#define IGNEUM_SHFL_XOR(dst, a, m) dst = intel_sub_group_shuffle_xor((a), (uint)(m)) -#define IGNEUM_BCAST0(dst, a) dst = sub_group_broadcast((a), 0u) -#else -// Local-memory exchange. Two buffers of IGNEUM_GROUP words alternate (xk counts exchanges), so one barrier per -// exchange is enough: a lane can only overwrite buffer b at exchange k+2 after passing barrier k+1, and every lane -// reaches barrier k+1 only after its read of buffer b at exchange k. The partner lid ^ m stays inside the lane's -// aligned run of 32 because m < 32. Control flow is uniform, so every work-item reaches every barrier. -#define IGNEUM_SHFL_XOR(dst, a, m) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid ^ (uint)(m))]; xk += 1u; } -#define IGNEUM_BCAST0(dst, a) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid & ~31u)]; xk += 1u; } -#endif - -static inline uint splitmix32(uint x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -// n is a literal in 1..31 at every call site. OpenCL rotate() rotates left by n modulo 32. -static inline uint rotl_imm(uint x, uint n) { return rotate(x, n); } -// Right rotation by n modulo 32 as a left rotation by (32 - n) modulo 32; n == 0 gives x. -static inline uint rotr_var(uint x, uint n) { return rotate(x, (0u - n) & 31u); } -static inline uint ds_elem(uint i, uint d0, uint d1) { - uint x = i ^ d0; - x *= 0x9E3779B1u; x ^= x >> 15; - x += d1; - x *= 0x85EBCA77u; x ^= x >> 13; - x *= 0xC2B2AE3Du; x ^= x >> 16; - return x; -} - -// Memory-hard dataset core (MEMHARD.md). Cache: 2^26 words in 2^16 segments of 64 chained ChaCha12 lines. -// Item: 8 rounds of 8 x seed-parameterised mixer + one 64-byte cache read, then 8 x final mixer (class v3, mixer multiplier 8, -// docs/plans/mixer-x4.md: the round key of application j of round r is 0x9E3779B9 * (r * m + j + 1)). All parameters are literals. -#define MH_CACHE_LINE_MASK 0x003fffffu -#define MH_SEGMENT_LINES 64u -#define MH_QR(a, b, c, d, r1, r2, r3, r4) { a += b; d ^= a; d = mh_rotl(d, r1); c += d; b ^= c; b = mh_rotl(b, r2); a += b; d ^= a; d = mh_rotl(d, r3); c += d; b ^= c; b = mh_rotl(b, r4); } -static inline uint mh_rotl(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 at every call site - -// y = ChaCha12 core(x) + x -static inline void mh_chacha_block(const uint* x, uint* y) { - for (uint i = 0u; i < 16u; ++i) y[i] = x[i]; - for (uint r = 0u; r < 6u; ++r) { - MH_QR(y[0], y[4], y[8], y[12], 16u, 12u, 8u, 7u) MH_QR(y[1], y[5], y[9], y[13], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[6], y[10], y[14], 16u, 12u, 8u, 7u) MH_QR(y[3], y[7], y[11], y[15], 16u, 12u, 8u, 7u) - MH_QR(y[0], y[5], y[10], y[15], 16u, 12u, 8u, 7u) MH_QR(y[1], y[6], y[11], y[12], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[7], y[8], y[13], 16u, 12u, 8u, 7u) MH_QR(y[3], y[4], y[9], y[14], 16u, 12u, 8u, 7u) - } - for (uint i = 0u; i < 16u; ++i) y[i] += x[i]; -} - -// One cache segment: 64 chained lines written at cache[seg * 1024]. in_j = prev ^ (sigma || K || seg || j || tag), prev_0 = 0. -static inline void mh_cache_segment(__global uint* cache, uint seg) { - uint prev[16]; uint x[16]; uint y[16]; - for (uint i = 0u; i < 16u; ++i) prev[i] = 0u; - for (uint j = 0u; j < MH_SEGMENT_LINES; ++j) { - x[0] = 0x61707865u ^ prev[0]; x[1] = 0x3320646eu ^ prev[1]; x[2] = 0x79622d32u ^ prev[2]; x[3] = 0x6b206574u ^ prev[3]; - x[4] = 0x3067619fu ^ prev[4]; - x[5] = 0x3c269176u ^ prev[5]; - x[6] = 0x84a03b03u ^ prev[6]; - x[7] = 0xf8c63294u ^ prev[7]; - x[8] = 0xff977c5bu ^ prev[8]; - x[9] = 0xe60def3eu ^ prev[9]; - x[10] = 0x63630141u ^ prev[10]; - x[11] = 0xb8fbcb58u ^ prev[11]; - x[12] = seg ^ prev[12]; x[13] = j ^ prev[13]; x[14] = 0x49676e65u ^ prev[14]; x[15] = 0x756d4d48u ^ prev[15]; - mh_chacha_block(x, y); - __global uint* line = cache + ((seg * MH_SEGMENT_LINES + j) * 16u); - for (uint i = 0u; i < 16u; ++i) { line[i] = y[i]; prev[i] = y[i]; } - } -} - -// M_r: per word (s ^ (RC + rk)) * MUL, then a column round and a diagonal round with the seed-drawn rotations. -static inline void mh_mixer(uint* s, uint rk) { - s[0] = (s[0] ^ (0xbab68293u + rk)) * 0x42146205u; - s[1] = (s[1] ^ (0xcc162340u + rk)) * 0x52cbe0fbu; - s[2] = (s[2] ^ (0x6ce151ccu + rk)) * 0x7ecf4a03u; - s[3] = (s[3] ^ (0xe62b8997u + rk)) * 0x6728907fu; - s[4] = (s[4] ^ (0xc9c80297u + rk)) * 0xd81d9751u; - s[5] = (s[5] ^ (0xf74a1654u + rk)) * 0x132952c3u; - s[6] = (s[6] ^ (0x3d704af5u + rk)) * 0xf60de277u; - s[7] = (s[7] ^ (0x3cf522b7u + rk)) * 0x05358035u; - s[8] = (s[8] ^ (0x2b9cac04u + rk)) * 0xbaf6499du; - s[9] = (s[9] ^ (0xa880ac10u + rk)) * 0xe4db9667u; - s[10] = (s[10] ^ (0x13e5dd1du + rk)) * 0x3e98f45du; - s[11] = (s[11] ^ (0x6fc3e233u + rk)) * 0xd0004eddu; - s[12] = (s[12] ^ (0x2d83eeacu + rk)) * 0x2691630du; - s[13] = (s[13] ^ (0x9006e8bfu + rk)) * 0x9beb3bcfu; - s[14] = (s[14] ^ (0x2c4b5362u + rk)) * 0xab310379u; - s[15] = (s[15] ^ (0x31b49ee2u + rk)) * 0x99cfb423u; - MH_QR(s[0], s[4], s[8], s[12], 20u, 20u, 19u, 4u) MH_QR(s[1], s[5], s[9], s[13], 20u, 20u, 19u, 4u) - MH_QR(s[2], s[6], s[10], s[14], 20u, 20u, 19u, 4u) MH_QR(s[3], s[7], s[11], s[15], 20u, 20u, 19u, 4u) - MH_QR(s[0], s[5], s[10], s[15], 26u, 3u, 3u, 27u) MH_QR(s[1], s[6], s[11], s[12], 26u, 3u, 3u, 27u) - MH_QR(s[2], s[7], s[8], s[13], 26u, 3u, 3u, 27u) MH_QR(s[3], s[4], s[9], s[14], 26u, 3u, 3u, 27u) -} - -// Item t: 16 words. s = (K, t * MUL[i] + RC[i]); 8 rounds of 8 x mixer + cache line s[0] & mask; 8 x final mixer. -static inline void mh_item(__global const uint* cache, uint t, uint* s) { - s[0] = 0x3067619fu; - s[1] = 0x3c269176u; - s[2] = 0x84a03b03u; - s[3] = 0xf8c63294u; - s[4] = 0xff977c5bu; - s[5] = 0xe60def3eu; - s[6] = 0x63630141u; - s[7] = 0xb8fbcb58u; - s[8] = t * 0x42146205u + 0xbab68293u; - s[9] = t * 0x52cbe0fbu + 0xcc162340u; - s[10] = t * 0x7ecf4a03u + 0x6ce151ccu; - s[11] = t * 0x6728907fu + 0xe62b8997u; - s[12] = t * 0xd81d9751u + 0xc9c80297u; - s[13] = t * 0x132952c3u + 0xf74a1654u; - s[14] = t * 0xf60de277u + 0x3d704af5u; - s[15] = t * 0x05358035u + 0x3cf522b7u; - for (uint r = 0u; r < 8u; ++r) { - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (r * 8u + j + 1u)); - __global const uint* line = cache + ((s[0] & MH_CACHE_LINE_MASK) * 16u); - for (uint i = 0u; i < 16u; ++i) s[i] ^= line[i]; - } - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (64u + j + 1u)); -} -// dataset[w] without the dataset: derive item w >> 4 and take word w & 15. -static inline uint mh_word(__global const uint* cache, uint w) { uint s[16]; mh_item(cache, w >> 4u, s); return s[w & 15u]; } - -// Memory-hard dataset (MEMHARD.md). One work-item per cache segment; one work-item per 64-byte dataset item. -// The same constants as memhard.h in this pack (one emitter, three dialects). -__kernel void igneum_cache_fill(__global uint* cache, uint nSegments) { - uint seg = (uint)get_global_id(0); - if (seg < nSegments) mh_cache_segment(cache, seg); -} -__kernel void igneum_build(__global uint* ds, __global const uint* cache, uint nItems) { - uint t = (uint)get_global_id(0); - if (t < nItems) { - uint s[16]; - mh_item(cache, t, s); - __global uint* d = ds + ((ulong)t * 16u); - for (uint i = 0u; i < 16u; ++i) d[i] = s[i]; - } -} - -// One hash per work-item. IGNEUM_GROUP is a multiple of 32; lane = lid & 31 and every exchange stays inside the -// lane's own aligned run of 32 work-items, exactly like simd_shuffle_xor inside a 32-wide Metal SIMD group and -// __shfl_xor_sync inside a CUDA warp. Control flow is uniform (no branches at all). -IGNEUM_KERNEL_HASH void igneum_hash(__global const uint* ds, __global ulong* out, uint baseNonce, uint mask) { - uint gid = (uint)get_global_id(0); - uint lid = (uint)get_local_id(0); - uint nonce = baseNonce + gid; - uint r0, r1, r2, r3, r4, r5, r6, r7; -#if IGNEUM_EXCHANGE == 0 - IGNEUM_LOCAL_WORDS(xch, 2 * IGNEUM_GROUP); - uint xk = 0u; -#else - (void)lid; -#endif - { uint x = nonce ^ 0xf71aee9fu; x += 0x9e3779b9u; x = splitmix32(x); r0 = x ^ 0xad930c88u; } // SEEDW[0], 0x9e3779b9u * 1u, SEEDW[1] - { uint x = nonce ^ 0xad930c88u; x += 0x3c6ef372u; x = splitmix32(x); r1 = x ^ 0x7f982573u; } // SEEDW[1], 0x9e3779b9u * 2u, SEEDW[2] - { uint x = nonce ^ 0x7f982573u; x += 0xdaa66d2bu; x = splitmix32(x); r2 = x ^ 0xa41f9137u; } // SEEDW[2], 0x9e3779b9u * 3u, SEEDW[3] - { uint x = nonce ^ 0xa41f9137u; x += 0x78dde6e4u; x = splitmix32(x); r3 = x ^ 0x76d803d6u; } // SEEDW[3], 0x9e3779b9u * 4u, SEEDW[4] - { uint x = nonce ^ 0x76d803d6u; x += 0x1715609du; x = splitmix32(x); r4 = x ^ 0x37b4a534u; } // SEEDW[4], 0x9e3779b9u * 5u, SEEDW[5] - { uint x = nonce ^ 0x37b4a534u; x += 0xb54cda56u; x = splitmix32(x); r5 = x ^ 0x4d3fb826u; } // SEEDW[5], 0x9e3779b9u * 6u, SEEDW[6] - { uint x = nonce ^ 0x4d3fb826u; x += 0x5384540fu; x = splitmix32(x); r6 = x ^ 0xff614dcbu; } // SEEDW[6], 0x9e3779b9u * 7u, SEEDW[7] - { uint x = nonce ^ 0xff614dcbu; x += 0xf1bbcdc8u; x = splitmix32(x); r7 = x ^ 0xf71aee9fu; } // SEEDW[7], 0x9e3779b9u * 8u, SEEDW[0] - - for (uint it = 0u; it < 8u; ++it) { - uint sel = r0; - r2 = r2 + r5 + ((((sel >> 2u) & 1u) != 0u) ? 0x894e457du : 0xe3e2ed7du); // 0 add - r7 = r4 * r0 + r7; // 1 mad - r3 = r3 - r6; // 2 sub - r5 = rotr_var(r5, r1); // 3 rotr - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 1u); r6 = r6 ^ t_; } // 4 shfl - r0 = r0 | r3; // 5 or - r0 = r0 * r1; // 6 mul - r7 = r7 * r6; // 7 mul - r3 = r3 ^ ds[r0 & mask]; // 8 load - // per-load shadow sub-block 0 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 0 - for (uint sh0 = 0u; sh0 < 27u; ++sh0) { - r5 = rotr_var(r5, r4); // s0 rotr - r6 = r6 - r5; // s1 sub - r7 = r7 - r6; // s2 sub - r5 = r5 * r4; // s3 mul - r7 = r7 ^ r6; // s4 xor - r2 = rotl_imm(r2, 23u); // s5 rotl - r7 = r7 ^ r4; // s6 xor - r0 = r0 + r7 + ((((sel >> 26u) & 1u) != 0u) ? 0x3051c491u : 0xdaef8862u); // s7 add - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 4u); r6 = r6 ^ t_; } // s8 shfl - r5 = r5 + r1 + ((((sel >> 26u) & 1u) != 0u) ? 0x5170d0b3u : 0xc20e7045u); // s9 add - r0 = r0 | r3; // s10 or - r5 = mul_hi(r5, r4); // s11 mulhi - r7 = r7 ^ r0; // s12 xor - r5 = r5 - r2; // s13 sub - r0 = r0 + r6 + ((((sel >> 0u) & 1u) != 0u) ? 0x2ead087fu : 0xc86d98a4u); // s14 add - r5 = r5 ^ r7; // s15 xor - } - r4 = r4 + r3 + ((((sel >> 22u) & 1u) != 0u) ? 0xd26d3573u : 0xce9bea84u); // 9 add - r2 = r2 * r4; // 10 mul - r3 = r3 ^ ds[r2 & mask]; // 11 load - // per-load shadow sub-block 1 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 1 - for (uint sh1 = 0u; sh1 < 27u; ++sh1) { - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 8u); r7 = r7 ^ t_; } // s16 shfl - r5 = r5 - r7; // s17 sub - r0 = r0 | r6; // s18 or - r0 = r0 + r5 + ((((sel >> 13u) & 1u) != 0u) ? 0xc47c70a8u : 0x4248b651u); // s19 add - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 16u); r4 = r4 ^ t_; } // s20 shfl - r0 = r0 + r4 + ((((sel >> 21u) & 1u) != 0u) ? 0x718c1008u : 0x1e35684fu); // s21 add - r0 = mul_hi(r0, r3); // s22 mulhi - r4 = r4 ^ r2; // s23 xor - r0 = r0 - r2; // s24 sub - r5 = rotl_imm(r5, 13u); // s25 rotl - r7 = r7 - r0; // s26 sub - r2 = r2 ^ r3; // s27 xor - r2 = r3 * r2 + r2; // s28 mad - r2 = r2 ^ r1; // s29 xor - r4 = mul_hi(r4, r0); // s30 mulhi - r6 = r6 ^ r4; // s31 xor - } - r4 = mul_hi(r4, r1); // 12 mulhi - r7 = mul_hi(r7, r3); // 13 mulhi - r3 = r3 + r2 + ((((sel >> 8u) & 1u) != 0u) ? 0xc2828a42u : 0x514f9ff4u); // 14 add - r1 = r3 * r2 + r1; // 15 mad - r2 = rotl_imm(r2, 7u); // 16 rotl - r1 = r1 ^ ds[r2 & mask]; // 17 load - // per-load shadow sub-block 2 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 2 - for (uint sh2 = 0u; sh2 < 27u; ++sh2) { - r4 = r4 ^ r2; // s32 xor - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 1u); r1 = r1 ^ t_; } // s33 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 1u); r5 = r5 ^ t_; } // s34 shfl - r4 = r4 - r0; // s35 sub - r6 = r6 - r3; // s36 sub - r2 = r2 | r6; // s37 or - r2 = rotl_imm(r2, 30u); // s38 rotl - r4 = rotr_var(r4, r7); // s39 rotr - r0 = r0 | r7; // s40 or - r2 = rotr_var(r2, r5); // s41 rotr - r1 = r3 * r3 + r1; // s42 mad - r6 = r5 * r5 + r6; // s43 mad - r1 = mul_hi(r1, r0); // s44 mulhi - r1 = r1 * r3; // s45 mul - r0 = r2 * r0 + r0; // s46 mad - r1 = r1 - r5; // s47 sub - } - r3 = r3 * r2; // 18 mul - r6 = rotl_imm(r6, 24u); // 19 rotl - r6 = r6 ^ ds[r3 & mask]; // 20 load - // per-load shadow sub-block 3 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 3 - for (uint sh3 = 0u; sh3 < 27u; ++sh3) { - r4 = r4 ^ r0; // s48 xor - r2 = rotr_var(r2, r0); // s49 rotr - r0 = mul_hi(r0, r5); // s50 mulhi - r7 = r7 | r5; // s51 or - r4 = r0 * r3 + r4; // s52 mad - r0 = rotr_var(r0, r4); // s53 rotr - r6 = r3 * r3 + r6; // s54 mad - r6 = r4 * r2 + r6; // s55 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r0, 2u); r5 = r5 ^ t_; } // s56 shfl - r7 = rotl_imm(r7, 21u); // s57 rotl - r3 = r3 ^ r7; // s58 xor - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 1u); r0 = r0 ^ t_; } // s59 shfl - r5 = r5 ^ r6; // s60 xor - r4 = rotr_var(r4, r7); // s61 rotr - r1 = r1 + r6 + ((((sel >> 10u) & 1u) != 0u) ? 0x31f87d74u : 0xf7ccf1e9u); // s62 add - r5 = r5 + r2 + ((((sel >> 15u) & 1u) != 0u) ? 0x9c2847f4u : 0x2f386099u); // s63 add - } - r1 = r1 ^ ds[r6 & mask]; // 21 load - // per-load shadow sub-block 4 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 4 - for (uint sh4 = 0u; sh4 < 27u; ++sh4) { - r7 = r7 + r0 + ((((sel >> 19u) & 1u) != 0u) ? 0xd3241188u : 0x1b30ce7au); // s64 add - r6 = r6 + r7 + ((((sel >> 13u) & 1u) != 0u) ? 0x9b42c3deu : 0x02dc8349u); // s65 add - r0 = rotl_imm(r0, 16u); // s66 rotl - r2 = rotl_imm(r2, 9u); // s67 rotl - r2 = r5 * r6 + r2; // s68 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 8u); r6 = r6 ^ t_; } // s69 shfl - r2 = r2 ^ r5; // s70 xor - r5 = mul_hi(r5, r1); // s71 mulhi - r6 = rotl_imm(r6, 29u); // s72 rotl - r0 = r0 - r7; // s73 sub - r5 = r2 * r2 + r5; // s74 mad - r0 = rotr_var(r0, r5); // s75 rotr - r7 = r7 + r2 + ((((sel >> 9u) & 1u) != 0u) ? 0x0da1ce17u : 0x05d36679u); // s76 add - r7 = r7 - r2; // s77 sub - r7 = rotr_var(r7, r0); // s78 rotr - r1 = r5 * r5 + r1; // s79 mad - } - r5 = r3 * r3 + r5; // 22 mad - r4 = r4 ^ ds[r7 & mask]; // 23 load - // per-load shadow sub-block 5 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 5 - for (uint sh5 = 0u; sh5 < 27u; ++sh5) { - r3 = rotr_var(r3, r0); // s80 rotr - r6 = r6 + r5 + ((((sel >> 2u) & 1u) != 0u) ? 0xadf5ef88u : 0xf14547bdu); // s81 add - r0 = r0 | r6; // s82 or - r1 = mul_hi(r1, r0); // s83 mulhi - r7 = r6 * r7 + r7; // s84 mad - r5 = rotl_imm(r5, 29u); // s85 rotl - r2 = r2 ^ r6; // s86 xor - r5 = r5 + r4 + ((((sel >> 24u) & 1u) != 0u) ? 0x35ff14aeu : 0xba4947c2u); // s87 add - r3 = r3 + r6 + ((((sel >> 3u) & 1u) != 0u) ? 0xf3900fc1u : 0xf6878beeu); // s88 add - r0 = r0 + r2 + ((((sel >> 12u) & 1u) != 0u) ? 0xf7e8f59fu : 0x926f3607u); // s89 add - r2 = r2 ^ r6; // s90 xor - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 2u); r3 = r3 ^ t_; } // s91 shfl - r6 = r6 - r1; // s92 sub - r5 = r5 + r1 + ((((sel >> 13u) & 1u) != 0u) ? 0x0e376f9cu : 0xdab36c29u); // s93 add - r6 = r1 * r1 + r6; // s94 mad - r3 = r3 ^ r4; // s95 xor - } - r6 = r6 ^ ds[r4 & mask]; // 24 load - // per-load shadow sub-block 6 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 6 - for (uint sh6 = 0u; sh6 < 27u; ++sh6) { - r6 = r5 * r2 + r6; // s96 mad - r7 = r7 + r3 + ((((sel >> 23u) & 1u) != 0u) ? 0x9f917747u : 0x204129e9u); // s97 add - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 16u); r5 = r5 ^ t_; } // s98 shfl - r2 = r2 * r6; // s99 mul - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 8u); r7 = r7 ^ t_; } // s100 shfl - r6 = r6 + r1 + ((((sel >> 10u) & 1u) != 0u) ? 0xe42fe974u : 0xe3b596a0u); // s101 add - r5 = rotl_imm(r5, 3u); // s102 rotl - r5 = r5 ^ r1; // s103 xor - r4 = r4 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0xc3d77ffbu : 0x705c3f94u); // s104 add - r0 = r0 + r6 + ((((sel >> 25u) & 1u) != 0u) ? 0xea02a4cau : 0xbc6fb42bu); // s105 add - r3 = r3 | r5; // s106 or - r0 = r0 - r6; // s107 sub - r0 = rotr_var(r0, r3); // s108 rotr - r5 = r5 + r4 + ((((sel >> 4u) & 1u) != 0u) ? 0x0357e63eu : 0xcb37ea87u); // s109 add - r7 = rotl_imm(r7, 5u); // s110 rotl - r4 = r4 ^ r1; // s111 xor - } - r5 = r5 * r7; // 25 mul - r0 = r0 * r3; // 26 mul - r0 = r0 | r3; // 27 or - r1 = r3 * r4 + r1; // 28 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 8u); r0 = r0 ^ t_; } // 29 shfl - r7 = r7 - r3; // 30 sub - r4 = r4 ^ r1; // 31 xor - r4 = r4 | r5; // 32 or - r3 = rotr_var(r3, r5); // 33 rotr - r4 = r4 ^ ds[r5 & mask]; // 34 load - // per-load shadow sub-block 7 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 7 - for (uint sh7 = 0u; sh7 < 27u; ++sh7) { - r6 = r6 ^ r5; // s112 xor - r1 = r1 | r3; // s113 or - r0 = r0 + r5 + ((((sel >> 3u) & 1u) != 0u) ? 0x7f8b8cd2u : 0x9aab0297u); // s114 add - r3 = r3 - r5; // s115 sub - r4 = rotr_var(r4, r2); // s116 rotr - r4 = r4 + r6 + ((((sel >> 6u) & 1u) != 0u) ? 0x59400b57u : 0xc511183fu); // s117 add - r6 = r6 - r1; // s118 sub - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 4u); r3 = r3 ^ t_; } // s119 shfl - r6 = r4 * r4 + r6; // s120 mad - r2 = rotl_imm(r2, 8u); // s121 rotl - r5 = r0 * r3 + r5; // s122 mad - r7 = r7 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x27c70dc0u : 0x5200c242u); // s123 add - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 4u); r3 = r3 ^ t_; } // s124 shfl - r0 = r0 + r1 + ((((sel >> 17u) & 1u) != 0u) ? 0xec2f5997u : 0x2d5028ceu); // s125 add - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 16u); r3 = r3 ^ t_; } // s126 shfl - r5 = r5 ^ r0; // s127 xor - } - r0 = r5 * r3 + r0; // 35 mad - r6 = r6 ^ ds[r3 & mask]; // 36 load - // per-load shadow sub-block 8 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 8 - for (uint sh8 = 0u; sh8 < 27u; ++sh8) { - r7 = r7 + r1 + ((((sel >> 10u) & 1u) != 0u) ? 0x09e8ede2u : 0x432f6c0du); // s128 add - r2 = r2 ^ r4; // s129 xor - r0 = r0 * r2; // s130 mul - r4 = r4 | r3; // s131 or - r3 = r3 | r7; // s132 or - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 1u); r7 = r7 ^ t_; } // s133 shfl - r5 = r5 + r2 + ((((sel >> 11u) & 1u) != 0u) ? 0xce4fdf8eu : 0xcf5ecc75u); // s134 add - r3 = r2 * r3 + r3; // s135 mad - r1 = rotl_imm(r1, 11u); // s136 rotl - r2 = r2 | r0; // s137 or - r3 = r3 ^ r4; // s138 xor - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 1u); r2 = r2 ^ t_; } // s139 shfl - r2 = rotl_imm(r2, 6u); // s140 rotl - r0 = r0 * r3; // s141 mul - r1 = r1 * r7; // s142 mul - r0 = r0 ^ r1; // s143 xor - } - r5 = r5 * r3; // 37 mul - r5 = r5 ^ r4; // 38 xor - r6 = r6 ^ r0; // 39 xor - r4 = rotr_var(r4, r0); // 40 rotr - r7 = r7 ^ r6; // 41 xor - r1 = r1 ^ ds[r7 & mask]; // 42 load - // per-load shadow sub-block 9 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 9 - for (uint sh9 = 0u; sh9 < 27u; ++sh9) { - r6 = r2 * r0 + r6; // s144 mad - r1 = rotl_imm(r1, 15u); // s145 rotl - r1 = rotl_imm(r1, 30u); // s146 rotl - r6 = r6 ^ r0; // s147 xor - r3 = r3 - r2; // s148 sub - r3 = r3 + r0 + ((((sel >> 5u) & 1u) != 0u) ? 0x11bbdeefu : 0xaa002f15u); // s149 add - r6 = r6 + r5 + ((((sel >> 9u) & 1u) != 0u) ? 0xff9d4b0eu : 0xbe64b2e2u); // s150 add - r1 = r1 * r4; // s151 mul - r5 = rotl_imm(r5, 3u); // s152 rotl - r5 = r5 * r2; // s153 mul - r2 = rotl_imm(r2, 21u); // s154 rotl - r5 = rotl_imm(r5, 18u); // s155 rotl - r1 = r1 ^ r5; // s156 xor - r1 = mul_hi(r1, r6); // s157 mulhi - r4 = r7 * r3 + r4; // s158 mad - r4 = r4 - r7; // s159 sub - } - r2 = r2 - r4; // 43 sub - r6 = mul_hi(r6, r7); // 44 mulhi - r3 = rotl_imm(r3, 13u); // 45 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 16u); r7 = r7 ^ t_; } // 46 shfl - r2 = rotl_imm(r2, 26u); // 47 rotl - r6 = r6 * r2; // 48 mul - r2 = r2 ^ ds[r3 & mask]; // 49 load - // per-load shadow sub-block 10 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 10 - for (uint sh10 = 0u; sh10 < 27u; ++sh10) { - r3 = r3 ^ r2; // s160 xor - r3 = mul_hi(r3, r4); // s161 mulhi - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 1u); r3 = r3 ^ t_; } // s162 shfl - r0 = r0 - r1; // s163 sub - { uint t_; IGNEUM_SHFL_XOR(t_, r0, 8u); r6 = r6 ^ t_; } // s164 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 2u); r4 = r4 ^ t_; } // s165 shfl - r3 = r3 | r6; // s166 or - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 4u); r6 = r6 ^ t_; } // s167 shfl - r2 = r2 ^ r1; // s168 xor - r5 = r5 ^ r1; // s169 xor - r5 = r5 ^ r0; // s170 xor - r3 = r6 * r0 + r3; // s171 mad - r3 = r3 - r0; // s172 sub - r6 = r6 + r0 + ((((sel >> 27u) & 1u) != 0u) ? 0xc72dc2a0u : 0x3b90694bu); // s173 add - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 8u); r7 = r7 ^ t_; } // s174 shfl - r5 = mul_hi(r5, r2); // s175 mulhi - } - r5 = r5 ^ ds[r1 & mask]; // 50 load - // per-load shadow sub-block 11 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 11 - for (uint sh11 = 0u; sh11 < 27u; ++sh11) { - r3 = r3 + r6 + ((((sel >> 5u) & 1u) != 0u) ? 0x4eb75843u : 0x9e65cebdu); // s176 add - r0 = r4 * r1 + r0; // s177 mad - r6 = rotr_var(r6, r7); // s178 rotr - r0 = r0 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0x7001d036u : 0x01e5b250u); // s179 add - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 16u); r5 = r5 ^ t_; } // s180 shfl - r3 = r3 * r0; // s181 mul - r0 = rotl_imm(r0, 23u); // s182 rotl - r7 = mul_hi(r7, r0); // s183 mulhi - r0 = r0 * r4; // s184 mul - r1 = r1 + r4 + ((((sel >> 19u) & 1u) != 0u) ? 0x91736711u : 0xbef14988u); // s185 add - r7 = r7 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x15e0cdf3u : 0xf5d741beu); // s186 add - r1 = r1 ^ r7; // s187 xor - r1 = r1 + r3 + ((((sel >> 21u) & 1u) != 0u) ? 0x7549bc3eu : 0x32be33c6u); // s188 add - r1 = r0 * r6 + r1; // s189 mad - r0 = rotr_var(r0, r5); // s190 rotr - r4 = r0 * r1 + r4; // s191 mad - } - r0 = rotl_imm(r0, 20u); // 51 rotl - r1 = r1 ^ r2; // 52 xor - r2 = r2 ^ ds[r1 & mask]; // 53 load - // per-load shadow sub-block 12 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 12 - for (uint sh12 = 0u; sh12 < 27u; ++sh12) { - r3 = r6 * r1 + r3; // s192 mad - r6 = rotl_imm(r6, 31u); // s193 rotl - r5 = rotl_imm(r5, 28u); // s194 rotl - r4 = r4 * r3; // s195 mul - r2 = r2 + r3 + ((((sel >> 4u) & 1u) != 0u) ? 0x7c3d6253u : 0x9d7aebbbu); // s196 add - r3 = r3 | r5; // s197 or - r6 = r6 - r0; // s198 sub - r7 = r7 * r4; // s199 mul - r3 = rotr_var(r3, r2); // s200 rotr - { uint t_; IGNEUM_SHFL_XOR(t_, r0, 16u); r4 = r4 ^ t_; } // s201 shfl - r7 = r7 ^ r3; // s202 xor - r4 = r4 | r3; // s203 or - r3 = rotr_var(r3, r6); // s204 rotr - r0 = rotr_var(r0, r1); // s205 rotr - r2 = r5 * r2 + r2; // s206 mad - r1 = mul_hi(r1, r5); // s207 mulhi - } - r4 = rotl_imm(r4, 20u); // 54 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 2u); r3 = r3 ^ t_; } // 55 shfl - r1 = r1 ^ ds[r0 & mask]; // 56 load - // per-load shadow sub-block 13 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 13 - for (uint sh13 = 0u; sh13 < 27u; ++sh13) { - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 1u); r4 = r4 ^ t_; } // s208 shfl - r7 = r5 * r7 + r7; // s209 mad - r6 = r6 ^ r3; // s210 xor - r3 = r3 ^ r2; // s211 xor - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 16u); r5 = r5 ^ t_; } // s212 shfl - r4 = rotl_imm(r4, 10u); // s213 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 16u); r5 = r5 ^ t_; } // s214 shfl - r3 = r3 | r0; // s215 or - r4 = r4 * r5; // s216 mul - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 16u); r0 = r0 ^ t_; } // s217 shfl - r0 = r0 * r5; // s218 mul - r0 = rotl_imm(r0, 13u); // s219 rotl - r1 = mul_hi(r1, r0); // s220 mulhi - r4 = r4 ^ r1; // s221 xor - r2 = r2 | r3; // s222 or - r0 = r0 ^ r4; // s223 xor - } - r3 = r3 ^ ds[r5 & mask]; // 57 load - // per-load shadow sub-block 14 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 14 - for (uint sh14 = 0u; sh14 < 27u; ++sh14) { - r5 = r6 * r1 + r5; // s224 mad - r4 = r4 ^ r3; // s225 xor - r5 = r5 + r0 + ((((sel >> 3u) & 1u) != 0u) ? 0xe4e51c75u : 0xfe960971u); // s226 add - r3 = mul_hi(r3, r2); // s227 mulhi - r2 = r2 ^ r6; // s228 xor - r1 = mul_hi(r1, r5); // s229 mulhi - r3 = r3 + r4 + ((((sel >> 25u) & 1u) != 0u) ? 0x4d597c08u : 0x48b3ce0au); // s230 add - r2 = rotr_var(r2, r3); // s231 rotr - r2 = r2 - r7; // s232 sub - r6 = r6 | r2; // s233 or - r0 = rotr_var(r0, r3); // s234 rotr - r4 = r4 + r3 + ((((sel >> 13u) & 1u) != 0u) ? 0xc9f1d54cu : 0x2ed8c878u); // s235 add - r0 = r3 * r7 + r0; // s236 mad - r3 = r3 + r5 + ((((sel >> 29u) & 1u) != 0u) ? 0xc572bd00u : 0x22e8b90au); // s237 add - r0 = mul_hi(r0, r4); // s238 mulhi - r6 = r6 * r5; // s239 mul - } - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 2u); r1 = r1 ^ t_; } // 58 shfl - r6 = r6 ^ ds[r4 & mask]; // 59 load - // per-load shadow sub-block 15 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 15 - for (uint sh15 = 0u; sh15 < 27u; ++sh15) { - r5 = r5 + r2 + ((((sel >> 20u) & 1u) != 0u) ? 0xb233f94fu : 0xc6b790e6u); // s240 add - r0 = r0 - r3; // s241 sub - r4 = r4 + r7 + ((((sel >> 4u) & 1u) != 0u) ? 0x534d924bu : 0x0f918d3bu); // s242 add - r5 = rotr_var(r5, r0); // s243 rotr - r1 = rotr_var(r1, r7); // s244 rotr - r2 = r2 | r0; // s245 or - r4 = r4 * r7; // s246 mul - r1 = r2 * r5 + r1; // s247 mad - r2 = r2 * r5; // s248 mul - r4 = r4 ^ r2; // s249 xor - r3 = r3 ^ r1; // s250 xor - r5 = rotr_var(r5, r7); // s251 rotr - r2 = r7 * r7 + r2; // s252 mad - r7 = rotr_var(r7, r3); // s253 rotr - r6 = r6 + r1 + ((((sel >> 21u) & 1u) != 0u) ? 0x86d27169u : 0xb8a27d95u); // s254 add - r6 = r6 | r7; // s255 or - } - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 4u); r7 = r7 ^ t_; } // 60 shfl - r5 = rotr_var(r5, r0); // 61 rotr - r0 = r0 + r6 + ((((sel >> 14u) & 1u) != 0u) ? 0xb13a5391u : 0x8c2e5c24u); // 62 add - r2 = r2 + r1 + ((((sel >> 21u) & 1u) != 0u) ? 0xf82fc8b5u : 0xb225b762u); // 63 add - } - uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((ulong)hi << 32) | (ulong)lo; -} - -#if IGNEUM_EXCHANGE != 0 -// Reports the sub-group size this device uses for a work-group of IGNEUM_GROUP items. host.c runs it only when the -// per-kernel query (clGetKernelSubGroupInfoKHR on igneum_hash) is unavailable; that query is preferred because a -// compiler may pick a different wave width per kernel (RDNA: wave32 or wave64). See WAVEFRONT.md. -IGNEUM_KERNEL_HASH void igneum_probe_subgroup(__global uint* out) { - if (get_local_id(0) == 0u) { out[0] = get_sub_group_size(); out[1] = get_num_sub_groups(); } -} -#endif - -// Header-bound variant (bind.rs): the init words come from initw, not SEEDW. Same body as igneum_hash. -IGNEUM_KERNEL_HASH void igneum_hash_bound(__global const uint* ds, __global ulong* out, uint baseNonce, uint mask, __global const uint* initw) { - uint gid = (uint)get_global_id(0); - uint lid = (uint)get_local_id(0); - uint nonce = baseNonce + gid; - uint r0, r1, r2, r3, r4, r5, r6, r7; - uint iw0 = initw[0], iw1 = initw[1], iw2 = initw[2], iw3 = initw[3], iw4 = initw[4], iw5 = initw[5], iw6 = initw[6], iw7 = initw[7]; -#if IGNEUM_EXCHANGE == 0 - IGNEUM_LOCAL_WORDS(xch, 2 * IGNEUM_GROUP); - uint xk = 0u; -#else - (void)lid; -#endif - { uint x = nonce ^ iw0; x += 0x9e3779b9u * 1u; x = splitmix32(x); r0 = x ^ iw1; } - { uint x = nonce ^ iw1; x += 0x9e3779b9u * 2u; x = splitmix32(x); r1 = x ^ iw2; } - { uint x = nonce ^ iw2; x += 0x9e3779b9u * 3u; x = splitmix32(x); r2 = x ^ iw3; } - { uint x = nonce ^ iw3; x += 0x9e3779b9u * 4u; x = splitmix32(x); r3 = x ^ iw4; } - { uint x = nonce ^ iw4; x += 0x9e3779b9u * 5u; x = splitmix32(x); r4 = x ^ iw5; } - { uint x = nonce ^ iw5; x += 0x9e3779b9u * 6u; x = splitmix32(x); r5 = x ^ iw6; } - { uint x = nonce ^ iw6; x += 0x9e3779b9u * 7u; x = splitmix32(x); r6 = x ^ iw7; } - { uint x = nonce ^ iw7; x += 0x9e3779b9u * 8u; x = splitmix32(x); r7 = x ^ iw0; } - - for (uint it = 0u; it < 8u; ++it) { - uint sel = r0; - r2 = r2 + r5 + ((((sel >> 2u) & 1u) != 0u) ? 0x894e457du : 0xe3e2ed7du); // 0 add - r7 = r4 * r0 + r7; // 1 mad - r3 = r3 - r6; // 2 sub - r5 = rotr_var(r5, r1); // 3 rotr - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 1u); r6 = r6 ^ t_; } // 4 shfl - r0 = r0 | r3; // 5 or - r0 = r0 * r1; // 6 mul - r7 = r7 * r6; // 7 mul - r3 = r3 ^ ds[r0 & mask]; // 8 load - // per-load shadow sub-block 0 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 0 - for (uint sh0 = 0u; sh0 < 27u; ++sh0) { - r5 = rotr_var(r5, r4); // s0 rotr - r6 = r6 - r5; // s1 sub - r7 = r7 - r6; // s2 sub - r5 = r5 * r4; // s3 mul - r7 = r7 ^ r6; // s4 xor - r2 = rotl_imm(r2, 23u); // s5 rotl - r7 = r7 ^ r4; // s6 xor - r0 = r0 + r7 + ((((sel >> 26u) & 1u) != 0u) ? 0x3051c491u : 0xdaef8862u); // s7 add - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 4u); r6 = r6 ^ t_; } // s8 shfl - r5 = r5 + r1 + ((((sel >> 26u) & 1u) != 0u) ? 0x5170d0b3u : 0xc20e7045u); // s9 add - r0 = r0 | r3; // s10 or - r5 = mul_hi(r5, r4); // s11 mulhi - r7 = r7 ^ r0; // s12 xor - r5 = r5 - r2; // s13 sub - r0 = r0 + r6 + ((((sel >> 0u) & 1u) != 0u) ? 0x2ead087fu : 0xc86d98a4u); // s14 add - r5 = r5 ^ r7; // s15 xor - } - r4 = r4 + r3 + ((((sel >> 22u) & 1u) != 0u) ? 0xd26d3573u : 0xce9bea84u); // 9 add - r2 = r2 * r4; // 10 mul - r3 = r3 ^ ds[r2 & mask]; // 11 load - // per-load shadow sub-block 1 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 1 - for (uint sh1 = 0u; sh1 < 27u; ++sh1) { - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 8u); r7 = r7 ^ t_; } // s16 shfl - r5 = r5 - r7; // s17 sub - r0 = r0 | r6; // s18 or - r0 = r0 + r5 + ((((sel >> 13u) & 1u) != 0u) ? 0xc47c70a8u : 0x4248b651u); // s19 add - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 16u); r4 = r4 ^ t_; } // s20 shfl - r0 = r0 + r4 + ((((sel >> 21u) & 1u) != 0u) ? 0x718c1008u : 0x1e35684fu); // s21 add - r0 = mul_hi(r0, r3); // s22 mulhi - r4 = r4 ^ r2; // s23 xor - r0 = r0 - r2; // s24 sub - r5 = rotl_imm(r5, 13u); // s25 rotl - r7 = r7 - r0; // s26 sub - r2 = r2 ^ r3; // s27 xor - r2 = r3 * r2 + r2; // s28 mad - r2 = r2 ^ r1; // s29 xor - r4 = mul_hi(r4, r0); // s30 mulhi - r6 = r6 ^ r4; // s31 xor - } - r4 = mul_hi(r4, r1); // 12 mulhi - r7 = mul_hi(r7, r3); // 13 mulhi - r3 = r3 + r2 + ((((sel >> 8u) & 1u) != 0u) ? 0xc2828a42u : 0x514f9ff4u); // 14 add - r1 = r3 * r2 + r1; // 15 mad - r2 = rotl_imm(r2, 7u); // 16 rotl - r1 = r1 ^ ds[r2 & mask]; // 17 load - // per-load shadow sub-block 2 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 2 - for (uint sh2 = 0u; sh2 < 27u; ++sh2) { - r4 = r4 ^ r2; // s32 xor - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 1u); r1 = r1 ^ t_; } // s33 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 1u); r5 = r5 ^ t_; } // s34 shfl - r4 = r4 - r0; // s35 sub - r6 = r6 - r3; // s36 sub - r2 = r2 | r6; // s37 or - r2 = rotl_imm(r2, 30u); // s38 rotl - r4 = rotr_var(r4, r7); // s39 rotr - r0 = r0 | r7; // s40 or - r2 = rotr_var(r2, r5); // s41 rotr - r1 = r3 * r3 + r1; // s42 mad - r6 = r5 * r5 + r6; // s43 mad - r1 = mul_hi(r1, r0); // s44 mulhi - r1 = r1 * r3; // s45 mul - r0 = r2 * r0 + r0; // s46 mad - r1 = r1 - r5; // s47 sub - } - r3 = r3 * r2; // 18 mul - r6 = rotl_imm(r6, 24u); // 19 rotl - r6 = r6 ^ ds[r3 & mask]; // 20 load - // per-load shadow sub-block 3 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 3 - for (uint sh3 = 0u; sh3 < 27u; ++sh3) { - r4 = r4 ^ r0; // s48 xor - r2 = rotr_var(r2, r0); // s49 rotr - r0 = mul_hi(r0, r5); // s50 mulhi - r7 = r7 | r5; // s51 or - r4 = r0 * r3 + r4; // s52 mad - r0 = rotr_var(r0, r4); // s53 rotr - r6 = r3 * r3 + r6; // s54 mad - r6 = r4 * r2 + r6; // s55 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r0, 2u); r5 = r5 ^ t_; } // s56 shfl - r7 = rotl_imm(r7, 21u); // s57 rotl - r3 = r3 ^ r7; // s58 xor - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 1u); r0 = r0 ^ t_; } // s59 shfl - r5 = r5 ^ r6; // s60 xor - r4 = rotr_var(r4, r7); // s61 rotr - r1 = r1 + r6 + ((((sel >> 10u) & 1u) != 0u) ? 0x31f87d74u : 0xf7ccf1e9u); // s62 add - r5 = r5 + r2 + ((((sel >> 15u) & 1u) != 0u) ? 0x9c2847f4u : 0x2f386099u); // s63 add - } - r1 = r1 ^ ds[r6 & mask]; // 21 load - // per-load shadow sub-block 4 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 4 - for (uint sh4 = 0u; sh4 < 27u; ++sh4) { - r7 = r7 + r0 + ((((sel >> 19u) & 1u) != 0u) ? 0xd3241188u : 0x1b30ce7au); // s64 add - r6 = r6 + r7 + ((((sel >> 13u) & 1u) != 0u) ? 0x9b42c3deu : 0x02dc8349u); // s65 add - r0 = rotl_imm(r0, 16u); // s66 rotl - r2 = rotl_imm(r2, 9u); // s67 rotl - r2 = r5 * r6 + r2; // s68 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 8u); r6 = r6 ^ t_; } // s69 shfl - r2 = r2 ^ r5; // s70 xor - r5 = mul_hi(r5, r1); // s71 mulhi - r6 = rotl_imm(r6, 29u); // s72 rotl - r0 = r0 - r7; // s73 sub - r5 = r2 * r2 + r5; // s74 mad - r0 = rotr_var(r0, r5); // s75 rotr - r7 = r7 + r2 + ((((sel >> 9u) & 1u) != 0u) ? 0x0da1ce17u : 0x05d36679u); // s76 add - r7 = r7 - r2; // s77 sub - r7 = rotr_var(r7, r0); // s78 rotr - r1 = r5 * r5 + r1; // s79 mad - } - r5 = r3 * r3 + r5; // 22 mad - r4 = r4 ^ ds[r7 & mask]; // 23 load - // per-load shadow sub-block 5 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 5 - for (uint sh5 = 0u; sh5 < 27u; ++sh5) { - r3 = rotr_var(r3, r0); // s80 rotr - r6 = r6 + r5 + ((((sel >> 2u) & 1u) != 0u) ? 0xadf5ef88u : 0xf14547bdu); // s81 add - r0 = r0 | r6; // s82 or - r1 = mul_hi(r1, r0); // s83 mulhi - r7 = r6 * r7 + r7; // s84 mad - r5 = rotl_imm(r5, 29u); // s85 rotl - r2 = r2 ^ r6; // s86 xor - r5 = r5 + r4 + ((((sel >> 24u) & 1u) != 0u) ? 0x35ff14aeu : 0xba4947c2u); // s87 add - r3 = r3 + r6 + ((((sel >> 3u) & 1u) != 0u) ? 0xf3900fc1u : 0xf6878beeu); // s88 add - r0 = r0 + r2 + ((((sel >> 12u) & 1u) != 0u) ? 0xf7e8f59fu : 0x926f3607u); // s89 add - r2 = r2 ^ r6; // s90 xor - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 2u); r3 = r3 ^ t_; } // s91 shfl - r6 = r6 - r1; // s92 sub - r5 = r5 + r1 + ((((sel >> 13u) & 1u) != 0u) ? 0x0e376f9cu : 0xdab36c29u); // s93 add - r6 = r1 * r1 + r6; // s94 mad - r3 = r3 ^ r4; // s95 xor - } - r6 = r6 ^ ds[r4 & mask]; // 24 load - // per-load shadow sub-block 6 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 6 - for (uint sh6 = 0u; sh6 < 27u; ++sh6) { - r6 = r5 * r2 + r6; // s96 mad - r7 = r7 + r3 + ((((sel >> 23u) & 1u) != 0u) ? 0x9f917747u : 0x204129e9u); // s97 add - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 16u); r5 = r5 ^ t_; } // s98 shfl - r2 = r2 * r6; // s99 mul - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 8u); r7 = r7 ^ t_; } // s100 shfl - r6 = r6 + r1 + ((((sel >> 10u) & 1u) != 0u) ? 0xe42fe974u : 0xe3b596a0u); // s101 add - r5 = rotl_imm(r5, 3u); // s102 rotl - r5 = r5 ^ r1; // s103 xor - r4 = r4 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0xc3d77ffbu : 0x705c3f94u); // s104 add - r0 = r0 + r6 + ((((sel >> 25u) & 1u) != 0u) ? 0xea02a4cau : 0xbc6fb42bu); // s105 add - r3 = r3 | r5; // s106 or - r0 = r0 - r6; // s107 sub - r0 = rotr_var(r0, r3); // s108 rotr - r5 = r5 + r4 + ((((sel >> 4u) & 1u) != 0u) ? 0x0357e63eu : 0xcb37ea87u); // s109 add - r7 = rotl_imm(r7, 5u); // s110 rotl - r4 = r4 ^ r1; // s111 xor - } - r5 = r5 * r7; // 25 mul - r0 = r0 * r3; // 26 mul - r0 = r0 | r3; // 27 or - r1 = r3 * r4 + r1; // 28 mad - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 8u); r0 = r0 ^ t_; } // 29 shfl - r7 = r7 - r3; // 30 sub - r4 = r4 ^ r1; // 31 xor - r4 = r4 | r5; // 32 or - r3 = rotr_var(r3, r5); // 33 rotr - r4 = r4 ^ ds[r5 & mask]; // 34 load - // per-load shadow sub-block 7 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 7 - for (uint sh7 = 0u; sh7 < 27u; ++sh7) { - r6 = r6 ^ r5; // s112 xor - r1 = r1 | r3; // s113 or - r0 = r0 + r5 + ((((sel >> 3u) & 1u) != 0u) ? 0x7f8b8cd2u : 0x9aab0297u); // s114 add - r3 = r3 - r5; // s115 sub - r4 = rotr_var(r4, r2); // s116 rotr - r4 = r4 + r6 + ((((sel >> 6u) & 1u) != 0u) ? 0x59400b57u : 0xc511183fu); // s117 add - r6 = r6 - r1; // s118 sub - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 4u); r3 = r3 ^ t_; } // s119 shfl - r6 = r4 * r4 + r6; // s120 mad - r2 = rotl_imm(r2, 8u); // s121 rotl - r5 = r0 * r3 + r5; // s122 mad - r7 = r7 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x27c70dc0u : 0x5200c242u); // s123 add - { uint t_; IGNEUM_SHFL_XOR(t_, r5, 4u); r3 = r3 ^ t_; } // s124 shfl - r0 = r0 + r1 + ((((sel >> 17u) & 1u) != 0u) ? 0xec2f5997u : 0x2d5028ceu); // s125 add - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 16u); r3 = r3 ^ t_; } // s126 shfl - r5 = r5 ^ r0; // s127 xor - } - r0 = r5 * r3 + r0; // 35 mad - r6 = r6 ^ ds[r3 & mask]; // 36 load - // per-load shadow sub-block 8 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 8 - for (uint sh8 = 0u; sh8 < 27u; ++sh8) { - r7 = r7 + r1 + ((((sel >> 10u) & 1u) != 0u) ? 0x09e8ede2u : 0x432f6c0du); // s128 add - r2 = r2 ^ r4; // s129 xor - r0 = r0 * r2; // s130 mul - r4 = r4 | r3; // s131 or - r3 = r3 | r7; // s132 or - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 1u); r7 = r7 ^ t_; } // s133 shfl - r5 = r5 + r2 + ((((sel >> 11u) & 1u) != 0u) ? 0xce4fdf8eu : 0xcf5ecc75u); // s134 add - r3 = r2 * r3 + r3; // s135 mad - r1 = rotl_imm(r1, 11u); // s136 rotl - r2 = r2 | r0; // s137 or - r3 = r3 ^ r4; // s138 xor - { uint t_; IGNEUM_SHFL_XOR(t_, r4, 1u); r2 = r2 ^ t_; } // s139 shfl - r2 = rotl_imm(r2, 6u); // s140 rotl - r0 = r0 * r3; // s141 mul - r1 = r1 * r7; // s142 mul - r0 = r0 ^ r1; // s143 xor - } - r5 = r5 * r3; // 37 mul - r5 = r5 ^ r4; // 38 xor - r6 = r6 ^ r0; // 39 xor - r4 = rotr_var(r4, r0); // 40 rotr - r7 = r7 ^ r6; // 41 xor - r1 = r1 ^ ds[r7 & mask]; // 42 load - // per-load shadow sub-block 9 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 9 - for (uint sh9 = 0u; sh9 < 27u; ++sh9) { - r6 = r2 * r0 + r6; // s144 mad - r1 = rotl_imm(r1, 15u); // s145 rotl - r1 = rotl_imm(r1, 30u); // s146 rotl - r6 = r6 ^ r0; // s147 xor - r3 = r3 - r2; // s148 sub - r3 = r3 + r0 + ((((sel >> 5u) & 1u) != 0u) ? 0x11bbdeefu : 0xaa002f15u); // s149 add - r6 = r6 + r5 + ((((sel >> 9u) & 1u) != 0u) ? 0xff9d4b0eu : 0xbe64b2e2u); // s150 add - r1 = r1 * r4; // s151 mul - r5 = rotl_imm(r5, 3u); // s152 rotl - r5 = r5 * r2; // s153 mul - r2 = rotl_imm(r2, 21u); // s154 rotl - r5 = rotl_imm(r5, 18u); // s155 rotl - r1 = r1 ^ r5; // s156 xor - r1 = mul_hi(r1, r6); // s157 mulhi - r4 = r7 * r3 + r4; // s158 mad - r4 = r4 - r7; // s159 sub - } - r2 = r2 - r4; // 43 sub - r6 = mul_hi(r6, r7); // 44 mulhi - r3 = rotl_imm(r3, 13u); // 45 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 16u); r7 = r7 ^ t_; } // 46 shfl - r2 = rotl_imm(r2, 26u); // 47 rotl - r6 = r6 * r2; // 48 mul - r2 = r2 ^ ds[r3 & mask]; // 49 load - // per-load shadow sub-block 10 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 10 - for (uint sh10 = 0u; sh10 < 27u; ++sh10) { - r3 = r3 ^ r2; // s160 xor - r3 = mul_hi(r3, r4); // s161 mulhi - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 1u); r3 = r3 ^ t_; } // s162 shfl - r0 = r0 - r1; // s163 sub - { uint t_; IGNEUM_SHFL_XOR(t_, r0, 8u); r6 = r6 ^ t_; } // s164 shfl - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 2u); r4 = r4 ^ t_; } // s165 shfl - r3 = r3 | r6; // s166 or - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 4u); r6 = r6 ^ t_; } // s167 shfl - r2 = r2 ^ r1; // s168 xor - r5 = r5 ^ r1; // s169 xor - r5 = r5 ^ r0; // s170 xor - r3 = r6 * r0 + r3; // s171 mad - r3 = r3 - r0; // s172 sub - r6 = r6 + r0 + ((((sel >> 27u) & 1u) != 0u) ? 0xc72dc2a0u : 0x3b90694bu); // s173 add - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 8u); r7 = r7 ^ t_; } // s174 shfl - r5 = mul_hi(r5, r2); // s175 mulhi - } - r5 = r5 ^ ds[r1 & mask]; // 50 load - // per-load shadow sub-block 11 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 11 - for (uint sh11 = 0u; sh11 < 27u; ++sh11) { - r3 = r3 + r6 + ((((sel >> 5u) & 1u) != 0u) ? 0x4eb75843u : 0x9e65cebdu); // s176 add - r0 = r4 * r1 + r0; // s177 mad - r6 = rotr_var(r6, r7); // s178 rotr - r0 = r0 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0x7001d036u : 0x01e5b250u); // s179 add - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 16u); r5 = r5 ^ t_; } // s180 shfl - r3 = r3 * r0; // s181 mul - r0 = rotl_imm(r0, 23u); // s182 rotl - r7 = mul_hi(r7, r0); // s183 mulhi - r0 = r0 * r4; // s184 mul - r1 = r1 + r4 + ((((sel >> 19u) & 1u) != 0u) ? 0x91736711u : 0xbef14988u); // s185 add - r7 = r7 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x15e0cdf3u : 0xf5d741beu); // s186 add - r1 = r1 ^ r7; // s187 xor - r1 = r1 + r3 + ((((sel >> 21u) & 1u) != 0u) ? 0x7549bc3eu : 0x32be33c6u); // s188 add - r1 = r0 * r6 + r1; // s189 mad - r0 = rotr_var(r0, r5); // s190 rotr - r4 = r0 * r1 + r4; // s191 mad - } - r0 = rotl_imm(r0, 20u); // 51 rotl - r1 = r1 ^ r2; // 52 xor - r2 = r2 ^ ds[r1 & mask]; // 53 load - // per-load shadow sub-block 12 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 12 - for (uint sh12 = 0u; sh12 < 27u; ++sh12) { - r3 = r6 * r1 + r3; // s192 mad - r6 = rotl_imm(r6, 31u); // s193 rotl - r5 = rotl_imm(r5, 28u); // s194 rotl - r4 = r4 * r3; // s195 mul - r2 = r2 + r3 + ((((sel >> 4u) & 1u) != 0u) ? 0x7c3d6253u : 0x9d7aebbbu); // s196 add - r3 = r3 | r5; // s197 or - r6 = r6 - r0; // s198 sub - r7 = r7 * r4; // s199 mul - r3 = rotr_var(r3, r2); // s200 rotr - { uint t_; IGNEUM_SHFL_XOR(t_, r0, 16u); r4 = r4 ^ t_; } // s201 shfl - r7 = r7 ^ r3; // s202 xor - r4 = r4 | r3; // s203 or - r3 = rotr_var(r3, r6); // s204 rotr - r0 = rotr_var(r0, r1); // s205 rotr - r2 = r5 * r2 + r2; // s206 mad - r1 = mul_hi(r1, r5); // s207 mulhi - } - r4 = rotl_imm(r4, 20u); // 54 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r1, 2u); r3 = r3 ^ t_; } // 55 shfl - r1 = r1 ^ ds[r0 & mask]; // 56 load - // per-load shadow sub-block 13 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 13 - for (uint sh13 = 0u; sh13 < 27u; ++sh13) { - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 1u); r4 = r4 ^ t_; } // s208 shfl - r7 = r5 * r7 + r7; // s209 mad - r6 = r6 ^ r3; // s210 xor - r3 = r3 ^ r2; // s211 xor - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 16u); r5 = r5 ^ t_; } // s212 shfl - r4 = rotl_imm(r4, 10u); // s213 rotl - { uint t_; IGNEUM_SHFL_XOR(t_, r6, 16u); r5 = r5 ^ t_; } // s214 shfl - r3 = r3 | r0; // s215 or - r4 = r4 * r5; // s216 mul - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 16u); r0 = r0 ^ t_; } // s217 shfl - r0 = r0 * r5; // s218 mul - r0 = rotl_imm(r0, 13u); // s219 rotl - r1 = mul_hi(r1, r0); // s220 mulhi - r4 = r4 ^ r1; // s221 xor - r2 = r2 | r3; // s222 or - r0 = r0 ^ r4; // s223 xor - } - r3 = r3 ^ ds[r5 & mask]; // 57 load - // per-load shadow sub-block 14 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 14 - for (uint sh14 = 0u; sh14 < 27u; ++sh14) { - r5 = r6 * r1 + r5; // s224 mad - r4 = r4 ^ r3; // s225 xor - r5 = r5 + r0 + ((((sel >> 3u) & 1u) != 0u) ? 0xe4e51c75u : 0xfe960971u); // s226 add - r3 = mul_hi(r3, r2); // s227 mulhi - r2 = r2 ^ r6; // s228 xor - r1 = mul_hi(r1, r5); // s229 mulhi - r3 = r3 + r4 + ((((sel >> 25u) & 1u) != 0u) ? 0x4d597c08u : 0x48b3ce0au); // s230 add - r2 = rotr_var(r2, r3); // s231 rotr - r2 = r2 - r7; // s232 sub - r6 = r6 | r2; // s233 or - r0 = rotr_var(r0, r3); // s234 rotr - r4 = r4 + r3 + ((((sel >> 13u) & 1u) != 0u) ? 0xc9f1d54cu : 0x2ed8c878u); // s235 add - r0 = r3 * r7 + r0; // s236 mad - r3 = r3 + r5 + ((((sel >> 29u) & 1u) != 0u) ? 0xc572bd00u : 0x22e8b90au); // s237 add - r0 = mul_hi(r0, r4); // s238 mulhi - r6 = r6 * r5; // s239 mul - } - { uint t_; IGNEUM_SHFL_XOR(t_, r2, 2u); r1 = r1 ^ t_; } // 58 shfl - r6 = r6 ^ ds[r4 & mask]; // 59 load - // per-load shadow sub-block 15 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 15 - for (uint sh15 = 0u; sh15 < 27u; ++sh15) { - r5 = r5 + r2 + ((((sel >> 20u) & 1u) != 0u) ? 0xb233f94fu : 0xc6b790e6u); // s240 add - r0 = r0 - r3; // s241 sub - r4 = r4 + r7 + ((((sel >> 4u) & 1u) != 0u) ? 0x534d924bu : 0x0f918d3bu); // s242 add - r5 = rotr_var(r5, r0); // s243 rotr - r1 = rotr_var(r1, r7); // s244 rotr - r2 = r2 | r0; // s245 or - r4 = r4 * r7; // s246 mul - r1 = r2 * r5 + r1; // s247 mad - r2 = r2 * r5; // s248 mul - r4 = r4 ^ r2; // s249 xor - r3 = r3 ^ r1; // s250 xor - r5 = rotr_var(r5, r7); // s251 rotr - r2 = r7 * r7 + r2; // s252 mad - r7 = rotr_var(r7, r3); // s253 rotr - r6 = r6 + r1 + ((((sel >> 21u) & 1u) != 0u) ? 0x86d27169u : 0xb8a27d95u); // s254 add - r6 = r6 | r7; // s255 or - } - { uint t_; IGNEUM_SHFL_XOR(t_, r3, 4u); r7 = r7 ^ t_; } // 60 shfl - r5 = rotr_var(r5, r0); // 61 rotr - r0 = r0 + r6 + ((((sel >> 14u) & 1u) != 0u) ? 0xb13a5391u : 0x8c2e5c24u); // 62 add - r2 = r2 + r1 + ((((sel >> 21u) & 1u) != 0u) ? 0xf82fc8b5u : 0xb225b762u); // 63 add - } - uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((ulong)hi << 32) | (ulong)lo; -} diff --git a/proto-cuda/packs-ca4/mx8_shl256x27_v2/kernel_bound.cu b/proto-cuda/packs-ca4/mx8_shl256x27_v2/kernel_bound.cu deleted file mode 100644 index 667f26d04..000000000 --- a/proto-cuda/packs-ca4/mx8_shl256x27_v2/kernel_bound.cu +++ /dev/null @@ -1,427 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// Header-bound twin of igneum_hash in kernel.cu: the init words come from a kernel argument, not SEEDW. -// Host declarations (also in program_bound.h if present): -// struct IgneumInitWords { uint32_t w[8]; }; -// cudaError_t igneum_launch_hash_bound(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask, -// IgneumInitWords iw, uint32_t nonces, uint32_t blockWarps); -// cudaError_t igneum_hash_bound_info(int* numRegs, int* blocksPerSM, uint32_t blockWarps); -#include -#include -#include "program.h" - -struct IgneumInitWords { uint32_t w[8]; }; - -__device__ __forceinline__ uint32_t splitmix32(uint32_t x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -__device__ __forceinline__ uint32_t rotl_imm(uint32_t x, uint32_t n) { return (x << n) | (x >> (32u - n)); } -__device__ __forceinline__ uint32_t rotr_var(uint32_t x, uint32_t n) { n &= 31u; return (x >> n) | (x << ((32u - n) & 31u)); } - -__global__ void igneum_hash_bound(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask, IgneumInitWords iw) { - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t nonce = baseNonce + gid; - uint32_t r0, r1, r2, r3, r4, r5, r6, r7; - { uint32_t x = nonce ^ iw.w[0]; x += 0x9e3779b9u * 1u; x = splitmix32(x); r0 = x ^ iw.w[1]; } - { uint32_t x = nonce ^ iw.w[1]; x += 0x9e3779b9u * 2u; x = splitmix32(x); r1 = x ^ iw.w[2]; } - { uint32_t x = nonce ^ iw.w[2]; x += 0x9e3779b9u * 3u; x = splitmix32(x); r2 = x ^ iw.w[3]; } - { uint32_t x = nonce ^ iw.w[3]; x += 0x9e3779b9u * 4u; x = splitmix32(x); r3 = x ^ iw.w[4]; } - { uint32_t x = nonce ^ iw.w[4]; x += 0x9e3779b9u * 5u; x = splitmix32(x); r4 = x ^ iw.w[5]; } - { uint32_t x = nonce ^ iw.w[5]; x += 0x9e3779b9u * 6u; x = splitmix32(x); r5 = x ^ iw.w[6]; } - { uint32_t x = nonce ^ iw.w[6]; x += 0x9e3779b9u * 7u; x = splitmix32(x); r6 = x ^ iw.w[7]; } - { uint32_t x = nonce ^ iw.w[7]; x += 0x9e3779b9u * 8u; x = splitmix32(x); r7 = x ^ iw.w[0]; } - - for (uint32_t it = 0u; it < 8u; ++it) { - uint32_t sel = r0; - r2 = r2 + r5 + ((((sel >> 2u) & 1u) != 0u) ? 0x894e457du : 0xe3e2ed7du); // 0 add - r7 = r4 * r0 + r7; // 1 mad - r3 = r3 - r6; // 2 sub - r5 = rotr_var(r5, r1); // 3 rotr - r6 = r6 ^ __shfl_xor_sync(0xffffffffu, r2, 1); // 4 shfl - r0 = r0 | r3; // 5 or - r0 = r0 * r1; // 6 mul - r7 = r7 * r6; // 7 mul - r3 = r3 ^ ds[r0 & mask]; // 8 load - // per-load shadow sub-block 0 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 0 - for (uint32_t sh0 = 0u; sh0 < 27u; ++sh0) { - r5 = rotr_var(r5, r4); // s0 rotr - r6 = r6 - r5; // s1 sub - r7 = r7 - r6; // s2 sub - r5 = r5 * r4; // s3 mul - r7 = r7 ^ r6; // s4 xor - r2 = rotl_imm(r2, 23u); // s5 rotl - r7 = r7 ^ r4; // s6 xor - r0 = r0 + r7 + ((((sel >> 26u) & 1u) != 0u) ? 0x3051c491u : 0xdaef8862u); // s7 add - r6 = r6 ^ __shfl_xor_sync(0xffffffffu, r1, 4); // s8 shfl - r5 = r5 + r1 + ((((sel >> 26u) & 1u) != 0u) ? 0x5170d0b3u : 0xc20e7045u); // s9 add - r0 = r0 | r3; // s10 or - r5 = __umulhi(r5, r4); // s11 mulhi - r7 = r7 ^ r0; // s12 xor - r5 = r5 - r2; // s13 sub - r0 = r0 + r6 + ((((sel >> 0u) & 1u) != 0u) ? 0x2ead087fu : 0xc86d98a4u); // s14 add - r5 = r5 ^ r7; // s15 xor - } - r4 = r4 + r3 + ((((sel >> 22u) & 1u) != 0u) ? 0xd26d3573u : 0xce9bea84u); // 9 add - r2 = r2 * r4; // 10 mul - r3 = r3 ^ ds[r2 & mask]; // 11 load - // per-load shadow sub-block 1 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 1 - for (uint32_t sh1 = 0u; sh1 < 27u; ++sh1) { - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r6, 8); // s16 shfl - r5 = r5 - r7; // s17 sub - r0 = r0 | r6; // s18 or - r0 = r0 + r5 + ((((sel >> 13u) & 1u) != 0u) ? 0xc47c70a8u : 0x4248b651u); // s19 add - r4 = r4 ^ __shfl_xor_sync(0xffffffffu, r5, 16); // s20 shfl - r0 = r0 + r4 + ((((sel >> 21u) & 1u) != 0u) ? 0x718c1008u : 0x1e35684fu); // s21 add - r0 = __umulhi(r0, r3); // s22 mulhi - r4 = r4 ^ r2; // s23 xor - r0 = r0 - r2; // s24 sub - r5 = rotl_imm(r5, 13u); // s25 rotl - r7 = r7 - r0; // s26 sub - r2 = r2 ^ r3; // s27 xor - r2 = r3 * r2 + r2; // s28 mad - r2 = r2 ^ r1; // s29 xor - r4 = __umulhi(r4, r0); // s30 mulhi - r6 = r6 ^ r4; // s31 xor - } - r4 = __umulhi(r4, r1); // 12 mulhi - r7 = __umulhi(r7, r3); // 13 mulhi - r3 = r3 + r2 + ((((sel >> 8u) & 1u) != 0u) ? 0xc2828a42u : 0x514f9ff4u); // 14 add - r1 = r3 * r2 + r1; // 15 mad - r2 = rotl_imm(r2, 7u); // 16 rotl - r1 = r1 ^ ds[r2 & mask]; // 17 load - // per-load shadow sub-block 2 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 2 - for (uint32_t sh2 = 0u; sh2 < 27u; ++sh2) { - r4 = r4 ^ r2; // s32 xor - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r2, 1); // s33 shfl - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r6, 1); // s34 shfl - r4 = r4 - r0; // s35 sub - r6 = r6 - r3; // s36 sub - r2 = r2 | r6; // s37 or - r2 = rotl_imm(r2, 30u); // s38 rotl - r4 = rotr_var(r4, r7); // s39 rotr - r0 = r0 | r7; // s40 or - r2 = rotr_var(r2, r5); // s41 rotr - r1 = r3 * r3 + r1; // s42 mad - r6 = r5 * r5 + r6; // s43 mad - r1 = __umulhi(r1, r0); // s44 mulhi - r1 = r1 * r3; // s45 mul - r0 = r2 * r0 + r0; // s46 mad - r1 = r1 - r5; // s47 sub - } - r3 = r3 * r2; // 18 mul - r6 = rotl_imm(r6, 24u); // 19 rotl - r6 = r6 ^ ds[r3 & mask]; // 20 load - // per-load shadow sub-block 3 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 3 - for (uint32_t sh3 = 0u; sh3 < 27u; ++sh3) { - r4 = r4 ^ r0; // s48 xor - r2 = rotr_var(r2, r0); // s49 rotr - r0 = __umulhi(r0, r5); // s50 mulhi - r7 = r7 | r5; // s51 or - r4 = r0 * r3 + r4; // s52 mad - r0 = rotr_var(r0, r4); // s53 rotr - r6 = r3 * r3 + r6; // s54 mad - r6 = r4 * r2 + r6; // s55 mad - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r0, 2); // s56 shfl - r7 = rotl_imm(r7, 21u); // s57 rotl - r3 = r3 ^ r7; // s58 xor - r0 = r0 ^ __shfl_xor_sync(0xffffffffu, r4, 1); // s59 shfl - r5 = r5 ^ r6; // s60 xor - r4 = rotr_var(r4, r7); // s61 rotr - r1 = r1 + r6 + ((((sel >> 10u) & 1u) != 0u) ? 0x31f87d74u : 0xf7ccf1e9u); // s62 add - r5 = r5 + r2 + ((((sel >> 15u) & 1u) != 0u) ? 0x9c2847f4u : 0x2f386099u); // s63 add - } - r1 = r1 ^ ds[r6 & mask]; // 21 load - // per-load shadow sub-block 4 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 4 - for (uint32_t sh4 = 0u; sh4 < 27u; ++sh4) { - r7 = r7 + r0 + ((((sel >> 19u) & 1u) != 0u) ? 0xd3241188u : 0x1b30ce7au); // s64 add - r6 = r6 + r7 + ((((sel >> 13u) & 1u) != 0u) ? 0x9b42c3deu : 0x02dc8349u); // s65 add - r0 = rotl_imm(r0, 16u); // s66 rotl - r2 = rotl_imm(r2, 9u); // s67 rotl - r2 = r5 * r6 + r2; // s68 mad - r6 = r6 ^ __shfl_xor_sync(0xffffffffu, r1, 8); // s69 shfl - r2 = r2 ^ r5; // s70 xor - r5 = __umulhi(r5, r1); // s71 mulhi - r6 = rotl_imm(r6, 29u); // s72 rotl - r0 = r0 - r7; // s73 sub - r5 = r2 * r2 + r5; // s74 mad - r0 = rotr_var(r0, r5); // s75 rotr - r7 = r7 + r2 + ((((sel >> 9u) & 1u) != 0u) ? 0x0da1ce17u : 0x05d36679u); // s76 add - r7 = r7 - r2; // s77 sub - r7 = rotr_var(r7, r0); // s78 rotr - r1 = r5 * r5 + r1; // s79 mad - } - r5 = r3 * r3 + r5; // 22 mad - r4 = r4 ^ ds[r7 & mask]; // 23 load - // per-load shadow sub-block 5 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 5 - for (uint32_t sh5 = 0u; sh5 < 27u; ++sh5) { - r3 = rotr_var(r3, r0); // s80 rotr - r6 = r6 + r5 + ((((sel >> 2u) & 1u) != 0u) ? 0xadf5ef88u : 0xf14547bdu); // s81 add - r0 = r0 | r6; // s82 or - r1 = __umulhi(r1, r0); // s83 mulhi - r7 = r6 * r7 + r7; // s84 mad - r5 = rotl_imm(r5, 29u); // s85 rotl - r2 = r2 ^ r6; // s86 xor - r5 = r5 + r4 + ((((sel >> 24u) & 1u) != 0u) ? 0x35ff14aeu : 0xba4947c2u); // s87 add - r3 = r3 + r6 + ((((sel >> 3u) & 1u) != 0u) ? 0xf3900fc1u : 0xf6878beeu); // s88 add - r0 = r0 + r2 + ((((sel >> 12u) & 1u) != 0u) ? 0xf7e8f59fu : 0x926f3607u); // s89 add - r2 = r2 ^ r6; // s90 xor - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r2, 2); // s91 shfl - r6 = r6 - r1; // s92 sub - r5 = r5 + r1 + ((((sel >> 13u) & 1u) != 0u) ? 0x0e376f9cu : 0xdab36c29u); // s93 add - r6 = r1 * r1 + r6; // s94 mad - r3 = r3 ^ r4; // s95 xor - } - r6 = r6 ^ ds[r4 & mask]; // 24 load - // per-load shadow sub-block 6 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 6 - for (uint32_t sh6 = 0u; sh6 < 27u; ++sh6) { - r6 = r5 * r2 + r6; // s96 mad - r7 = r7 + r3 + ((((sel >> 23u) & 1u) != 0u) ? 0x9f917747u : 0x204129e9u); // s97 add - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r1, 16); // s98 shfl - r2 = r2 * r6; // s99 mul - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r3, 8); // s100 shfl - r6 = r6 + r1 + ((((sel >> 10u) & 1u) != 0u) ? 0xe42fe974u : 0xe3b596a0u); // s101 add - r5 = rotl_imm(r5, 3u); // s102 rotl - r5 = r5 ^ r1; // s103 xor - r4 = r4 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0xc3d77ffbu : 0x705c3f94u); // s104 add - r0 = r0 + r6 + ((((sel >> 25u) & 1u) != 0u) ? 0xea02a4cau : 0xbc6fb42bu); // s105 add - r3 = r3 | r5; // s106 or - r0 = r0 - r6; // s107 sub - r0 = rotr_var(r0, r3); // s108 rotr - r5 = r5 + r4 + ((((sel >> 4u) & 1u) != 0u) ? 0x0357e63eu : 0xcb37ea87u); // s109 add - r7 = rotl_imm(r7, 5u); // s110 rotl - r4 = r4 ^ r1; // s111 xor - } - r5 = r5 * r7; // 25 mul - r0 = r0 * r3; // 26 mul - r0 = r0 | r3; // 27 or - r1 = r3 * r4 + r1; // 28 mad - r0 = r0 ^ __shfl_xor_sync(0xffffffffu, r2, 8); // 29 shfl - r7 = r7 - r3; // 30 sub - r4 = r4 ^ r1; // 31 xor - r4 = r4 | r5; // 32 or - r3 = rotr_var(r3, r5); // 33 rotr - r4 = r4 ^ ds[r5 & mask]; // 34 load - // per-load shadow sub-block 7 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 7 - for (uint32_t sh7 = 0u; sh7 < 27u; ++sh7) { - r6 = r6 ^ r5; // s112 xor - r1 = r1 | r3; // s113 or - r0 = r0 + r5 + ((((sel >> 3u) & 1u) != 0u) ? 0x7f8b8cd2u : 0x9aab0297u); // s114 add - r3 = r3 - r5; // s115 sub - r4 = rotr_var(r4, r2); // s116 rotr - r4 = r4 + r6 + ((((sel >> 6u) & 1u) != 0u) ? 0x59400b57u : 0xc511183fu); // s117 add - r6 = r6 - r1; // s118 sub - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r1, 4); // s119 shfl - r6 = r4 * r4 + r6; // s120 mad - r2 = rotl_imm(r2, 8u); // s121 rotl - r5 = r0 * r3 + r5; // s122 mad - r7 = r7 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x27c70dc0u : 0x5200c242u); // s123 add - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r5, 4); // s124 shfl - r0 = r0 + r1 + ((((sel >> 17u) & 1u) != 0u) ? 0xec2f5997u : 0x2d5028ceu); // s125 add - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r6, 16); // s126 shfl - r5 = r5 ^ r0; // s127 xor - } - r0 = r5 * r3 + r0; // 35 mad - r6 = r6 ^ ds[r3 & mask]; // 36 load - // per-load shadow sub-block 8 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 8 - for (uint32_t sh8 = 0u; sh8 < 27u; ++sh8) { - r7 = r7 + r1 + ((((sel >> 10u) & 1u) != 0u) ? 0x09e8ede2u : 0x432f6c0du); // s128 add - r2 = r2 ^ r4; // s129 xor - r0 = r0 * r2; // s130 mul - r4 = r4 | r3; // s131 or - r3 = r3 | r7; // s132 or - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r3, 1); // s133 shfl - r5 = r5 + r2 + ((((sel >> 11u) & 1u) != 0u) ? 0xce4fdf8eu : 0xcf5ecc75u); // s134 add - r3 = r2 * r3 + r3; // s135 mad - r1 = rotl_imm(r1, 11u); // s136 rotl - r2 = r2 | r0; // s137 or - r3 = r3 ^ r4; // s138 xor - r2 = r2 ^ __shfl_xor_sync(0xffffffffu, r4, 1); // s139 shfl - r2 = rotl_imm(r2, 6u); // s140 rotl - r0 = r0 * r3; // s141 mul - r1 = r1 * r7; // s142 mul - r0 = r0 ^ r1; // s143 xor - } - r5 = r5 * r3; // 37 mul - r5 = r5 ^ r4; // 38 xor - r6 = r6 ^ r0; // 39 xor - r4 = rotr_var(r4, r0); // 40 rotr - r7 = r7 ^ r6; // 41 xor - r1 = r1 ^ ds[r7 & mask]; // 42 load - // per-load shadow sub-block 9 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 9 - for (uint32_t sh9 = 0u; sh9 < 27u; ++sh9) { - r6 = r2 * r0 + r6; // s144 mad - r1 = rotl_imm(r1, 15u); // s145 rotl - r1 = rotl_imm(r1, 30u); // s146 rotl - r6 = r6 ^ r0; // s147 xor - r3 = r3 - r2; // s148 sub - r3 = r3 + r0 + ((((sel >> 5u) & 1u) != 0u) ? 0x11bbdeefu : 0xaa002f15u); // s149 add - r6 = r6 + r5 + ((((sel >> 9u) & 1u) != 0u) ? 0xff9d4b0eu : 0xbe64b2e2u); // s150 add - r1 = r1 * r4; // s151 mul - r5 = rotl_imm(r5, 3u); // s152 rotl - r5 = r5 * r2; // s153 mul - r2 = rotl_imm(r2, 21u); // s154 rotl - r5 = rotl_imm(r5, 18u); // s155 rotl - r1 = r1 ^ r5; // s156 xor - r1 = __umulhi(r1, r6); // s157 mulhi - r4 = r7 * r3 + r4; // s158 mad - r4 = r4 - r7; // s159 sub - } - r2 = r2 - r4; // 43 sub - r6 = __umulhi(r6, r7); // 44 mulhi - r3 = rotl_imm(r3, 13u); // 45 rotl - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r6, 16); // 46 shfl - r2 = rotl_imm(r2, 26u); // 47 rotl - r6 = r6 * r2; // 48 mul - r2 = r2 ^ ds[r3 & mask]; // 49 load - // per-load shadow sub-block 10 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 10 - for (uint32_t sh10 = 0u; sh10 < 27u; ++sh10) { - r3 = r3 ^ r2; // s160 xor - r3 = __umulhi(r3, r4); // s161 mulhi - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r2, 1); // s162 shfl - r0 = r0 - r1; // s163 sub - r6 = r6 ^ __shfl_xor_sync(0xffffffffu, r0, 8); // s164 shfl - r4 = r4 ^ __shfl_xor_sync(0xffffffffu, r2, 2); // s165 shfl - r3 = r3 | r6; // s166 or - r6 = r6 ^ __shfl_xor_sync(0xffffffffu, r3, 4); // s167 shfl - r2 = r2 ^ r1; // s168 xor - r5 = r5 ^ r1; // s169 xor - r5 = r5 ^ r0; // s170 xor - r3 = r6 * r0 + r3; // s171 mad - r3 = r3 - r0; // s172 sub - r6 = r6 + r0 + ((((sel >> 27u) & 1u) != 0u) ? 0xc72dc2a0u : 0x3b90694bu); // s173 add - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r2, 8); // s174 shfl - r5 = __umulhi(r5, r2); // s175 mulhi - } - r5 = r5 ^ ds[r1 & mask]; // 50 load - // per-load shadow sub-block 11 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 11 - for (uint32_t sh11 = 0u; sh11 < 27u; ++sh11) { - r3 = r3 + r6 + ((((sel >> 5u) & 1u) != 0u) ? 0x4eb75843u : 0x9e65cebdu); // s176 add - r0 = r4 * r1 + r0; // s177 mad - r6 = rotr_var(r6, r7); // s178 rotr - r0 = r0 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0x7001d036u : 0x01e5b250u); // s179 add - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r2, 16); // s180 shfl - r3 = r3 * r0; // s181 mul - r0 = rotl_imm(r0, 23u); // s182 rotl - r7 = __umulhi(r7, r0); // s183 mulhi - r0 = r0 * r4; // s184 mul - r1 = r1 + r4 + ((((sel >> 19u) & 1u) != 0u) ? 0x91736711u : 0xbef14988u); // s185 add - r7 = r7 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x15e0cdf3u : 0xf5d741beu); // s186 add - r1 = r1 ^ r7; // s187 xor - r1 = r1 + r3 + ((((sel >> 21u) & 1u) != 0u) ? 0x7549bc3eu : 0x32be33c6u); // s188 add - r1 = r0 * r6 + r1; // s189 mad - r0 = rotr_var(r0, r5); // s190 rotr - r4 = r0 * r1 + r4; // s191 mad - } - r0 = rotl_imm(r0, 20u); // 51 rotl - r1 = r1 ^ r2; // 52 xor - r2 = r2 ^ ds[r1 & mask]; // 53 load - // per-load shadow sub-block 12 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 12 - for (uint32_t sh12 = 0u; sh12 < 27u; ++sh12) { - r3 = r6 * r1 + r3; // s192 mad - r6 = rotl_imm(r6, 31u); // s193 rotl - r5 = rotl_imm(r5, 28u); // s194 rotl - r4 = r4 * r3; // s195 mul - r2 = r2 + r3 + ((((sel >> 4u) & 1u) != 0u) ? 0x7c3d6253u : 0x9d7aebbbu); // s196 add - r3 = r3 | r5; // s197 or - r6 = r6 - r0; // s198 sub - r7 = r7 * r4; // s199 mul - r3 = rotr_var(r3, r2); // s200 rotr - r4 = r4 ^ __shfl_xor_sync(0xffffffffu, r0, 16); // s201 shfl - r7 = r7 ^ r3; // s202 xor - r4 = r4 | r3; // s203 or - r3 = rotr_var(r3, r6); // s204 rotr - r0 = rotr_var(r0, r1); // s205 rotr - r2 = r5 * r2 + r2; // s206 mad - r1 = __umulhi(r1, r5); // s207 mulhi - } - r4 = rotl_imm(r4, 20u); // 54 rotl - r3 = r3 ^ __shfl_xor_sync(0xffffffffu, r1, 2); // 55 shfl - r1 = r1 ^ ds[r0 & mask]; // 56 load - // per-load shadow sub-block 13 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 13 - for (uint32_t sh13 = 0u; sh13 < 27u; ++sh13) { - r4 = r4 ^ __shfl_xor_sync(0xffffffffu, r3, 1); // s208 shfl - r7 = r5 * r7 + r7; // s209 mad - r6 = r6 ^ r3; // s210 xor - r3 = r3 ^ r2; // s211 xor - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r6, 16); // s212 shfl - r4 = rotl_imm(r4, 10u); // s213 rotl - r5 = r5 ^ __shfl_xor_sync(0xffffffffu, r6, 16); // s214 shfl - r3 = r3 | r0; // s215 or - r4 = r4 * r5; // s216 mul - r0 = r0 ^ __shfl_xor_sync(0xffffffffu, r3, 16); // s217 shfl - r0 = r0 * r5; // s218 mul - r0 = rotl_imm(r0, 13u); // s219 rotl - r1 = __umulhi(r1, r0); // s220 mulhi - r4 = r4 ^ r1; // s221 xor - r2 = r2 | r3; // s222 or - r0 = r0 ^ r4; // s223 xor - } - r3 = r3 ^ ds[r5 & mask]; // 57 load - // per-load shadow sub-block 14 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 14 - for (uint32_t sh14 = 0u; sh14 < 27u; ++sh14) { - r5 = r6 * r1 + r5; // s224 mad - r4 = r4 ^ r3; // s225 xor - r5 = r5 + r0 + ((((sel >> 3u) & 1u) != 0u) ? 0xe4e51c75u : 0xfe960971u); // s226 add - r3 = __umulhi(r3, r2); // s227 mulhi - r2 = r2 ^ r6; // s228 xor - r1 = __umulhi(r1, r5); // s229 mulhi - r3 = r3 + r4 + ((((sel >> 25u) & 1u) != 0u) ? 0x4d597c08u : 0x48b3ce0au); // s230 add - r2 = rotr_var(r2, r3); // s231 rotr - r2 = r2 - r7; // s232 sub - r6 = r6 | r2; // s233 or - r0 = rotr_var(r0, r3); // s234 rotr - r4 = r4 + r3 + ((((sel >> 13u) & 1u) != 0u) ? 0xc9f1d54cu : 0x2ed8c878u); // s235 add - r0 = r3 * r7 + r0; // s236 mad - r3 = r3 + r5 + ((((sel >> 29u) & 1u) != 0u) ? 0xc572bd00u : 0x22e8b90au); // s237 add - r0 = __umulhi(r0, r4); // s238 mulhi - r6 = r6 * r5; // s239 mul - } - r1 = r1 ^ __shfl_xor_sync(0xffffffffu, r2, 2); // 58 shfl - r6 = r6 ^ ds[r4 & mask]; // 59 load - // per-load shadow sub-block 15 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 15 - for (uint32_t sh15 = 0u; sh15 < 27u; ++sh15) { - r5 = r5 + r2 + ((((sel >> 20u) & 1u) != 0u) ? 0xb233f94fu : 0xc6b790e6u); // s240 add - r0 = r0 - r3; // s241 sub - r4 = r4 + r7 + ((((sel >> 4u) & 1u) != 0u) ? 0x534d924bu : 0x0f918d3bu); // s242 add - r5 = rotr_var(r5, r0); // s243 rotr - r1 = rotr_var(r1, r7); // s244 rotr - r2 = r2 | r0; // s245 or - r4 = r4 * r7; // s246 mul - r1 = r2 * r5 + r1; // s247 mad - r2 = r2 * r5; // s248 mul - r4 = r4 ^ r2; // s249 xor - r3 = r3 ^ r1; // s250 xor - r5 = rotr_var(r5, r7); // s251 rotr - r2 = r7 * r7 + r2; // s252 mad - r7 = rotr_var(r7, r3); // s253 rotr - r6 = r6 + r1 + ((((sel >> 21u) & 1u) != 0u) ? 0x86d27169u : 0xb8a27d95u); // s254 add - r6 = r6 | r7; // s255 or - } - r7 = r7 ^ __shfl_xor_sync(0xffffffffu, r3, 4); // 60 shfl - r5 = rotr_var(r5, r0); // 61 rotr - r0 = r0 + r6 + ((((sel >> 14u) & 1u) != 0u) ? 0xb13a5391u : 0x8c2e5c24u); // 62 add - r2 = r2 + r1 + ((((sel >> 21u) & 1u) != 0u) ? 0xf82fc8b5u : 0xb225b762u); // 63 add - } - uint32_t lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint32_t hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((uint64_t)hi << 32) | (uint64_t)lo; -} - -cudaError_t igneum_launch_hash_bound(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask, - IgneumInitWords iw, uint32_t nonces, uint32_t blockWarps) { - if (blockWarps == 0u || blockWarps > 32u) return cudaErrorInvalidValue; - uint32_t block = 32u * blockWarps; - if (nonces == 0u || (nonces % block) != 0u) return cudaErrorInvalidValue; - igneum_hash_bound<<>>(ds, out, baseNonce, mask, iw); - return cudaGetLastError(); -} - -cudaError_t igneum_hash_bound_info(int* numRegs, int* blocksPerSM, uint32_t blockWarps) { - cudaFuncAttributes attr; - cudaError_t e = cudaFuncGetAttributes(&attr, igneum_hash_bound); - if (e != cudaSuccess) return e; - *numRegs = attr.numRegs; - return cudaOccupancyMaxActiveBlocksPerMultiprocessor(blocksPerSM, igneum_hash_bound, (int)(32u * blockWarps), 0); -} diff --git a/proto-cuda/packs-ca4/mx8_shl256x27_v2/memhard.h b/proto-cuda/packs-ca4/mx8_shl256x27_v2/memhard.h deleted file mode 100644 index f7f34c732..000000000 --- a/proto-cuda/packs-ca4/mx8_shl256x27_v2/memhard.h +++ /dev/null @@ -1,109 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// Memory-hard dataset core, the same text that the Mac's Metal kernels and CPU verifier were checked against. -// Included by kernel.cu (device), host.cu (host reference) and proto-opencl/host.c (C99 host reference). -// See proto-metal/MEMHARD.md for the construction. kernel.cl carries the same text in OpenCL C. -#pragma once -#ifdef __cplusplus -#include -#else -#include -#endif -#if defined(__CUDACC__) -#define IGNEUM_HD __host__ __device__ __forceinline__ -#elif defined(_MSC_VER) && !defined(__cplusplus) -#define IGNEUM_HD static __inline -#else -#define IGNEUM_HD static inline -#endif -// Memory-hard dataset core (MEMHARD.md). Cache: 2^26 words in 2^16 segments of 64 chained ChaCha12 lines. -// Item: 8 rounds of 8 x seed-parameterised mixer + one 64-byte cache read, then 8 x final mixer (class v3, mixer multiplier 8, -// docs/plans/mixer-x4.md: the round key of application j of round r is 0x9E3779B9 * (r * m + j + 1)). All parameters are literals. -#define MH_CACHE_LINE_MASK 0x003fffffu -#define MH_SEGMENT_LINES 64u -#define MH_QR(a, b, c, d, r1, r2, r3, r4) { a += b; d ^= a; d = mh_rotl(d, r1); c += d; b ^= c; b = mh_rotl(b, r2); a += b; d ^= a; d = mh_rotl(d, r3); c += d; b ^= c; b = mh_rotl(b, r4); } -IGNEUM_HD uint32_t mh_rotl(uint32_t x, uint32_t n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 at every call site - -// y = ChaCha12 core(x) + x -IGNEUM_HD void mh_chacha_block(const uint32_t* x, uint32_t* y) { - for (uint32_t i = 0u; i < 16u; ++i) y[i] = x[i]; - for (uint32_t r = 0u; r < 6u; ++r) { - MH_QR(y[0], y[4], y[8], y[12], 16u, 12u, 8u, 7u) MH_QR(y[1], y[5], y[9], y[13], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[6], y[10], y[14], 16u, 12u, 8u, 7u) MH_QR(y[3], y[7], y[11], y[15], 16u, 12u, 8u, 7u) - MH_QR(y[0], y[5], y[10], y[15], 16u, 12u, 8u, 7u) MH_QR(y[1], y[6], y[11], y[12], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[7], y[8], y[13], 16u, 12u, 8u, 7u) MH_QR(y[3], y[4], y[9], y[14], 16u, 12u, 8u, 7u) - } - for (uint32_t i = 0u; i < 16u; ++i) y[i] += x[i]; -} - -// One cache segment: 64 chained lines written at cache[seg * 1024]. in_j = prev ^ (sigma || K || seg || j || tag), prev_0 = 0. -IGNEUM_HD void mh_cache_segment(uint32_t* cache, uint32_t seg) { - uint32_t prev[16]; uint32_t x[16]; uint32_t y[16]; - for (uint32_t i = 0u; i < 16u; ++i) prev[i] = 0u; - for (uint32_t j = 0u; j < MH_SEGMENT_LINES; ++j) { - x[0] = 0x61707865u ^ prev[0]; x[1] = 0x3320646eu ^ prev[1]; x[2] = 0x79622d32u ^ prev[2]; x[3] = 0x6b206574u ^ prev[3]; - x[4] = 0x3067619fu ^ prev[4]; - x[5] = 0x3c269176u ^ prev[5]; - x[6] = 0x84a03b03u ^ prev[6]; - x[7] = 0xf8c63294u ^ prev[7]; - x[8] = 0xff977c5bu ^ prev[8]; - x[9] = 0xe60def3eu ^ prev[9]; - x[10] = 0x63630141u ^ prev[10]; - x[11] = 0xb8fbcb58u ^ prev[11]; - x[12] = seg ^ prev[12]; x[13] = j ^ prev[13]; x[14] = 0x49676e65u ^ prev[14]; x[15] = 0x756d4d48u ^ prev[15]; - mh_chacha_block(x, y); - uint32_t* line = cache + ((seg * MH_SEGMENT_LINES + j) * 16u); - for (uint32_t i = 0u; i < 16u; ++i) { line[i] = y[i]; prev[i] = y[i]; } - } -} - -// M_r: per word (s ^ (RC + rk)) * MUL, then a column round and a diagonal round with the seed-drawn rotations. -IGNEUM_HD void mh_mixer(uint32_t* s, uint32_t rk) { - s[0] = (s[0] ^ (0xbab68293u + rk)) * 0x42146205u; - s[1] = (s[1] ^ (0xcc162340u + rk)) * 0x52cbe0fbu; - s[2] = (s[2] ^ (0x6ce151ccu + rk)) * 0x7ecf4a03u; - s[3] = (s[3] ^ (0xe62b8997u + rk)) * 0x6728907fu; - s[4] = (s[4] ^ (0xc9c80297u + rk)) * 0xd81d9751u; - s[5] = (s[5] ^ (0xf74a1654u + rk)) * 0x132952c3u; - s[6] = (s[6] ^ (0x3d704af5u + rk)) * 0xf60de277u; - s[7] = (s[7] ^ (0x3cf522b7u + rk)) * 0x05358035u; - s[8] = (s[8] ^ (0x2b9cac04u + rk)) * 0xbaf6499du; - s[9] = (s[9] ^ (0xa880ac10u + rk)) * 0xe4db9667u; - s[10] = (s[10] ^ (0x13e5dd1du + rk)) * 0x3e98f45du; - s[11] = (s[11] ^ (0x6fc3e233u + rk)) * 0xd0004eddu; - s[12] = (s[12] ^ (0x2d83eeacu + rk)) * 0x2691630du; - s[13] = (s[13] ^ (0x9006e8bfu + rk)) * 0x9beb3bcfu; - s[14] = (s[14] ^ (0x2c4b5362u + rk)) * 0xab310379u; - s[15] = (s[15] ^ (0x31b49ee2u + rk)) * 0x99cfb423u; - MH_QR(s[0], s[4], s[8], s[12], 20u, 20u, 19u, 4u) MH_QR(s[1], s[5], s[9], s[13], 20u, 20u, 19u, 4u) - MH_QR(s[2], s[6], s[10], s[14], 20u, 20u, 19u, 4u) MH_QR(s[3], s[7], s[11], s[15], 20u, 20u, 19u, 4u) - MH_QR(s[0], s[5], s[10], s[15], 26u, 3u, 3u, 27u) MH_QR(s[1], s[6], s[11], s[12], 26u, 3u, 3u, 27u) - MH_QR(s[2], s[7], s[8], s[13], 26u, 3u, 3u, 27u) MH_QR(s[3], s[4], s[9], s[14], 26u, 3u, 3u, 27u) -} - -// Item t: 16 words. s = (K, t * MUL[i] + RC[i]); 8 rounds of 8 x mixer + cache line s[0] & mask; 8 x final mixer. -IGNEUM_HD void mh_item(const uint32_t* cache, uint32_t t, uint32_t* s) { - s[0] = 0x3067619fu; - s[1] = 0x3c269176u; - s[2] = 0x84a03b03u; - s[3] = 0xf8c63294u; - s[4] = 0xff977c5bu; - s[5] = 0xe60def3eu; - s[6] = 0x63630141u; - s[7] = 0xb8fbcb58u; - s[8] = t * 0x42146205u + 0xbab68293u; - s[9] = t * 0x52cbe0fbu + 0xcc162340u; - s[10] = t * 0x7ecf4a03u + 0x6ce151ccu; - s[11] = t * 0x6728907fu + 0xe62b8997u; - s[12] = t * 0xd81d9751u + 0xc9c80297u; - s[13] = t * 0x132952c3u + 0xf74a1654u; - s[14] = t * 0xf60de277u + 0x3d704af5u; - s[15] = t * 0x05358035u + 0x3cf522b7u; - for (uint32_t r = 0u; r < 8u; ++r) { - for (uint32_t j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (r * 8u + j + 1u)); - const uint32_t* line = cache + ((s[0] & MH_CACHE_LINE_MASK) * 16u); - for (uint32_t i = 0u; i < 16u; ++i) s[i] ^= line[i]; - } - for (uint32_t j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (64u + j + 1u)); -} -// dataset[w] without the dataset: derive item w >> 4 and take word w & 15. -IGNEUM_HD uint32_t mh_word(const uint32_t* cache, uint32_t w) { uint32_t s[16]; mh_item(cache, w >> 4u, s); return s[w & 15u]; } diff --git a/proto-cuda/packs-ca4/mx8_shl256x27_v2/memhard.metal b/proto-cuda/packs-ca4/mx8_shl256x27_v2/memhard.metal deleted file mode 100644 index 01b263d6d..000000000 --- a/proto-cuda/packs-ca4/mx8_shl256x27_v2/memhard.metal +++ /dev/null @@ -1,107 +0,0 @@ -#include -using namespace metal; -// Memory-hard dataset core (MEMHARD.md). Cache: 2^26 words in 2^16 segments of 64 chained ChaCha12 lines. -// Item: 8 rounds of 8 x seed-parameterised mixer + one 64-byte cache read, then 8 x final mixer (class v3, mixer multiplier 8, -// docs/plans/mixer-x4.md: the round key of application j of round r is 0x9E3779B9 * (r * m + j + 1)). All parameters are literals. -#define MH_CACHE_LINE_MASK 0x003fffffu -#define MH_SEGMENT_LINES 64u -#define MH_QR(a, b, c, d, r1, r2, r3, r4) { a += b; d ^= a; d = mh_rotl(d, r1); c += d; b ^= c; b = mh_rotl(b, r2); a += b; d ^= a; d = mh_rotl(d, r3); c += d; b ^= c; b = mh_rotl(b, r4); } -inline uint mh_rotl(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 at every call site - -// y = ChaCha12 core(x) + x -inline void mh_chacha_block(const thread uint* x, thread uint* y) { - for (uint i = 0u; i < 16u; ++i) y[i] = x[i]; - for (uint r = 0u; r < 6u; ++r) { - MH_QR(y[0], y[4], y[8], y[12], 16u, 12u, 8u, 7u) MH_QR(y[1], y[5], y[9], y[13], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[6], y[10], y[14], 16u, 12u, 8u, 7u) MH_QR(y[3], y[7], y[11], y[15], 16u, 12u, 8u, 7u) - MH_QR(y[0], y[5], y[10], y[15], 16u, 12u, 8u, 7u) MH_QR(y[1], y[6], y[11], y[12], 16u, 12u, 8u, 7u) - MH_QR(y[2], y[7], y[8], y[13], 16u, 12u, 8u, 7u) MH_QR(y[3], y[4], y[9], y[14], 16u, 12u, 8u, 7u) - } - for (uint i = 0u; i < 16u; ++i) y[i] += x[i]; -} - -// One cache segment: 64 chained lines written at cache[seg * 1024]. in_j = prev ^ (sigma || K || seg || j || tag), prev_0 = 0. -inline void mh_cache_segment(device uint* cache, uint seg) { - uint prev[16]; uint x[16]; uint y[16]; - for (uint i = 0u; i < 16u; ++i) prev[i] = 0u; - for (uint j = 0u; j < MH_SEGMENT_LINES; ++j) { - x[0] = 0x61707865u ^ prev[0]; x[1] = 0x3320646eu ^ prev[1]; x[2] = 0x79622d32u ^ prev[2]; x[3] = 0x6b206574u ^ prev[3]; - x[4] = 0x3067619fu ^ prev[4]; - x[5] = 0x3c269176u ^ prev[5]; - x[6] = 0x84a03b03u ^ prev[6]; - x[7] = 0xf8c63294u ^ prev[7]; - x[8] = 0xff977c5bu ^ prev[8]; - x[9] = 0xe60def3eu ^ prev[9]; - x[10] = 0x63630141u ^ prev[10]; - x[11] = 0xb8fbcb58u ^ prev[11]; - x[12] = seg ^ prev[12]; x[13] = j ^ prev[13]; x[14] = 0x49676e65u ^ prev[14]; x[15] = 0x756d4d48u ^ prev[15]; - mh_chacha_block(x, y); - device uint* line = cache + ((seg * MH_SEGMENT_LINES + j) * 16u); - for (uint i = 0u; i < 16u; ++i) { line[i] = y[i]; prev[i] = y[i]; } - } -} - -// M_r: per word (s ^ (RC + rk)) * MUL, then a column round and a diagonal round with the seed-drawn rotations. -inline void mh_mixer(thread uint* s, uint rk) { - s[0] = (s[0] ^ (0xbab68293u + rk)) * 0x42146205u; - s[1] = (s[1] ^ (0xcc162340u + rk)) * 0x52cbe0fbu; - s[2] = (s[2] ^ (0x6ce151ccu + rk)) * 0x7ecf4a03u; - s[3] = (s[3] ^ (0xe62b8997u + rk)) * 0x6728907fu; - s[4] = (s[4] ^ (0xc9c80297u + rk)) * 0xd81d9751u; - s[5] = (s[5] ^ (0xf74a1654u + rk)) * 0x132952c3u; - s[6] = (s[6] ^ (0x3d704af5u + rk)) * 0xf60de277u; - s[7] = (s[7] ^ (0x3cf522b7u + rk)) * 0x05358035u; - s[8] = (s[8] ^ (0x2b9cac04u + rk)) * 0xbaf6499du; - s[9] = (s[9] ^ (0xa880ac10u + rk)) * 0xe4db9667u; - s[10] = (s[10] ^ (0x13e5dd1du + rk)) * 0x3e98f45du; - s[11] = (s[11] ^ (0x6fc3e233u + rk)) * 0xd0004eddu; - s[12] = (s[12] ^ (0x2d83eeacu + rk)) * 0x2691630du; - s[13] = (s[13] ^ (0x9006e8bfu + rk)) * 0x9beb3bcfu; - s[14] = (s[14] ^ (0x2c4b5362u + rk)) * 0xab310379u; - s[15] = (s[15] ^ (0x31b49ee2u + rk)) * 0x99cfb423u; - MH_QR(s[0], s[4], s[8], s[12], 20u, 20u, 19u, 4u) MH_QR(s[1], s[5], s[9], s[13], 20u, 20u, 19u, 4u) - MH_QR(s[2], s[6], s[10], s[14], 20u, 20u, 19u, 4u) MH_QR(s[3], s[7], s[11], s[15], 20u, 20u, 19u, 4u) - MH_QR(s[0], s[5], s[10], s[15], 26u, 3u, 3u, 27u) MH_QR(s[1], s[6], s[11], s[12], 26u, 3u, 3u, 27u) - MH_QR(s[2], s[7], s[8], s[13], 26u, 3u, 3u, 27u) MH_QR(s[3], s[4], s[9], s[14], 26u, 3u, 3u, 27u) -} - -// Item t: 16 words. s = (K, t * MUL[i] + RC[i]); 8 rounds of 8 x mixer + cache line s[0] & mask; 8 x final mixer. -inline void mh_item(device const uint* cache, uint t, thread uint* s) { - s[0] = 0x3067619fu; - s[1] = 0x3c269176u; - s[2] = 0x84a03b03u; - s[3] = 0xf8c63294u; - s[4] = 0xff977c5bu; - s[5] = 0xe60def3eu; - s[6] = 0x63630141u; - s[7] = 0xb8fbcb58u; - s[8] = t * 0x42146205u + 0xbab68293u; - s[9] = t * 0x52cbe0fbu + 0xcc162340u; - s[10] = t * 0x7ecf4a03u + 0x6ce151ccu; - s[11] = t * 0x6728907fu + 0xe62b8997u; - s[12] = t * 0xd81d9751u + 0xc9c80297u; - s[13] = t * 0x132952c3u + 0xf74a1654u; - s[14] = t * 0xf60de277u + 0x3d704af5u; - s[15] = t * 0x05358035u + 0x3cf522b7u; - for (uint r = 0u; r < 8u; ++r) { - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (r * 8u + j + 1u)); - device const uint* line = cache + ((s[0] & MH_CACHE_LINE_MASK) * 16u); - for (uint i = 0u; i < 16u; ++i) s[i] ^= line[i]; - } - for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (64u + j + 1u)); -} -// dataset[w] without the dataset: derive item w >> 4 and take word w & 15. -inline uint mh_word(device const uint* cache, uint w) { uint s[16]; mh_item(cache, w >> 4u, s); return s[w & 15u]; } - -// One thread per segment (2^16 threads). -kernel void igneum_cache_fill(device uint* cache [[buffer(0)]], uint gid [[thread_position_in_grid]]) { - mh_cache_segment(cache, gid); -} -// One thread per 64-byte item (dataset words / 16 threads). -kernel void igneum_build(device const uint* cache [[buffer(0)]], device uint* dataset [[buffer(1)]], - uint gid [[thread_position_in_grid]]) { - uint s[16]; - mh_item(cache, gid, s); - device uint* d = dataset + gid * 16u; - for (uint i = 0u; i < 16u; ++i) d[i] = s[i]; -} diff --git a/proto-cuda/packs-ca4/mx8_shl256x27_v2/program.h b/proto-cuda/packs-ca4/mx8_shl256x27_v2/program.h deleted file mode 100644 index 801fadcb3..000000000 --- a/proto-cuda/packs-ca4/mx8_shl256x27_v2/program.h +++ /dev/null @@ -1,72 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// Program metadata for host.cu plus the launch wrappers defined in kernel.cu. -// Also included by proto-opencl/host.c (C99), which defines IGNEUM_NO_CUDA first and reads only the macros. -#pragma once -#ifdef __cplusplus -#include -#else -#include -#endif -#ifndef IGNEUM_NO_CUDA -#include -#endif - -#define IGNEUM_SEED_STRING "igneum-genesis" -#define IGNEUM_SEED_BYTES_HEX "69676e65756d2d67656e65736973" -#define IGNEUM_GENERATOR 2 -#define IGNEUM_PROGRAM_ATTEMPT 3 -#define IGNEUM_PROGRAM_ID 0xbd64b207a30413fbull -#define IGNEUM_DAY_STRING "2026-10-03" -#define IGNEUM_DAY_BYTES_HEX "6461792f323032362d31302d3033" -#define IGNEUM_DAY0 0x3067619fu -#define IGNEUM_DAY1 0x3c269176u -#define IGNEUM_DATASET_LOG2 28 -#define IGNEUM_MASK 0x0fffffffu -#define IGNEUM_LANES 32 -#define IGNEUM_ITERATIONS 8 -#define IGNEUM_INSTR_COUNT 64 -#define IGNEUM_LOADS_PER_HASH 128 -#define IGNEUM_WIDE_LOADS_PER_HASH 0 -#define IGNEUM_OP_MIX "load=16 mul=8 rotl=6 shfl=6 add=5 mad=5 xor=5 rotr=4 mulhi=3 or=3 sub=3" -// Class v3 construction (Counter ASIC 2.0, 5 October 2026, docs/plans/mixer-x4.md): version 2 loads; the dataset item -// derivation applies the mixer IGNEUM_MIXER_MULT times per round (memhard.h), and the cache follows the growth rule. -#define IGNEUM_LOAD_CLASS "mx8+shl256x27" -#define IGNEUM_CLASS_MIXER_MULT 8 -#define IGNEUM_CACHE_GROWTH 1 // 1: cache words = 2^(26 + doublings(day)), doublings = floor(log2(1 + day / 1460)) -#define IGNEUM_LOAD_SLOTS 16 -#define IGNEUM_LOAD_MIX { 100, 0, 0 } -#define IGNEUM_LOAD_WIDTH_COUNTS { 16, 0, 0 } // loads of 4, 16, 64 bytes per program -#define IGNEUM_BYTES_PER_HASH 512 -#define IGNEUM_FOLD_ROT 11 -#define IGNEUM_FOLD_MUL 0x9e3779b1u -// Latency-shadow block (Counter ASIC 3.0 item 8, docs/analysis/latency-shadow-2026-10-06.md): NOT the lottery hash. A block of -// IGNEUM_SHADOW_INSTRS ALU instructions (the ten non-load families) runs IGNEUM_SHADOW_REPS times at the end of every -// iteration; the 16 loads, the acceptance rule and the base program are the class's without the shadow. -#define IGNEUM_SHADOW_INSTRS 256 -#define IGNEUM_SHADOW_REPS 27 -#define IGNEUM_SHADOW_INSTRS_PER_HASH 55296 -#define IGNEUM_SHADOW_OP_MIX "add=42 xor=39 mad=29 shfl=27 rotl=23 rotr=22 sub=22 or=19 mul=17 mulhi=16" -// Counter ASIC 4.0 research (experimental): the block is placed per load, sub-block j (instrs / 16) after the j-th load. -#define IGNEUM_SHADOW_PER_LOAD 1 -// 0 = closed-form dataset (ds_elem), 1 = memory-hard cache construction (MEMHARD.md, memhard.h) -#define IGNEUM_DATASET_MODE 1 - -#define IGNEUM_SEEDW_INIT { 0xf71aee9fu, 0xad930c88u, 0x7f982573u, 0xa41f9137u, 0x76d803d6u, 0x37b4a534u, 0x4d3fb826u, 0xff614dcbu } -#define IGNEUM_KEY_INIT { 0x3067619fu, 0x3c269176u, 0x84a03b03u, 0xf8c63294u, 0xff977c5bu, 0xe60def3eu, 0x63630141u, 0xb8fbcb58u } -#define IGNEUM_CACHE_LOG2_WORDS 26 -#define IGNEUM_CACHE_SEGMENT_LOG2_LINES 6 -#define IGNEUM_CACHE_SEGMENTS 65536u -#define IGNEUM_ITEM_ROUNDS 8 -#define IGNEUM_MIXER_MULT 8 // mixer applications per round and after the last read (class v3, docs/plans/mixer-x4.md) -#define IGNEUM_MIX_ROT_INIT { 20u, 20u, 19u, 4u, 26u, 3u, 3u, 27u } -#define IGNEUM_MIX_MUL_INIT { 0x42146205u, 0x52cbe0fbu, 0x7ecf4a03u, 0x6728907fu, 0xd81d9751u, 0x132952c3u, 0xf60de277u, 0x05358035u, 0xbaf6499du, 0xe4db9667u, 0x3e98f45du, 0xd0004eddu, 0x2691630du, 0x9beb3bcfu, 0xab310379u, 0x99cfb423u } -#define IGNEUM_MIX_RC_INIT { 0xbab68293u, 0xcc162340u, 0x6ce151ccu, 0xe62b8997u, 0xc9c80297u, 0xf74a1654u, 0x3d704af5u, 0x3cf522b7u, 0x2b9cac04u, 0xa880ac10u, 0x13e5dd1du, 0x6fc3e233u, 0x2d83eeacu, 0x9006e8bfu, 0x2c4b5362u, 0x31b49ee2u } - -#ifndef IGNEUM_NO_CUDA -// Defined in kernel.cu. All launch on the default stream and return cudaGetLastError(). -cudaError_t igneum_launch_cache_fill(uint32_t* cache, uint32_t nSegments); -cudaError_t igneum_launch_build(uint32_t* ds, const uint32_t* cache, uint32_t nItems); -cudaError_t igneum_launch_hash(const uint32_t* ds, uint64_t* out, uint32_t baseNonce, uint32_t mask, - uint32_t nonces, uint32_t blockWarps); -cudaError_t igneum_hash_info(int* numRegs, int* blocksPerSM, uint32_t blockWarps); -#endif diff --git a/proto-cuda/packs-ca4/mx8_shl256x27_v2/program.json b/proto-cuda/packs-ca4/mx8_shl256x27_v2/program.json deleted file mode 100644 index 1ca99c6ed..000000000 --- a/proto-cuda/packs-ca4/mx8_shl256x27_v2/program.json +++ /dev/null @@ -1,390 +0,0 @@ -{ - "format": "igneum-program-pack-3", - "generator": 2, - "attempt": 3, - "program_id": "0xbd64b207a30413fb", - "program_id_derivation": "FNV-1a 64 over 'igneum-program/' || generator_le32 || seed_words as little-endian bytes || attempt_le32", - "dataset_mode": "memory-hard", - "seed": "igneum-genesis", - "seed_bytes": "69676e65756d2d67656e65736973", - "seed_words": ["0xf71aee9f", "0xad930c88", "0x7f982573", "0xa41f9137", "0x76d803d6", "0x37b4a534", "0x4d3fb826", "0xff614dcb"], - "seed_derivation": "seed_words = FNV-1a 64 over seed_bytes (attempt 0) or seed_bytes || attempt_le32 (attempt k >= 1), basis ^ (salt * 0x9E3779B97F4A7C15) for salt 0..3, then h ^= h>>33; h *= 0xff51afd7ed558ccd; h ^= h>>33; words[2*salt] = low 32, words[2*salt+1] = high 32", - "generator_rule": "version 2: exactly 16 load slots drawn first from instructions 1..63 (partial Fisher-Yates), the other 48 ops from the ten non-load weights (sum 75); a load's source is drawn from the registers other than dst written by an earlier instruction and not read by a load since; the candidate must pass the acceptance rule of spec 01 section 1.4.6 (static: no cyclically stale load source, every register has an injecting write; dynamic: 64 units on the seed-keyed closed-form dataset with no constant register bit, no lane-constant load site, under 164 saturated final values, every output bit within 136 of 1024, distinct addresses above 245760), else the next attempt of the seed is tried", - "lanes": 32, - "registers": 8, - "iterations": 8, - "instruction_count": 64, - "loads_per_hash": 128, - "load_class": "mx8+shl256x27", - "mixer_mult": 8, - "cache_growth": true, - "mixer": "class v3 (Counter ASIC 2.0, 5 October 2026, docs/plans/mixer-x4.md): every mixer application of the item derivation is 8 applications with round keys (r * 8 + j + 1) * 0x9E3779B9, the 8 dependent cache reads per item unchanged; cache growth rule option C: cache words = 2^(26 + doublings(day)), dataset words = 2^(genesis_log2 + doublings(day)), doublings(day) = floor(log2(1 + day / 1460)) for day = days since genesis", - "load_slots": 16, - "load_mix_percent_4_16_64": [100, 0, 0], - "load_width_counts_4_16_64": [16, 0, 0], - "bytes_per_hash": 512, - "wide_load": "read-width experiment (5 October 2026, docs/plans/read-width.md), NOT the lottery hash: a load of W words (width field, 4 or 16) reads dataset[b .. b + W) with b = (src & mask) & ~(W - 1) and folds every word into dst: x = dst ^ w[0]; for j in 1..W: x = (rotl(x, 11) * 0x9e3779b1) ^ w[j]; dst = x; width 1 is the plain load; the width is drawn per instruction from the class mix with one extra below(100) draw after the nine of version 2, and the program id is FNV-1a 64 over 'igneum-program-rw/' || generator_le32 || seed words || attempt_le32 || mix[3] || load_slots", - "op_mix": {"load": 16, "mul": 8, "rotl": 6, "shfl": 6, "add": 5, "mad": 5, "xor": 5, "rotr": 4, "mulhi": 3, "or": 3, "sub": 3}, - "register_init": "for i in 0..7: x = nonce ^ seed_words[i]; x += 0x9e3779b9 * (i+1) (mod 2^32); x = splitmix32(x); r[i] = x ^ seed_words[(i+1) & 7]", - "splitmix32": "x ^= x>>16; x *= 0x7feb352d; x ^= x>>15; x *= 0x846ca68b; x ^= x>>16", - "iteration": "sel = r0 sampled once at the top of each iteration, then all instructions in order", - "output": "lo = r0 ^ rotl(r1,7) ^ rotl(r2,14) ^ rotl(r3,21); hi = r4 ^ rotl(r5,9) ^ rotl(r6,18) ^ rotl(r7,27); out = (hi << 32) | lo", - "op_semantics": { - "add": "dst = dst + src + (bit `bit` of sel ? imm2 : imm)", - "sub": "dst = dst - src", - "mul": "dst = dst * src (low 32)", - "mulhi": "dst = high 32 bits of dst * src", - "xor": "dst = dst ^ src", - "or": "dst = dst | src", - "rotl": "dst = rotl(dst, rot), rot in 1..31", - "rotr": "dst = rotr(dst, src & 31)", - "mad": "dst = src * src2 + dst", - "shfl": "dst = dst ^ (src of lane (lane ^ mask)), mask in {1,2,4,8,16}, within the 32-lane warp", - "load": "dst = dst ^ dataset[src & dataset.mask]", - "wload": "base = (src of lane 0 & dataset.mask) & ~31; dst = dst ^ dataset[base + lane] (warp-coalesced 128-byte load, lever b, only when --wide-frac > 0)" - }, - "dataset": { - "log2_words": 28, - "bytes": 1073741824, - "mask": "0x0fffffff", - "day": "2026-10-03", - "day_bytes": "6461792f323032362d31302d3033", - "day_words_from": "seed_words_from_bytes(day_bytes)", - "d0": "0x3067619f", - "d1": "0x3c269176", - "mode": "memory-hard", - "spec": "proto-metal/MEMHARD.md", - "key": ["0x3067619f", "0x3c269176", "0x84a03b03", "0xf8c63294", "0xff977c5b", "0xe60def3e", "0x63630141", "0xb8fbcb58"], - "key_derivation": "the 8 words of seed_words_from_bytes(day_bytes); d0, d1 are key[0], key[1]", - "cache": {"log2_words": 26, "bytes": 268435456, "line_words": 16, "segment_lines": 64, "segments": 65536, "block": "ChaCha12 core + feed-forward, rotations 16 12 8 7", "sigma": ["0x61707865", "0x3320646e", "0x79622d32", "0x6b206574"], "tag": ["0x49676e65", "0x756d4d48"], "chain": "in_j = prev_line ^ (sigma[0..3] || key[0..7] || seg || j || tag[0..1]); line_j = block(in_j); prev_0 = 0"}, - "mixer": {"draw": "SplitMix64 seeded with key[0] | key[1] << 32: rot[0..7] = 1 + next() % 31, mul[0..15] = low32(next()) | 1, rc[0..15] = low32(next())", "rot": [20, 20, 19, 4, 26, 3, 3, 27], "mul": ["0x42146205", "0x52cbe0fb", "0x7ecf4a03", "0x6728907f", "0xd81d9751", "0x132952c3", "0xf60de277", "0x05358035", "0xbaf6499d", "0xe4db9667", "0x3e98f45d", "0xd0004edd", "0x2691630d", "0x9beb3bcf", "0xab310379", "0x99cfb423"], "rc": ["0xbab68293", "0xcc162340", "0x6ce151cc", "0xe62b8997", "0xc9c80297", "0xf74a1654", "0x3d704af5", "0x3cf522b7", "0x2b9cac04", "0xa880ac10", "0x13e5dd1d", "0x6fc3e233", "0x2d83eeac", "0x9006e8bf", "0x2c4b5362", "0x31b49ee2"], "round": "for i in 0..15: s[i] = (s[i] ^ (rc[i] + (r+1) * 0x9E3779B9)) * mul[i]; then quarter rounds on columns (0,4,8,12) (1,5,9,13) (2,6,10,14) (3,7,11,15) with rot[0..3] and diagonals (0,5,10,15) (1,6,11,12) (2,7,8,13) (3,4,9,14) with rot[4..7]", "quarter_round": "a += b; d ^= a; d = rotl(d, r1); c += d; b ^= c; b = rotl(b, r2); a += b; d ^= a; d = rotl(d, r3); c += d; b ^= c; b = rotl(b, r4)"}, - "mixer_mult": 8, - "item": "s[0..7] = key; s[8+i] = t * mul[i] + rc[i] for i in 0..7; for r in 0..7: for j in 0..7: s = M(s, rk = (r * 8 + j + 1) * 0x9E3779B9); line = s[0] & 0x003fffff; s[i] ^= cache[line * 16 + i]; then for j in 0..7: s = M(s, rk = (64 + j + 1) * 0x9E3779B9); item(t) = s", - "word": "dataset[w] = item(w >> 4)[w & 15]" - }, - "shadow": {"instrs": 256, "reps": 27, "instrs_per_hash": 55296, "op_mix": {"add": 42, "xor": 39, "mad": 29, "shfl": 27, "rotl": 23, "rotr": 22, "sub": 22, "or": 19, "mul": 17, "mulhi": 16}, "rule": "Counter ASIC 3.0 item 8 (docs/analysis/latency-shadow-2026-10-06.md): after the 64 base instructions the program stream draws instrs more ALU instructions (the non-load table, nine draws each, the source as on an ALU slot); the block runs reps times at the end of every iteration with the iteration's sel; the acceptance rule interprets the base program only", "program_id_suffix": "'shadow/' || instrs_le16 || reps_le16", "instructions": [ - {"i": 0, "op": "rotr", "dst": 5, "src": 4, "src2": 6, "imm": "0x7b829e00", "imm2": "0x3c467e11", "rot": 1, "bit": 5, "mask": 4}, - {"i": 1, "op": "sub", "dst": 6, "src": 5, "src2": 3, "imm": "0xd7d51004", "imm2": "0x0ad2b27c", "rot": 23, "bit": 17, "mask": 4}, - {"i": 2, "op": "sub", "dst": 7, "src": 6, "src2": 3, "imm": "0xc90ee7a1", "imm2": "0xd9ecb052", "rot": 20, "bit": 25, "mask": 8}, - {"i": 3, "op": "mul", "dst": 5, "src": 4, "src2": 7, "imm": "0x420c0864", "imm2": "0x44a36c3c", "rot": 22, "bit": 26, "mask": 16}, - {"i": 4, "op": "xor", "dst": 7, "src": 6, "src2": 2, "imm": "0x23d1dbf3", "imm2": "0x851dcf48", "rot": 27, "bit": 31, "mask": 4}, - {"i": 5, "op": "rotl", "dst": 2, "src": 1, "src2": 5, "imm": "0xbeaf7569", "imm2": "0x358fdb07", "rot": 23, "bit": 2, "mask": 2}, - {"i": 6, "op": "xor", "dst": 7, "src": 4, "src2": 1, "imm": "0x779ddfcd", "imm2": "0xb93ae93c", "rot": 12, "bit": 15, "mask": 2}, - {"i": 7, "op": "add", "dst": 0, "src": 7, "src2": 7, "imm": "0xdaef8862", "imm2": "0x3051c491", "rot": 26, "bit": 26, "mask": 4}, - {"i": 8, "op": "shfl", "dst": 6, "src": 1, "src2": 5, "imm": "0x191b7f7e", "imm2": "0xe9c5693c", "rot": 30, "bit": 7, "mask": 4}, - {"i": 9, "op": "add", "dst": 5, "src": 1, "src2": 2, "imm": "0xc20e7045", "imm2": "0x5170d0b3", "rot": 5, "bit": 26, "mask": 1}, - {"i": 10, "op": "or", "dst": 0, "src": 3, "src2": 2, "imm": "0xa27ed074", "imm2": "0x1f2af192", "rot": 8, "bit": 26, "mask": 1}, - {"i": 11, "op": "mulhi", "dst": 5, "src": 4, "src2": 2, "imm": "0xaadbe6cc", "imm2": "0x24fa9dde", "rot": 13, "bit": 29, "mask": 16}, - {"i": 12, "op": "xor", "dst": 7, "src": 0, "src2": 4, "imm": "0xe787dbb1", "imm2": "0x54c46723", "rot": 1, "bit": 10, "mask": 4}, - {"i": 13, "op": "sub", "dst": 5, "src": 2, "src2": 1, "imm": "0x9d6a004e", "imm2": "0xb5b43329", "rot": 23, "bit": 6, "mask": 1}, - {"i": 14, "op": "add", "dst": 0, "src": 6, "src2": 0, "imm": "0xc86d98a4", "imm2": "0x2ead087f", "rot": 13, "bit": 0, "mask": 1}, - {"i": 15, "op": "xor", "dst": 5, "src": 7, "src2": 1, "imm": "0xa4205ee0", "imm2": "0x4f1d5bba", "rot": 7, "bit": 14, "mask": 16}, - {"i": 16, "op": "shfl", "dst": 7, "src": 6, "src2": 7, "imm": "0x6eb29f0d", "imm2": "0xb7af9e4e", "rot": 2, "bit": 19, "mask": 8}, - {"i": 17, "op": "sub", "dst": 5, "src": 7, "src2": 4, "imm": "0x11aa4853", "imm2": "0x2ce0c575", "rot": 12, "bit": 17, "mask": 16}, - {"i": 18, "op": "or", "dst": 0, "src": 6, "src2": 3, "imm": "0xed5226eb", "imm2": "0xcd376171", "rot": 18, "bit": 9, "mask": 16}, - {"i": 19, "op": "add", "dst": 0, "src": 5, "src2": 7, "imm": "0x4248b651", "imm2": "0xc47c70a8", "rot": 22, "bit": 13, "mask": 2}, - {"i": 20, "op": "shfl", "dst": 4, "src": 5, "src2": 2, "imm": "0xbead1759", "imm2": "0x1b913aee", "rot": 10, "bit": 14, "mask": 16}, - {"i": 21, "op": "add", "dst": 0, "src": 4, "src2": 5, "imm": "0x1e35684f", "imm2": "0x718c1008", "rot": 28, "bit": 21, "mask": 16}, - {"i": 22, "op": "mulhi", "dst": 0, "src": 3, "src2": 0, "imm": "0xe0cc85e3", "imm2": "0x5100e95e", "rot": 25, "bit": 3, "mask": 1}, - {"i": 23, "op": "xor", "dst": 4, "src": 2, "src2": 5, "imm": "0x36be80da", "imm2": "0x597fddd9", "rot": 20, "bit": 11, "mask": 2}, - {"i": 24, "op": "sub", "dst": 0, "src": 2, "src2": 2, "imm": "0x1c0fe722", "imm2": "0x9711da80", "rot": 22, "bit": 5, "mask": 1}, - {"i": 25, "op": "rotl", "dst": 5, "src": 4, "src2": 7, "imm": "0x2e3b3317", "imm2": "0x9e9da27f", "rot": 13, "bit": 21, "mask": 2}, - {"i": 26, "op": "sub", "dst": 7, "src": 0, "src2": 5, "imm": "0x397e8f2f", "imm2": "0x8c3f5941", "rot": 25, "bit": 9, "mask": 16}, - {"i": 27, "op": "xor", "dst": 2, "src": 3, "src2": 2, "imm": "0xa2d897d9", "imm2": "0x5e7a443f", "rot": 29, "bit": 20, "mask": 8}, - {"i": 28, "op": "mad", "dst": 2, "src": 3, "src2": 2, "imm": "0xa61e2ee2", "imm2": "0xe6535252", "rot": 26, "bit": 5, "mask": 1}, - {"i": 29, "op": "xor", "dst": 2, "src": 1, "src2": 1, "imm": "0x1c380f0c", "imm2": "0xf38cddf8", "rot": 17, "bit": 10, "mask": 8}, - {"i": 30, "op": "mulhi", "dst": 4, "src": 0, "src2": 1, "imm": "0x9ab5e6ed", "imm2": "0x9ae01059", "rot": 23, "bit": 5, "mask": 1}, - {"i": 31, "op": "xor", "dst": 6, "src": 4, "src2": 0, "imm": "0x8651f798", "imm2": "0xfcf55e9b", "rot": 1, "bit": 25, "mask": 1}, - {"i": 32, "op": "xor", "dst": 4, "src": 2, "src2": 6, "imm": "0xcb3c03c9", "imm2": "0xcc94a49e", "rot": 5, "bit": 10, "mask": 1}, - {"i": 33, "op": "shfl", "dst": 1, "src": 2, "src2": 3, "imm": "0x23451aa5", "imm2": "0x1ff0cbd2", "rot": 25, "bit": 25, "mask": 1}, - {"i": 34, "op": "shfl", "dst": 5, "src": 6, "src2": 5, "imm": "0x0a169154", "imm2": "0x9baa3264", "rot": 22, "bit": 3, "mask": 1}, - {"i": 35, "op": "sub", "dst": 4, "src": 0, "src2": 2, "imm": "0xa47d5cd6", "imm2": "0x99878331", "rot": 31, "bit": 28, "mask": 2}, - {"i": 36, "op": "sub", "dst": 6, "src": 3, "src2": 6, "imm": "0xab2ad546", "imm2": "0x53e15c19", "rot": 2, "bit": 10, "mask": 16}, - {"i": 37, "op": "or", "dst": 2, "src": 6, "src2": 2, "imm": "0xe123bfae", "imm2": "0xf0cde891", "rot": 16, "bit": 18, "mask": 2}, - {"i": 38, "op": "rotl", "dst": 2, "src": 6, "src2": 7, "imm": "0x3cc04288", "imm2": "0x85c70387", "rot": 30, "bit": 1, "mask": 8}, - {"i": 39, "op": "rotr", "dst": 4, "src": 7, "src2": 1, "imm": "0x80f31d36", "imm2": "0x8c32279d", "rot": 14, "bit": 22, "mask": 16}, - {"i": 40, "op": "or", "dst": 0, "src": 7, "src2": 1, "imm": "0x91a27c69", "imm2": "0xa94ba679", "rot": 16, "bit": 22, "mask": 16}, - {"i": 41, "op": "rotr", "dst": 2, "src": 5, "src2": 5, "imm": "0x5f5ca871", "imm2": "0xa66f6e1e", "rot": 30, "bit": 3, "mask": 8}, - {"i": 42, "op": "mad", "dst": 1, "src": 3, "src2": 3, "imm": "0xf16c6fbe", "imm2": "0xba65dbf1", "rot": 17, "bit": 10, "mask": 4}, - {"i": 43, "op": "mad", "dst": 6, "src": 5, "src2": 5, "imm": "0xee3e3937", "imm2": "0xf2ca16c2", "rot": 18, "bit": 9, "mask": 8}, - {"i": 44, "op": "mulhi", "dst": 1, "src": 0, "src2": 1, "imm": "0xb6ddbd42", "imm2": "0x8c9b9c55", "rot": 19, "bit": 24, "mask": 1}, - {"i": 45, "op": "mul", "dst": 1, "src": 3, "src2": 0, "imm": "0x5bde9611", "imm2": "0xa49dcc94", "rot": 9, "bit": 20, "mask": 16}, - {"i": 46, "op": "mad", "dst": 0, "src": 2, "src2": 0, "imm": "0xed018e02", "imm2": "0xb437c59e", "rot": 10, "bit": 30, "mask": 4}, - {"i": 47, "op": "sub", "dst": 1, "src": 5, "src2": 0, "imm": "0xc977dd30", "imm2": "0xd2194591", "rot": 25, "bit": 13, "mask": 2}, - {"i": 48, "op": "xor", "dst": 4, "src": 0, "src2": 2, "imm": "0x1e586e67", "imm2": "0x0bdc920a", "rot": 14, "bit": 25, "mask": 2}, - {"i": 49, "op": "rotr", "dst": 2, "src": 0, "src2": 4, "imm": "0xced53464", "imm2": "0x2b87e9aa", "rot": 27, "bit": 3, "mask": 16}, - {"i": 50, "op": "mulhi", "dst": 0, "src": 5, "src2": 4, "imm": "0xdfd051ba", "imm2": "0xe95248a6", "rot": 9, "bit": 9, "mask": 2}, - {"i": 51, "op": "or", "dst": 7, "src": 5, "src2": 0, "imm": "0xa2cb118c", "imm2": "0x357931db", "rot": 2, "bit": 12, "mask": 2}, - {"i": 52, "op": "mad", "dst": 4, "src": 0, "src2": 3, "imm": "0x1ea2a1ac", "imm2": "0x8b0c5c54", "rot": 15, "bit": 15, "mask": 16}, - {"i": 53, "op": "rotr", "dst": 0, "src": 4, "src2": 0, "imm": "0xbd1268fe", "imm2": "0x4cab0138", "rot": 9, "bit": 8, "mask": 2}, - {"i": 54, "op": "mad", "dst": 6, "src": 3, "src2": 3, "imm": "0xfd5c9198", "imm2": "0x6942bae5", "rot": 15, "bit": 22, "mask": 16}, - {"i": 55, "op": "mad", "dst": 6, "src": 4, "src2": 2, "imm": "0x4a01c39c", "imm2": "0xca9deebe", "rot": 30, "bit": 25, "mask": 1}, - {"i": 56, "op": "shfl", "dst": 5, "src": 0, "src2": 6, "imm": "0x5a5c8edc", "imm2": "0x36f40134", "rot": 1, "bit": 13, "mask": 2}, - {"i": 57, "op": "rotl", "dst": 7, "src": 3, "src2": 2, "imm": "0x62950b95", "imm2": "0x1bac0994", "rot": 21, "bit": 12, "mask": 4}, - {"i": 58, "op": "xor", "dst": 3, "src": 7, "src2": 0, "imm": "0x4dca8e46", "imm2": "0x79c853b5", "rot": 25, "bit": 11, "mask": 1}, - {"i": 59, "op": "shfl", "dst": 0, "src": 4, "src2": 6, "imm": "0x4c2b533c", "imm2": "0xf4b09101", "rot": 3, "bit": 4, "mask": 1}, - {"i": 60, "op": "xor", "dst": 5, "src": 6, "src2": 1, "imm": "0xc41d4acd", "imm2": "0xc42c4716", "rot": 31, "bit": 21, "mask": 8}, - {"i": 61, "op": "rotr", "dst": 4, "src": 7, "src2": 4, "imm": "0xdafe1dfd", "imm2": "0xa4b90067", "rot": 4, "bit": 0, "mask": 2}, - {"i": 62, "op": "add", "dst": 1, "src": 6, "src2": 2, "imm": "0xf7ccf1e9", "imm2": "0x31f87d74", "rot": 12, "bit": 10, "mask": 2}, - {"i": 63, "op": "add", "dst": 5, "src": 2, "src2": 2, "imm": "0x2f386099", "imm2": "0x9c2847f4", "rot": 10, "bit": 15, "mask": 4}, - {"i": 64, "op": "add", "dst": 7, "src": 0, "src2": 3, "imm": "0x1b30ce7a", "imm2": "0xd3241188", "rot": 14, "bit": 19, "mask": 2}, - {"i": 65, "op": "add", "dst": 6, "src": 7, "src2": 4, "imm": "0x02dc8349", "imm2": "0x9b42c3de", "rot": 18, "bit": 13, "mask": 4}, - {"i": 66, "op": "rotl", "dst": 0, "src": 5, "src2": 6, "imm": "0xd41f93ed", "imm2": "0x2183702d", "rot": 16, "bit": 30, "mask": 16}, - {"i": 67, "op": "rotl", "dst": 2, "src": 6, "src2": 5, "imm": "0xf8423070", "imm2": "0xf48afd52", "rot": 9, "bit": 8, "mask": 2}, - {"i": 68, "op": "mad", "dst": 2, "src": 5, "src2": 6, "imm": "0xb98ae05a", "imm2": "0x97eec6a3", "rot": 18, "bit": 10, "mask": 16}, - {"i": 69, "op": "shfl", "dst": 6, "src": 1, "src2": 0, "imm": "0x176f02ea", "imm2": "0x703e1cab", "rot": 10, "bit": 19, "mask": 8}, - {"i": 70, "op": "xor", "dst": 2, "src": 5, "src2": 7, "imm": "0x14922644", "imm2": "0x1a18b3ed", "rot": 22, "bit": 27, "mask": 1}, - {"i": 71, "op": "mulhi", "dst": 5, "src": 1, "src2": 4, "imm": "0x05d04820", "imm2": "0x60e43e74", "rot": 25, "bit": 31, "mask": 4}, - {"i": 72, "op": "rotl", "dst": 6, "src": 0, "src2": 5, "imm": "0xf04e4109", "imm2": "0xf1ddb551", "rot": 29, "bit": 20, "mask": 1}, - {"i": 73, "op": "sub", "dst": 0, "src": 7, "src2": 7, "imm": "0xe1f20354", "imm2": "0xcd89bdb8", "rot": 2, "bit": 4, "mask": 16}, - {"i": 74, "op": "mad", "dst": 5, "src": 2, "src2": 2, "imm": "0xfb42bfa3", "imm2": "0xe1df26b9", "rot": 21, "bit": 10, "mask": 4}, - {"i": 75, "op": "rotr", "dst": 0, "src": 5, "src2": 6, "imm": "0x67376b2f", "imm2": "0xc02ef691", "rot": 11, "bit": 21, "mask": 1}, - {"i": 76, "op": "add", "dst": 7, "src": 2, "src2": 2, "imm": "0x05d36679", "imm2": "0x0da1ce17", "rot": 7, "bit": 9, "mask": 1}, - {"i": 77, "op": "sub", "dst": 7, "src": 2, "src2": 3, "imm": "0x8841ade4", "imm2": "0x02f2395d", "rot": 15, "bit": 29, "mask": 1}, - {"i": 78, "op": "rotr", "dst": 7, "src": 0, "src2": 2, "imm": "0x10a0bc35", "imm2": "0x71df32ce", "rot": 29, "bit": 6, "mask": 4}, - {"i": 79, "op": "mad", "dst": 1, "src": 5, "src2": 5, "imm": "0x953044fb", "imm2": "0x688d575c", "rot": 18, "bit": 1, "mask": 8}, - {"i": 80, "op": "rotr", "dst": 3, "src": 0, "src2": 3, "imm": "0xd773d95b", "imm2": "0x96c0a95c", "rot": 17, "bit": 20, "mask": 16}, - {"i": 81, "op": "add", "dst": 6, "src": 5, "src2": 4, "imm": "0xf14547bd", "imm2": "0xadf5ef88", "rot": 10, "bit": 2, "mask": 4}, - {"i": 82, "op": "or", "dst": 0, "src": 6, "src2": 3, "imm": "0x2d973325", "imm2": "0x3cdc59f1", "rot": 3, "bit": 20, "mask": 8}, - {"i": 83, "op": "mulhi", "dst": 1, "src": 0, "src2": 6, "imm": "0x850e8c17", "imm2": "0xd83841f9", "rot": 30, "bit": 23, "mask": 16}, - {"i": 84, "op": "mad", "dst": 7, "src": 6, "src2": 7, "imm": "0xbc7fb049", "imm2": "0xed9928df", "rot": 4, "bit": 3, "mask": 4}, - {"i": 85, "op": "rotl", "dst": 5, "src": 0, "src2": 6, "imm": "0xf6305281", "imm2": "0x95a40a03", "rot": 29, "bit": 28, "mask": 2}, - {"i": 86, "op": "xor", "dst": 2, "src": 6, "src2": 0, "imm": "0xd59ffa4f", "imm2": "0x254a4101", "rot": 9, "bit": 28, "mask": 2}, - {"i": 87, "op": "add", "dst": 5, "src": 4, "src2": 2, "imm": "0xba4947c2", "imm2": "0x35ff14ae", "rot": 9, "bit": 24, "mask": 4}, - {"i": 88, "op": "add", "dst": 3, "src": 6, "src2": 4, "imm": "0xf6878bee", "imm2": "0xf3900fc1", "rot": 18, "bit": 3, "mask": 4}, - {"i": 89, "op": "add", "dst": 0, "src": 2, "src2": 5, "imm": "0x926f3607", "imm2": "0xf7e8f59f", "rot": 22, "bit": 12, "mask": 16}, - {"i": 90, "op": "xor", "dst": 2, "src": 6, "src2": 6, "imm": "0x8b009dbb", "imm2": "0x87e4be1e", "rot": 9, "bit": 4, "mask": 8}, - {"i": 91, "op": "shfl", "dst": 3, "src": 2, "src2": 3, "imm": "0x35a452ba", "imm2": "0x1fb223aa", "rot": 15, "bit": 28, "mask": 2}, - {"i": 92, "op": "sub", "dst": 6, "src": 1, "src2": 2, "imm": "0xd26d2497", "imm2": "0x6ace9269", "rot": 20, "bit": 17, "mask": 4}, - {"i": 93, "op": "add", "dst": 5, "src": 1, "src2": 3, "imm": "0xdab36c29", "imm2": "0x0e376f9c", "rot": 27, "bit": 13, "mask": 4}, - {"i": 94, "op": "mad", "dst": 6, "src": 1, "src2": 1, "imm": "0x29e2923e", "imm2": "0x67a97747", "rot": 16, "bit": 30, "mask": 2}, - {"i": 95, "op": "xor", "dst": 3, "src": 4, "src2": 3, "imm": "0x982f8e78", "imm2": "0xdbb7d73f", "rot": 6, "bit": 26, "mask": 8}, - {"i": 96, "op": "mad", "dst": 6, "src": 5, "src2": 2, "imm": "0x760e08de", "imm2": "0x12f3375f", "rot": 27, "bit": 7, "mask": 16}, - {"i": 97, "op": "add", "dst": 7, "src": 3, "src2": 7, "imm": "0x204129e9", "imm2": "0x9f917747", "rot": 26, "bit": 23, "mask": 1}, - {"i": 98, "op": "shfl", "dst": 5, "src": 1, "src2": 5, "imm": "0x0f6aca43", "imm2": "0xdc5cea76", "rot": 29, "bit": 5, "mask": 16}, - {"i": 99, "op": "mul", "dst": 2, "src": 6, "src2": 4, "imm": "0x6abaa10f", "imm2": "0xe0968a9b", "rot": 22, "bit": 27, "mask": 8}, - {"i": 100, "op": "shfl", "dst": 7, "src": 3, "src2": 7, "imm": "0x48d68211", "imm2": "0x9d514118", "rot": 12, "bit": 9, "mask": 8}, - {"i": 101, "op": "add", "dst": 6, "src": 1, "src2": 7, "imm": "0xe3b596a0", "imm2": "0xe42fe974", "rot": 25, "bit": 10, "mask": 2}, - {"i": 102, "op": "rotl", "dst": 5, "src": 1, "src2": 6, "imm": "0xc9afa2dd", "imm2": "0x8441c699", "rot": 3, "bit": 5, "mask": 2}, - {"i": 103, "op": "xor", "dst": 5, "src": 1, "src2": 0, "imm": "0xff93d0c4", "imm2": "0x2b65c415", "rot": 12, "bit": 9, "mask": 16}, - {"i": 104, "op": "add", "dst": 4, "src": 1, "src2": 5, "imm": "0x705c3f94", "imm2": "0xc3d77ffb", "rot": 24, "bit": 27, "mask": 1}, - {"i": 105, "op": "add", "dst": 0, "src": 6, "src2": 0, "imm": "0xbc6fb42b", "imm2": "0xea02a4ca", "rot": 13, "bit": 25, "mask": 4}, - {"i": 106, "op": "or", "dst": 3, "src": 5, "src2": 3, "imm": "0x07559d58", "imm2": "0x5b76e4b5", "rot": 12, "bit": 21, "mask": 1}, - {"i": 107, "op": "sub", "dst": 0, "src": 6, "src2": 7, "imm": "0x70f0b3f2", "imm2": "0xf862cea0", "rot": 20, "bit": 23, "mask": 4}, - {"i": 108, "op": "rotr", "dst": 0, "src": 3, "src2": 1, "imm": "0x93724f38", "imm2": "0x4717fcd8", "rot": 3, "bit": 5, "mask": 8}, - {"i": 109, "op": "add", "dst": 5, "src": 4, "src2": 0, "imm": "0xcb37ea87", "imm2": "0x0357e63e", "rot": 27, "bit": 4, "mask": 2}, - {"i": 110, "op": "rotl", "dst": 7, "src": 3, "src2": 6, "imm": "0x70543783", "imm2": "0x85fcb2f9", "rot": 5, "bit": 13, "mask": 4}, - {"i": 111, "op": "xor", "dst": 4, "src": 1, "src2": 6, "imm": "0x325f187e", "imm2": "0xc4ba0312", "rot": 12, "bit": 14, "mask": 2}, - {"i": 112, "op": "xor", "dst": 6, "src": 5, "src2": 1, "imm": "0xcad70d8a", "imm2": "0x6d6c7bc7", "rot": 5, "bit": 7, "mask": 2}, - {"i": 113, "op": "or", "dst": 1, "src": 3, "src2": 0, "imm": "0x3569dfa0", "imm2": "0xbfab6be2", "rot": 17, "bit": 21, "mask": 8}, - {"i": 114, "op": "add", "dst": 0, "src": 5, "src2": 5, "imm": "0x9aab0297", "imm2": "0x7f8b8cd2", "rot": 21, "bit": 3, "mask": 16}, - {"i": 115, "op": "sub", "dst": 3, "src": 5, "src2": 1, "imm": "0xaaf5f8b5", "imm2": "0xb629e1be", "rot": 24, "bit": 6, "mask": 1}, - {"i": 116, "op": "rotr", "dst": 4, "src": 2, "src2": 6, "imm": "0x7f466189", "imm2": "0xf52bd6af", "rot": 13, "bit": 13, "mask": 16}, - {"i": 117, "op": "add", "dst": 4, "src": 6, "src2": 7, "imm": "0xc511183f", "imm2": "0x59400b57", "rot": 17, "bit": 6, "mask": 8}, - {"i": 118, "op": "sub", "dst": 6, "src": 1, "src2": 4, "imm": "0xf997f264", "imm2": "0x948760fa", "rot": 29, "bit": 3, "mask": 1}, - {"i": 119, "op": "shfl", "dst": 3, "src": 1, "src2": 0, "imm": "0xb9fb8af4", "imm2": "0x735dfe1f", "rot": 24, "bit": 24, "mask": 4}, - {"i": 120, "op": "mad", "dst": 6, "src": 4, "src2": 4, "imm": "0x415a06d6", "imm2": "0xd0160990", "rot": 18, "bit": 11, "mask": 16}, - {"i": 121, "op": "rotl", "dst": 2, "src": 7, "src2": 4, "imm": "0x78dc55da", "imm2": "0x36084523", "rot": 8, "bit": 15, "mask": 16}, - {"i": 122, "op": "mad", "dst": 5, "src": 0, "src2": 3, "imm": "0x951b8e8c", "imm2": "0x3247ab68", "rot": 8, "bit": 25, "mask": 8}, - {"i": 123, "op": "add", "dst": 7, "src": 6, "src2": 7, "imm": "0x5200c242", "imm2": "0x27c70dc0", "rot": 23, "bit": 27, "mask": 4}, - {"i": 124, "op": "shfl", "dst": 3, "src": 5, "src2": 7, "imm": "0x5512f7bd", "imm2": "0x83d356df", "rot": 1, "bit": 25, "mask": 4}, - {"i": 125, "op": "add", "dst": 0, "src": 1, "src2": 3, "imm": "0x2d5028ce", "imm2": "0xec2f5997", "rot": 27, "bit": 17, "mask": 1}, - {"i": 126, "op": "shfl", "dst": 3, "src": 6, "src2": 0, "imm": "0xb2e45b6a", "imm2": "0xee317a17", "rot": 14, "bit": 19, "mask": 16}, - {"i": 127, "op": "xor", "dst": 5, "src": 0, "src2": 3, "imm": "0xe7a12b57", "imm2": "0x4e0dcf60", "rot": 9, "bit": 11, "mask": 4}, - {"i": 128, "op": "add", "dst": 7, "src": 1, "src2": 0, "imm": "0x432f6c0d", "imm2": "0x09e8ede2", "rot": 20, "bit": 10, "mask": 8}, - {"i": 129, "op": "xor", "dst": 2, "src": 4, "src2": 5, "imm": "0x76c67109", "imm2": "0x3f54d25d", "rot": 4, "bit": 12, "mask": 1}, - {"i": 130, "op": "mul", "dst": 0, "src": 2, "src2": 6, "imm": "0x4eca1bc0", "imm2": "0x77c29a67", "rot": 8, "bit": 22, "mask": 4}, - {"i": 131, "op": "or", "dst": 4, "src": 3, "src2": 7, "imm": "0x3195a6fe", "imm2": "0xa1e44e04", "rot": 6, "bit": 13, "mask": 4}, - {"i": 132, "op": "or", "dst": 3, "src": 7, "src2": 0, "imm": "0x059c55ae", "imm2": "0x0a0818b4", "rot": 22, "bit": 8, "mask": 4}, - {"i": 133, "op": "shfl", "dst": 7, "src": 3, "src2": 5, "imm": "0xbd84d73f", "imm2": "0xfcbc80e3", "rot": 8, "bit": 27, "mask": 1}, - {"i": 134, "op": "add", "dst": 5, "src": 2, "src2": 6, "imm": "0xcf5ecc75", "imm2": "0xce4fdf8e", "rot": 31, "bit": 11, "mask": 16}, - {"i": 135, "op": "mad", "dst": 3, "src": 2, "src2": 3, "imm": "0x91f38b1f", "imm2": "0xf12e182b", "rot": 11, "bit": 16, "mask": 2}, - {"i": 136, "op": "rotl", "dst": 1, "src": 2, "src2": 7, "imm": "0x1df61118", "imm2": "0xecebf83b", "rot": 11, "bit": 10, "mask": 8}, - {"i": 137, "op": "or", "dst": 2, "src": 0, "src2": 6, "imm": "0x1b217afb", "imm2": "0x009986f1", "rot": 27, "bit": 17, "mask": 4}, - {"i": 138, "op": "xor", "dst": 3, "src": 4, "src2": 2, "imm": "0xa5b0e8e3", "imm2": "0x8344f4bc", "rot": 25, "bit": 23, "mask": 2}, - {"i": 139, "op": "shfl", "dst": 2, "src": 4, "src2": 1, "imm": "0xe95b436b", "imm2": "0x04f47b79", "rot": 12, "bit": 6, "mask": 1}, - {"i": 140, "op": "rotl", "dst": 2, "src": 6, "src2": 0, "imm": "0x278ba5d4", "imm2": "0x304cc572", "rot": 6, "bit": 31, "mask": 1}, - {"i": 141, "op": "mul", "dst": 0, "src": 3, "src2": 6, "imm": "0xfa5febcc", "imm2": "0x2462f50a", "rot": 25, "bit": 24, "mask": 2}, - {"i": 142, "op": "mul", "dst": 1, "src": 7, "src2": 7, "imm": "0xadab0c26", "imm2": "0x83cefec3", "rot": 3, "bit": 3, "mask": 16}, - {"i": 143, "op": "xor", "dst": 0, "src": 1, "src2": 3, "imm": "0x23cacfae", "imm2": "0x4acf331d", "rot": 13, "bit": 27, "mask": 16}, - {"i": 144, "op": "mad", "dst": 6, "src": 2, "src2": 0, "imm": "0xd3cd907e", "imm2": "0x99806cb9", "rot": 13, "bit": 4, "mask": 4}, - {"i": 145, "op": "rotl", "dst": 1, "src": 4, "src2": 3, "imm": "0xafad23ae", "imm2": "0x919a563b", "rot": 15, "bit": 29, "mask": 2}, - {"i": 146, "op": "rotl", "dst": 1, "src": 4, "src2": 3, "imm": "0x8ff43176", "imm2": "0x20b58068", "rot": 30, "bit": 9, "mask": 4}, - {"i": 147, "op": "xor", "dst": 6, "src": 0, "src2": 1, "imm": "0x7b929413", "imm2": "0x7a7e2fa9", "rot": 7, "bit": 9, "mask": 16}, - {"i": 148, "op": "sub", "dst": 3, "src": 2, "src2": 7, "imm": "0xf2346114", "imm2": "0x74d45ef6", "rot": 17, "bit": 5, "mask": 16}, - {"i": 149, "op": "add", "dst": 3, "src": 0, "src2": 2, "imm": "0xaa002f15", "imm2": "0x11bbdeef", "rot": 22, "bit": 5, "mask": 8}, - {"i": 150, "op": "add", "dst": 6, "src": 5, "src2": 5, "imm": "0xbe64b2e2", "imm2": "0xff9d4b0e", "rot": 14, "bit": 9, "mask": 1}, - {"i": 151, "op": "mul", "dst": 1, "src": 4, "src2": 1, "imm": "0x80ab9255", "imm2": "0x7313b029", "rot": 16, "bit": 5, "mask": 2}, - {"i": 152, "op": "rotl", "dst": 5, "src": 6, "src2": 1, "imm": "0xc5ca5727", "imm2": "0x0153e735", "rot": 3, "bit": 2, "mask": 8}, - {"i": 153, "op": "mul", "dst": 5, "src": 2, "src2": 5, "imm": "0x1598de73", "imm2": "0x99d642dc", "rot": 28, "bit": 30, "mask": 2}, - {"i": 154, "op": "rotl", "dst": 2, "src": 7, "src2": 2, "imm": "0x9f68d6f0", "imm2": "0x29c72445", "rot": 21, "bit": 8, "mask": 1}, - {"i": 155, "op": "rotl", "dst": 5, "src": 0, "src2": 1, "imm": "0xed20980f", "imm2": "0x34b14e75", "rot": 18, "bit": 3, "mask": 2}, - {"i": 156, "op": "xor", "dst": 1, "src": 5, "src2": 2, "imm": "0x5e127c80", "imm2": "0x9d48bd29", "rot": 19, "bit": 14, "mask": 2}, - {"i": 157, "op": "mulhi", "dst": 1, "src": 6, "src2": 5, "imm": "0xd924c751", "imm2": "0x794a134e", "rot": 20, "bit": 31, "mask": 8}, - {"i": 158, "op": "mad", "dst": 4, "src": 7, "src2": 3, "imm": "0x1353b41d", "imm2": "0x34568db3", "rot": 24, "bit": 11, "mask": 16}, - {"i": 159, "op": "sub", "dst": 4, "src": 7, "src2": 7, "imm": "0xfe36ca46", "imm2": "0x915bb25f", "rot": 19, "bit": 10, "mask": 8}, - {"i": 160, "op": "xor", "dst": 3, "src": 2, "src2": 6, "imm": "0xa5a7b77f", "imm2": "0x3101857c", "rot": 3, "bit": 21, "mask": 8}, - {"i": 161, "op": "mulhi", "dst": 3, "src": 4, "src2": 1, "imm": "0x376ee390", "imm2": "0xd6af078e", "rot": 12, "bit": 8, "mask": 4}, - {"i": 162, "op": "shfl", "dst": 3, "src": 2, "src2": 2, "imm": "0x95741acf", "imm2": "0x90ea2feb", "rot": 7, "bit": 15, "mask": 1}, - {"i": 163, "op": "sub", "dst": 0, "src": 1, "src2": 2, "imm": "0x3176da5b", "imm2": "0xd433b4be", "rot": 30, "bit": 31, "mask": 8}, - {"i": 164, "op": "shfl", "dst": 6, "src": 0, "src2": 4, "imm": "0x0502a0cc", "imm2": "0x45dbed17", "rot": 15, "bit": 18, "mask": 8}, - {"i": 165, "op": "shfl", "dst": 4, "src": 2, "src2": 5, "imm": "0x1fbeb1fb", "imm2": "0xc363e4c5", "rot": 3, "bit": 1, "mask": 2}, - {"i": 166, "op": "or", "dst": 3, "src": 6, "src2": 6, "imm": "0x5522496d", "imm2": "0x0e23496d", "rot": 11, "bit": 8, "mask": 2}, - {"i": 167, "op": "shfl", "dst": 6, "src": 3, "src2": 3, "imm": "0xc1f312c5", "imm2": "0x7679c16e", "rot": 7, "bit": 23, "mask": 4}, - {"i": 168, "op": "xor", "dst": 2, "src": 1, "src2": 0, "imm": "0xdfdbdee2", "imm2": "0x054ab2d1", "rot": 17, "bit": 1, "mask": 8}, - {"i": 169, "op": "xor", "dst": 5, "src": 1, "src2": 1, "imm": "0x4913a0ba", "imm2": "0x8681ae62", "rot": 19, "bit": 29, "mask": 16}, - {"i": 170, "op": "xor", "dst": 5, "src": 0, "src2": 7, "imm": "0x11a04a63", "imm2": "0x5d646386", "rot": 12, "bit": 31, "mask": 1}, - {"i": 171, "op": "mad", "dst": 3, "src": 6, "src2": 0, "imm": "0x66b954f7", "imm2": "0x615c24af", "rot": 22, "bit": 17, "mask": 8}, - {"i": 172, "op": "sub", "dst": 3, "src": 0, "src2": 5, "imm": "0x071a3544", "imm2": "0xa1d3128f", "rot": 24, "bit": 4, "mask": 4}, - {"i": 173, "op": "add", "dst": 6, "src": 0, "src2": 1, "imm": "0x3b90694b", "imm2": "0xc72dc2a0", "rot": 24, "bit": 27, "mask": 2}, - {"i": 174, "op": "shfl", "dst": 7, "src": 2, "src2": 7, "imm": "0x50f474e4", "imm2": "0x1138a9f1", "rot": 17, "bit": 16, "mask": 8}, - {"i": 175, "op": "mulhi", "dst": 5, "src": 2, "src2": 4, "imm": "0x09561616", "imm2": "0x8cc5086a", "rot": 26, "bit": 14, "mask": 4}, - {"i": 176, "op": "add", "dst": 3, "src": 6, "src2": 7, "imm": "0x9e65cebd", "imm2": "0x4eb75843", "rot": 29, "bit": 5, "mask": 8}, - {"i": 177, "op": "mad", "dst": 0, "src": 4, "src2": 1, "imm": "0xa95b4929", "imm2": "0x7a61b006", "rot": 16, "bit": 22, "mask": 4}, - {"i": 178, "op": "rotr", "dst": 6, "src": 7, "src2": 5, "imm": "0x9a34a23e", "imm2": "0xb22c37c9", "rot": 1, "bit": 22, "mask": 4}, - {"i": 179, "op": "add", "dst": 0, "src": 7, "src2": 0, "imm": "0x01e5b250", "imm2": "0x7001d036", "rot": 3, "bit": 31, "mask": 4}, - {"i": 180, "op": "shfl", "dst": 5, "src": 2, "src2": 5, "imm": "0xa7f6a337", "imm2": "0xe645a2c2", "rot": 6, "bit": 4, "mask": 16}, - {"i": 181, "op": "mul", "dst": 3, "src": 0, "src2": 4, "imm": "0x0f2b1ce4", "imm2": "0x1046c3c8", "rot": 22, "bit": 28, "mask": 8}, - {"i": 182, "op": "rotl", "dst": 0, "src": 4, "src2": 7, "imm": "0xc7e8ae1c", "imm2": "0x98f9b08e", "rot": 23, "bit": 7, "mask": 1}, - {"i": 183, "op": "mulhi", "dst": 7, "src": 0, "src2": 6, "imm": "0xb64797fa", "imm2": "0x613b63ba", "rot": 17, "bit": 11, "mask": 2}, - {"i": 184, "op": "mul", "dst": 0, "src": 4, "src2": 6, "imm": "0xfcddd784", "imm2": "0x85012b36", "rot": 13, "bit": 19, "mask": 2}, - {"i": 185, "op": "add", "dst": 1, "src": 4, "src2": 1, "imm": "0xbef14988", "imm2": "0x91736711", "rot": 28, "bit": 19, "mask": 2}, - {"i": 186, "op": "add", "dst": 7, "src": 2, "src2": 3, "imm": "0xf5d741be", "imm2": "0x15e0cdf3", "rot": 28, "bit": 6, "mask": 16}, - {"i": 187, "op": "xor", "dst": 1, "src": 7, "src2": 0, "imm": "0xac0a6f4a", "imm2": "0x5fb7de9b", "rot": 13, "bit": 5, "mask": 16}, - {"i": 188, "op": "add", "dst": 1, "src": 3, "src2": 4, "imm": "0x32be33c6", "imm2": "0x7549bc3e", "rot": 10, "bit": 21, "mask": 2}, - {"i": 189, "op": "mad", "dst": 1, "src": 0, "src2": 6, "imm": "0x1c7ce36b", "imm2": "0x31fd592c", "rot": 29, "bit": 3, "mask": 8}, - {"i": 190, "op": "rotr", "dst": 0, "src": 5, "src2": 5, "imm": "0x5d8729d9", "imm2": "0x65b562d0", "rot": 18, "bit": 17, "mask": 8}, - {"i": 191, "op": "mad", "dst": 4, "src": 0, "src2": 1, "imm": "0x8f7c8ec4", "imm2": "0xf2b4257c", "rot": 15, "bit": 14, "mask": 2}, - {"i": 192, "op": "mad", "dst": 3, "src": 6, "src2": 1, "imm": "0xd6bd20bf", "imm2": "0xaf9177ad", "rot": 4, "bit": 8, "mask": 8}, - {"i": 193, "op": "rotl", "dst": 6, "src": 5, "src2": 1, "imm": "0xb3978896", "imm2": "0x240377d5", "rot": 31, "bit": 21, "mask": 16}, - {"i": 194, "op": "rotl", "dst": 5, "src": 1, "src2": 5, "imm": "0x09a5a134", "imm2": "0x0e861d51", "rot": 28, "bit": 31, "mask": 8}, - {"i": 195, "op": "mul", "dst": 4, "src": 3, "src2": 6, "imm": "0x4be7a174", "imm2": "0xd3a7b457", "rot": 3, "bit": 30, "mask": 4}, - {"i": 196, "op": "add", "dst": 2, "src": 3, "src2": 0, "imm": "0x9d7aebbb", "imm2": "0x7c3d6253", "rot": 19, "bit": 4, "mask": 16}, - {"i": 197, "op": "or", "dst": 3, "src": 5, "src2": 1, "imm": "0xc1e98a9d", "imm2": "0x12f8d9c0", "rot": 4, "bit": 29, "mask": 1}, - {"i": 198, "op": "sub", "dst": 6, "src": 0, "src2": 5, "imm": "0x9fabde83", "imm2": "0xc1aa2826", "rot": 13, "bit": 14, "mask": 4}, - {"i": 199, "op": "mul", "dst": 7, "src": 4, "src2": 2, "imm": "0x32653761", "imm2": "0x17d8a6c5", "rot": 23, "bit": 4, "mask": 2}, - {"i": 200, "op": "rotr", "dst": 3, "src": 2, "src2": 2, "imm": "0xd7c4c307", "imm2": "0x9664a047", "rot": 21, "bit": 0, "mask": 16}, - {"i": 201, "op": "shfl", "dst": 4, "src": 0, "src2": 2, "imm": "0x058d2aa6", "imm2": "0xaf120942", "rot": 18, "bit": 27, "mask": 16}, - {"i": 202, "op": "xor", "dst": 7, "src": 3, "src2": 6, "imm": "0x3941fc7c", "imm2": "0x8ae9a794", "rot": 30, "bit": 13, "mask": 4}, - {"i": 203, "op": "or", "dst": 4, "src": 3, "src2": 3, "imm": "0x71058171", "imm2": "0xf05194e4", "rot": 23, "bit": 25, "mask": 4}, - {"i": 204, "op": "rotr", "dst": 3, "src": 6, "src2": 2, "imm": "0xe770de5a", "imm2": "0x00ba5114", "rot": 15, "bit": 0, "mask": 16}, - {"i": 205, "op": "rotr", "dst": 0, "src": 1, "src2": 7, "imm": "0xfa875b85", "imm2": "0x7b600f65", "rot": 17, "bit": 22, "mask": 8}, - {"i": 206, "op": "mad", "dst": 2, "src": 5, "src2": 2, "imm": "0x77c92526", "imm2": "0x0eeae451", "rot": 26, "bit": 31, "mask": 2}, - {"i": 207, "op": "mulhi", "dst": 1, "src": 5, "src2": 2, "imm": "0xe1d887d8", "imm2": "0xb09c96eb", "rot": 16, "bit": 24, "mask": 1}, - {"i": 208, "op": "shfl", "dst": 4, "src": 3, "src2": 7, "imm": "0x5742fb2a", "imm2": "0x8307f5f0", "rot": 2, "bit": 21, "mask": 1}, - {"i": 209, "op": "mad", "dst": 7, "src": 5, "src2": 7, "imm": "0xd3c4a9bd", "imm2": "0xaf46c6f6", "rot": 30, "bit": 27, "mask": 16}, - {"i": 210, "op": "xor", "dst": 6, "src": 3, "src2": 5, "imm": "0x9542f3b3", "imm2": "0xb959da41", "rot": 30, "bit": 9, "mask": 16}, - {"i": 211, "op": "xor", "dst": 3, "src": 2, "src2": 5, "imm": "0x01da033a", "imm2": "0x1e968e2b", "rot": 7, "bit": 22, "mask": 2}, - {"i": 212, "op": "shfl", "dst": 5, "src": 6, "src2": 6, "imm": "0x09f6758f", "imm2": "0x682f117e", "rot": 21, "bit": 30, "mask": 16}, - {"i": 213, "op": "rotl", "dst": 4, "src": 1, "src2": 2, "imm": "0xdb97b8fd", "imm2": "0x17c9d6e5", "rot": 10, "bit": 18, "mask": 8}, - {"i": 214, "op": "shfl", "dst": 5, "src": 6, "src2": 0, "imm": "0x7770463b", "imm2": "0x1b775f29", "rot": 8, "bit": 9, "mask": 16}, - {"i": 215, "op": "or", "dst": 3, "src": 0, "src2": 4, "imm": "0x030be847", "imm2": "0x28ce30d6", "rot": 24, "bit": 13, "mask": 8}, - {"i": 216, "op": "mul", "dst": 4, "src": 5, "src2": 0, "imm": "0xd57210c4", "imm2": "0xf11a023e", "rot": 20, "bit": 28, "mask": 8}, - {"i": 217, "op": "shfl", "dst": 0, "src": 3, "src2": 0, "imm": "0xe8de950c", "imm2": "0x84c408f4", "rot": 13, "bit": 22, "mask": 16}, - {"i": 218, "op": "mul", "dst": 0, "src": 5, "src2": 2, "imm": "0x034cca70", "imm2": "0x0dee748d", "rot": 17, "bit": 6, "mask": 1}, - {"i": 219, "op": "rotl", "dst": 0, "src": 3, "src2": 4, "imm": "0x4581784a", "imm2": "0x98998596", "rot": 13, "bit": 19, "mask": 2}, - {"i": 220, "op": "mulhi", "dst": 1, "src": 0, "src2": 5, "imm": "0x4d564305", "imm2": "0x5e5dac5e", "rot": 28, "bit": 31, "mask": 1}, - {"i": 221, "op": "xor", "dst": 4, "src": 1, "src2": 5, "imm": "0xdbf5ebeb", "imm2": "0xbf5e17eb", "rot": 29, "bit": 31, "mask": 8}, - {"i": 222, "op": "or", "dst": 2, "src": 3, "src2": 5, "imm": "0x15729ed6", "imm2": "0xa983f52f", "rot": 9, "bit": 19, "mask": 4}, - {"i": 223, "op": "xor", "dst": 0, "src": 4, "src2": 1, "imm": "0xffd11221", "imm2": "0xe34d6a6e", "rot": 21, "bit": 31, "mask": 16}, - {"i": 224, "op": "mad", "dst": 5, "src": 6, "src2": 1, "imm": "0x7e34f1f6", "imm2": "0x294c6931", "rot": 17, "bit": 30, "mask": 2}, - {"i": 225, "op": "xor", "dst": 4, "src": 3, "src2": 7, "imm": "0x01e76e71", "imm2": "0xe6632ffd", "rot": 18, "bit": 7, "mask": 8}, - {"i": 226, "op": "add", "dst": 5, "src": 0, "src2": 2, "imm": "0xfe960971", "imm2": "0xe4e51c75", "rot": 16, "bit": 3, "mask": 4}, - {"i": 227, "op": "mulhi", "dst": 3, "src": 2, "src2": 3, "imm": "0x63f4f95a", "imm2": "0x52cfc500", "rot": 4, "bit": 2, "mask": 2}, - {"i": 228, "op": "xor", "dst": 2, "src": 6, "src2": 3, "imm": "0x89e0458f", "imm2": "0x4fc30fcf", "rot": 25, "bit": 19, "mask": 8}, - {"i": 229, "op": "mulhi", "dst": 1, "src": 5, "src2": 5, "imm": "0x58ac55c8", "imm2": "0xdd7ec950", "rot": 27, "bit": 10, "mask": 8}, - {"i": 230, "op": "add", "dst": 3, "src": 4, "src2": 3, "imm": "0x48b3ce0a", "imm2": "0x4d597c08", "rot": 7, "bit": 25, "mask": 2}, - {"i": 231, "op": "rotr", "dst": 2, "src": 3, "src2": 7, "imm": "0xc215de10", "imm2": "0x40b73d08", "rot": 5, "bit": 11, "mask": 8}, - {"i": 232, "op": "sub", "dst": 2, "src": 7, "src2": 4, "imm": "0xca2afec4", "imm2": "0x576725b0", "rot": 12, "bit": 10, "mask": 8}, - {"i": 233, "op": "or", "dst": 6, "src": 2, "src2": 6, "imm": "0x4c44fac9", "imm2": "0x3a2576a0", "rot": 10, "bit": 3, "mask": 2}, - {"i": 234, "op": "rotr", "dst": 0, "src": 3, "src2": 6, "imm": "0xd20b4883", "imm2": "0xf5214ee2", "rot": 3, "bit": 20, "mask": 1}, - {"i": 235, "op": "add", "dst": 4, "src": 3, "src2": 5, "imm": "0x2ed8c878", "imm2": "0xc9f1d54c", "rot": 21, "bit": 13, "mask": 16}, - {"i": 236, "op": "mad", "dst": 0, "src": 3, "src2": 7, "imm": "0x43d96935", "imm2": "0x2c20b192", "rot": 7, "bit": 17, "mask": 4}, - {"i": 237, "op": "add", "dst": 3, "src": 5, "src2": 6, "imm": "0x22e8b90a", "imm2": "0xc572bd00", "rot": 10, "bit": 29, "mask": 2}, - {"i": 238, "op": "mulhi", "dst": 0, "src": 4, "src2": 2, "imm": "0xa2cbbdff", "imm2": "0x042f9ba6", "rot": 1, "bit": 14, "mask": 8}, - {"i": 239, "op": "mul", "dst": 6, "src": 5, "src2": 3, "imm": "0xcc024898", "imm2": "0x7b36c6a9", "rot": 14, "bit": 1, "mask": 2}, - {"i": 240, "op": "add", "dst": 5, "src": 2, "src2": 3, "imm": "0xc6b790e6", "imm2": "0xb233f94f", "rot": 12, "bit": 20, "mask": 16}, - {"i": 241, "op": "sub", "dst": 0, "src": 3, "src2": 0, "imm": "0x8e7ccb1c", "imm2": "0xae1d4c3a", "rot": 19, "bit": 29, "mask": 4}, - {"i": 242, "op": "add", "dst": 4, "src": 7, "src2": 4, "imm": "0x0f918d3b", "imm2": "0x534d924b", "rot": 1, "bit": 4, "mask": 8}, - {"i": 243, "op": "rotr", "dst": 5, "src": 0, "src2": 0, "imm": "0x5515f499", "imm2": "0x3a5a5a09", "rot": 20, "bit": 28, "mask": 8}, - {"i": 244, "op": "rotr", "dst": 1, "src": 7, "src2": 2, "imm": "0x15de3e44", "imm2": "0x2d00a4c8", "rot": 6, "bit": 7, "mask": 1}, - {"i": 245, "op": "or", "dst": 2, "src": 0, "src2": 4, "imm": "0xc1d19687", "imm2": "0xa3f06c6f", "rot": 9, "bit": 19, "mask": 2}, - {"i": 246, "op": "mul", "dst": 4, "src": 7, "src2": 4, "imm": "0x09250afe", "imm2": "0xb09720b1", "rot": 1, "bit": 2, "mask": 8}, - {"i": 247, "op": "mad", "dst": 1, "src": 2, "src2": 5, "imm": "0xb6793fd5", "imm2": "0x32837c74", "rot": 13, "bit": 28, "mask": 1}, - {"i": 248, "op": "mul", "dst": 2, "src": 5, "src2": 0, "imm": "0x1f0bde8f", "imm2": "0x98e78f9b", "rot": 15, "bit": 16, "mask": 8}, - {"i": 249, "op": "xor", "dst": 4, "src": 2, "src2": 6, "imm": "0x3923d007", "imm2": "0x9357d221", "rot": 23, "bit": 24, "mask": 8}, - {"i": 250, "op": "xor", "dst": 3, "src": 1, "src2": 1, "imm": "0xaabf43a0", "imm2": "0xbe559b93", "rot": 21, "bit": 30, "mask": 16}, - {"i": 251, "op": "rotr", "dst": 5, "src": 7, "src2": 0, "imm": "0x4aa1bf2a", "imm2": "0x5b33ea1c", "rot": 24, "bit": 4, "mask": 8}, - {"i": 252, "op": "mad", "dst": 2, "src": 7, "src2": 7, "imm": "0xef234434", "imm2": "0x96d5854d", "rot": 15, "bit": 30, "mask": 2}, - {"i": 253, "op": "rotr", "dst": 7, "src": 3, "src2": 4, "imm": "0x4ea7e652", "imm2": "0x6b77a754", "rot": 16, "bit": 19, "mask": 2}, - {"i": 254, "op": "add", "dst": 6, "src": 1, "src2": 1, "imm": "0xb8a27d95", "imm2": "0x86d27169", "rot": 8, "bit": 21, "mask": 4}, - {"i": 255, "op": "or", "dst": 6, "src": 7, "src2": 1, "imm": "0x2e74a663", "imm2": "0x45dff7ca", "rot": 25, "bit": 2, "mask": 1} - ]}, - "shadow_placement": "per_load", - "instructions": [ - {"i": 0, "op": "add", "dst": 2, "src": 5, "src2": 3, "imm": "0xe3e2ed7d", "imm2": "0x894e457d", "rot": 13, "bit": 2, "mask": 1, "width": 1}, - {"i": 1, "op": "mad", "dst": 7, "src": 4, "src2": 0, "imm": "0x48baccba", "imm2": "0x6e9738d1", "rot": 21, "bit": 30, "mask": 8, "width": 1}, - {"i": 2, "op": "sub", "dst": 3, "src": 6, "src2": 0, "imm": "0x686a83d3", "imm2": "0xeb5efcc2", "rot": 10, "bit": 10, "mask": 16, "width": 1}, - {"i": 3, "op": "rotr", "dst": 5, "src": 1, "src2": 0, "imm": "0xb2c12490", "imm2": "0x7f13e52b", "rot": 25, "bit": 8, "mask": 4, "width": 1}, - {"i": 4, "op": "shfl", "dst": 6, "src": 2, "src2": 6, "imm": "0xc8f420a8", "imm2": "0x608237a0", "rot": 26, "bit": 11, "mask": 1, "width": 1}, - {"i": 5, "op": "or", "dst": 0, "src": 3, "src2": 7, "imm": "0xc7e2251e", "imm2": "0x3e36b8d5", "rot": 30, "bit": 7, "mask": 1, "width": 1}, - {"i": 6, "op": "mul", "dst": 0, "src": 1, "src2": 5, "imm": "0xa5b4da15", "imm2": "0x7cb74643", "rot": 16, "bit": 20, "mask": 2, "width": 1}, - {"i": 7, "op": "mul", "dst": 7, "src": 6, "src2": 4, "imm": "0x013ce090", "imm2": "0xb938a092", "rot": 9, "bit": 29, "mask": 4, "width": 1}, - {"i": 8, "op": "load", "dst": 3, "src": 0, "src2": 6, "imm": "0xf604ccf0", "imm2": "0xf02dffb7", "rot": 26, "bit": 28, "mask": 1, "width": 1}, - {"i": 9, "op": "add", "dst": 4, "src": 3, "src2": 3, "imm": "0xce9bea84", "imm2": "0xd26d3573", "rot": 20, "bit": 22, "mask": 1, "width": 1}, - {"i": 10, "op": "mul", "dst": 2, "src": 4, "src2": 6, "imm": "0x98ac1503", "imm2": "0xb3626dcf", "rot": 22, "bit": 13, "mask": 2, "width": 1}, - {"i": 11, "op": "load", "dst": 3, "src": 2, "src2": 2, "imm": "0xdb84e449", "imm2": "0x538dd18d", "rot": 30, "bit": 5, "mask": 2, "width": 1}, - {"i": 12, "op": "mulhi", "dst": 4, "src": 1, "src2": 7, "imm": "0x7aa83111", "imm2": "0x2a98226a", "rot": 26, "bit": 3, "mask": 1, "width": 1}, - {"i": 13, "op": "mulhi", "dst": 7, "src": 3, "src2": 1, "imm": "0xd2c17bd2", "imm2": "0x2876c049", "rot": 1, "bit": 11, "mask": 16, "width": 1}, - {"i": 14, "op": "add", "dst": 3, "src": 2, "src2": 6, "imm": "0x514f9ff4", "imm2": "0xc2828a42", "rot": 28, "bit": 8, "mask": 4, "width": 1}, - {"i": 15, "op": "mad", "dst": 1, "src": 3, "src2": 2, "imm": "0xf76025ba", "imm2": "0x82d27507", "rot": 31, "bit": 18, "mask": 16, "width": 1}, - {"i": 16, "op": "rotl", "dst": 2, "src": 5, "src2": 0, "imm": "0x7c61ca9c", "imm2": "0x36277625", "rot": 7, "bit": 8, "mask": 8, "width": 1}, - {"i": 17, "op": "load", "dst": 1, "src": 2, "src2": 3, "imm": "0x740a280b", "imm2": "0x383281a7", "rot": 27, "bit": 31, "mask": 16, "width": 1}, - {"i": 18, "op": "mul", "dst": 3, "src": 2, "src2": 6, "imm": "0x064bd0b5", "imm2": "0xd1c0fc47", "rot": 25, "bit": 31, "mask": 16, "width": 1}, - {"i": 19, "op": "rotl", "dst": 6, "src": 7, "src2": 4, "imm": "0xab459e33", "imm2": "0x95f59404", "rot": 24, "bit": 23, "mask": 4, "width": 1}, - {"i": 20, "op": "load", "dst": 6, "src": 3, "src2": 3, "imm": "0x016542dd", "imm2": "0x78d9e35c", "rot": 23, "bit": 4, "mask": 2, "width": 1}, - {"i": 21, "op": "load", "dst": 1, "src": 6, "src2": 4, "imm": "0x3b09488c", "imm2": "0x5dff13e2", "rot": 14, "bit": 29, "mask": 4, "width": 1}, - {"i": 22, "op": "mad", "dst": 5, "src": 3, "src2": 3, "imm": "0x07524f6c", "imm2": "0x1618637f", "rot": 16, "bit": 19, "mask": 1, "width": 1}, - {"i": 23, "op": "load", "dst": 4, "src": 7, "src2": 0, "imm": "0x32a5d5e9", "imm2": "0x375464f7", "rot": 6, "bit": 18, "mask": 4, "width": 1}, - {"i": 24, "op": "load", "dst": 6, "src": 4, "src2": 1, "imm": "0xa0e15b48", "imm2": "0x656763e3", "rot": 3, "bit": 12, "mask": 16, "width": 1}, - {"i": 25, "op": "mul", "dst": 5, "src": 7, "src2": 0, "imm": "0xa79e75f0", "imm2": "0x6a5e8dac", "rot": 13, "bit": 7, "mask": 2, "width": 1}, - {"i": 26, "op": "mul", "dst": 0, "src": 3, "src2": 1, "imm": "0xe3a32543", "imm2": "0x40df802b", "rot": 10, "bit": 19, "mask": 1, "width": 1}, - {"i": 27, "op": "or", "dst": 0, "src": 3, "src2": 7, "imm": "0x58c20b95", "imm2": "0x5bfa6339", "rot": 23, "bit": 17, "mask": 2, "width": 1}, - {"i": 28, "op": "mad", "dst": 1, "src": 3, "src2": 4, "imm": "0xf8cf6f87", "imm2": "0x40bf2b52", "rot": 12, "bit": 22, "mask": 8, "width": 1}, - {"i": 29, "op": "shfl", "dst": 0, "src": 2, "src2": 6, "imm": "0x6f84cf36", "imm2": "0x7936172c", "rot": 2, "bit": 2, "mask": 8, "width": 1}, - {"i": 30, "op": "sub", "dst": 7, "src": 3, "src2": 2, "imm": "0xb77c8c79", "imm2": "0x525a96d8", "rot": 16, "bit": 16, "mask": 8, "width": 1}, - {"i": 31, "op": "xor", "dst": 4, "src": 1, "src2": 4, "imm": "0xad6463e3", "imm2": "0x3874789b", "rot": 27, "bit": 28, "mask": 1, "width": 1}, - {"i": 32, "op": "or", "dst": 4, "src": 5, "src2": 1, "imm": "0x4d700827", "imm2": "0x2dbc3dc4", "rot": 10, "bit": 30, "mask": 16, "width": 1}, - {"i": 33, "op": "rotr", "dst": 3, "src": 5, "src2": 0, "imm": "0x719a64fe", "imm2": "0xb6ba21b7", "rot": 1, "bit": 25, "mask": 1, "width": 1}, - {"i": 34, "op": "load", "dst": 4, "src": 5, "src2": 7, "imm": "0x7af41ac2", "imm2": "0x043993ef", "rot": 1, "bit": 31, "mask": 1, "width": 1}, - {"i": 35, "op": "mad", "dst": 0, "src": 5, "src2": 3, "imm": "0xaa75a56c", "imm2": "0x98eedda3", "rot": 19, "bit": 28, "mask": 16, "width": 1}, - {"i": 36, "op": "load", "dst": 6, "src": 3, "src2": 3, "imm": "0xb30a9fef", "imm2": "0xae0b99d7", "rot": 10, "bit": 26, "mask": 1, "width": 1}, - {"i": 37, "op": "mul", "dst": 5, "src": 3, "src2": 6, "imm": "0x5bda14ce", "imm2": "0x93932ff3", "rot": 2, "bit": 7, "mask": 1, "width": 1}, - {"i": 38, "op": "xor", "dst": 5, "src": 4, "src2": 5, "imm": "0x6087cdca", "imm2": "0x5ade3557", "rot": 7, "bit": 13, "mask": 1, "width": 1}, - {"i": 39, "op": "xor", "dst": 6, "src": 0, "src2": 4, "imm": "0x618cbb10", "imm2": "0x3a79c600", "rot": 1, "bit": 21, "mask": 8, "width": 1}, - {"i": 40, "op": "rotr", "dst": 4, "src": 0, "src2": 5, "imm": "0x206b88d8", "imm2": "0x25c6e9b5", "rot": 25, "bit": 1, "mask": 16, "width": 1}, - {"i": 41, "op": "xor", "dst": 7, "src": 6, "src2": 7, "imm": "0x5e1baea1", "imm2": "0xa6016845", "rot": 30, "bit": 16, "mask": 8, "width": 1}, - {"i": 42, "op": "load", "dst": 1, "src": 7, "src2": 1, "imm": "0xca654c28", "imm2": "0x1d5f5e32", "rot": 29, "bit": 12, "mask": 1, "width": 1}, - {"i": 43, "op": "sub", "dst": 2, "src": 4, "src2": 7, "imm": "0x6abb678b", "imm2": "0x062adc76", "rot": 1, "bit": 20, "mask": 1, "width": 1}, - {"i": 44, "op": "mulhi", "dst": 6, "src": 7, "src2": 0, "imm": "0xfef225a8", "imm2": "0x9fb4c8ef", "rot": 7, "bit": 15, "mask": 4, "width": 1}, - {"i": 45, "op": "rotl", "dst": 3, "src": 1, "src2": 2, "imm": "0xc8dff63b", "imm2": "0xfc726054", "rot": 13, "bit": 22, "mask": 4, "width": 1}, - {"i": 46, "op": "shfl", "dst": 7, "src": 6, "src2": 2, "imm": "0x83bf4499", "imm2": "0xd1f9aaeb", "rot": 13, "bit": 22, "mask": 16, "width": 1}, - {"i": 47, "op": "rotl", "dst": 2, "src": 7, "src2": 6, "imm": "0x592d1583", "imm2": "0x7d216bb4", "rot": 26, "bit": 23, "mask": 8, "width": 1}, - {"i": 48, "op": "mul", "dst": 6, "src": 2, "src2": 5, "imm": "0xa2d3a9c5", "imm2": "0xe46c97c7", "rot": 8, "bit": 4, "mask": 1, "width": 1}, - {"i": 49, "op": "load", "dst": 2, "src": 3, "src2": 5, "imm": "0xc97cc7d0", "imm2": "0x11de360b", "rot": 24, "bit": 4, "mask": 4, "width": 1}, - {"i": 50, "op": "load", "dst": 5, "src": 1, "src2": 5, "imm": "0x51d9e142", "imm2": "0x4949e464", "rot": 6, "bit": 21, "mask": 8, "width": 1}, - {"i": 51, "op": "rotl", "dst": 0, "src": 7, "src2": 5, "imm": "0x47ed408a", "imm2": "0xfbc78474", "rot": 20, "bit": 0, "mask": 1, "width": 1}, - {"i": 52, "op": "xor", "dst": 1, "src": 2, "src2": 0, "imm": "0x548a8b43", "imm2": "0x05fa3627", "rot": 24, "bit": 26, "mask": 2, "width": 1}, - {"i": 53, "op": "load", "dst": 2, "src": 1, "src2": 3, "imm": "0x13137433", "imm2": "0x4da0f56c", "rot": 18, "bit": 10, "mask": 4, "width": 1}, - {"i": 54, "op": "rotl", "dst": 4, "src": 3, "src2": 4, "imm": "0xeb8027b8", "imm2": "0x672311d2", "rot": 20, "bit": 19, "mask": 1, "width": 1}, - {"i": 55, "op": "shfl", "dst": 3, "src": 1, "src2": 7, "imm": "0x273a617a", "imm2": "0x38456825", "rot": 21, "bit": 15, "mask": 2, "width": 1}, - {"i": 56, "op": "load", "dst": 1, "src": 0, "src2": 3, "imm": "0x80aaf238", "imm2": "0xcb72fcc7", "rot": 25, "bit": 21, "mask": 4, "width": 1}, - {"i": 57, "op": "load", "dst": 3, "src": 5, "src2": 5, "imm": "0xc3797c55", "imm2": "0x5ff4d264", "rot": 13, "bit": 10, "mask": 2, "width": 1}, - {"i": 58, "op": "shfl", "dst": 1, "src": 2, "src2": 6, "imm": "0xc24867f2", "imm2": "0x41627f2d", "rot": 24, "bit": 31, "mask": 2, "width": 1}, - {"i": 59, "op": "load", "dst": 6, "src": 4, "src2": 6, "imm": "0x4e68a668", "imm2": "0x47cb76dc", "rot": 6, "bit": 9, "mask": 2, "width": 1}, - {"i": 60, "op": "shfl", "dst": 7, "src": 3, "src2": 3, "imm": "0x1b39c5c8", "imm2": "0x8f3693ee", "rot": 10, "bit": 22, "mask": 4, "width": 1}, - {"i": 61, "op": "rotr", "dst": 5, "src": 0, "src2": 0, "imm": "0x85b99d10", "imm2": "0x253b1522", "rot": 26, "bit": 31, "mask": 4, "width": 1}, - {"i": 62, "op": "add", "dst": 0, "src": 6, "src2": 1, "imm": "0x8c2e5c24", "imm2": "0xb13a5391", "rot": 23, "bit": 14, "mask": 16, "width": 1}, - {"i": 63, "op": "add", "dst": 2, "src": 1, "src2": 7, "imm": "0xb225b762", "imm2": "0xf82fc8b5", "rot": 11, "bit": 21, "mask": 1, "width": 1} - ] -} diff --git a/proto-cuda/packs-ca4/mx8_shl256x27_v2/program.metal b/proto-cuda/packs-ca4/mx8_shl256x27_v2/program.metal deleted file mode 100644 index 45e42e15c..000000000 --- a/proto-cuda/packs-ca4/mx8_shl256x27_v2/program.metal +++ /dev/null @@ -1,413 +0,0 @@ -#include -using namespace metal; - -#define MASK 0x0fffffffu -constant uint SEEDW[8] = { 0xf71aee9fu, 0xad930c88u, 0x7f982573u, 0xa41f9137u, 0x76d803d6u, 0x37b4a534u, 0x4d3fb826u, 0xff614dcbu }; - -inline uint splitmix32(uint x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -inline uint rotl_imm(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 -inline uint rotr_var(uint x, uint n) { n &= 31u; return (x >> n) | (x << ((32u - n) & 31u)); } -inline uint ds_elem(uint i, uint d0, uint d1) { - uint x = i ^ d0; - x *= 0x9E3779B1u; x ^= x >> 15; - x += d1; - x *= 0x85EBCA77u; x ^= x >> 13; - x *= 0xC2B2AE3Du; x ^= x >> 16; - return x; -} - -kernel void igneum_hash(device const uint* dataset [[buffer(0)]], - device ulong* out [[buffer(1)]], - constant uint& baseNonce [[buffer(2)]], - uint gid [[thread_position_in_grid]]) { - uint nonce = baseNonce + gid; - uint r0, r1, r2, r3, r4, r5, r6, r7; - { uint x = nonce ^ SEEDW[0]; x += 0x9e3779b9u * 1u; x = splitmix32(x); r0 = x ^ SEEDW[1]; } - { uint x = nonce ^ SEEDW[1]; x += 0x9e3779b9u * 2u; x = splitmix32(x); r1 = x ^ SEEDW[2]; } - { uint x = nonce ^ SEEDW[2]; x += 0x9e3779b9u * 3u; x = splitmix32(x); r2 = x ^ SEEDW[3]; } - { uint x = nonce ^ SEEDW[3]; x += 0x9e3779b9u * 4u; x = splitmix32(x); r3 = x ^ SEEDW[4]; } - { uint x = nonce ^ SEEDW[4]; x += 0x9e3779b9u * 5u; x = splitmix32(x); r4 = x ^ SEEDW[5]; } - { uint x = nonce ^ SEEDW[5]; x += 0x9e3779b9u * 6u; x = splitmix32(x); r5 = x ^ SEEDW[6]; } - { uint x = nonce ^ SEEDW[6]; x += 0x9e3779b9u * 7u; x = splitmix32(x); r6 = x ^ SEEDW[7]; } - { uint x = nonce ^ SEEDW[7]; x += 0x9e3779b9u * 8u; x = splitmix32(x); r7 = x ^ SEEDW[0]; } - - for (uint it = 0u; it < 8u; ++it) { - uint sel = r0; - r2 = r2 + r5 + select(0xe3e2ed7du, 0x894e457du, ((sel >> 2u) & 1u) != 0u); // 0 - r7 = r4 * r0 + r7; // 1 - r3 = r3 - r6; // 2 - r5 = rotr_var(r5, r1); // 3 - r6 = r6 ^ simd_shuffle_xor(r2, (ushort)1); // 4 - r0 = r0 | r3; // 5 - r0 = r0 * r1; // 6 - r7 = r7 * r6; // 7 - r3 = r3 ^ dataset[r0 & MASK]; // 8 - // per-load shadow sub-block 0 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 0 - for (uint sh0 = 0u; sh0 < 27u; ++sh0) { - r5 = rotr_var(r5, r4); // s0 rotr - r6 = r6 - r5; // s1 sub - r7 = r7 - r6; // s2 sub - r5 = r5 * r4; // s3 mul - r7 = r7 ^ r6; // s4 xor - r2 = rotl_imm(r2, 23u); // s5 rotl - r7 = r7 ^ r4; // s6 xor - r0 = r0 + r7 + select(0xdaef8862u, 0x3051c491u, ((sel >> 26u) & 1u) != 0u); // s7 add - r6 = r6 ^ simd_shuffle_xor(r1, (ushort)4); // s8 shfl - r5 = r5 + r1 + select(0xc20e7045u, 0x5170d0b3u, ((sel >> 26u) & 1u) != 0u); // s9 add - r0 = r0 | r3; // s10 or - r5 = mulhi(r5, r4); // s11 mulhi - r7 = r7 ^ r0; // s12 xor - r5 = r5 - r2; // s13 sub - r0 = r0 + r6 + select(0xc86d98a4u, 0x2ead087fu, ((sel >> 0u) & 1u) != 0u); // s14 add - r5 = r5 ^ r7; // s15 xor - } - r4 = r4 + r3 + select(0xce9bea84u, 0xd26d3573u, ((sel >> 22u) & 1u) != 0u); // 9 - r2 = r2 * r4; // 10 - r3 = r3 ^ dataset[r2 & MASK]; // 11 - // per-load shadow sub-block 1 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 1 - for (uint sh1 = 0u; sh1 < 27u; ++sh1) { - r7 = r7 ^ simd_shuffle_xor(r6, (ushort)8); // s16 shfl - r5 = r5 - r7; // s17 sub - r0 = r0 | r6; // s18 or - r0 = r0 + r5 + select(0x4248b651u, 0xc47c70a8u, ((sel >> 13u) & 1u) != 0u); // s19 add - r4 = r4 ^ simd_shuffle_xor(r5, (ushort)16); // s20 shfl - r0 = r0 + r4 + select(0x1e35684fu, 0x718c1008u, ((sel >> 21u) & 1u) != 0u); // s21 add - r0 = mulhi(r0, r3); // s22 mulhi - r4 = r4 ^ r2; // s23 xor - r0 = r0 - r2; // s24 sub - r5 = rotl_imm(r5, 13u); // s25 rotl - r7 = r7 - r0; // s26 sub - r2 = r2 ^ r3; // s27 xor - r2 = r3 * r2 + r2; // s28 mad - r2 = r2 ^ r1; // s29 xor - r4 = mulhi(r4, r0); // s30 mulhi - r6 = r6 ^ r4; // s31 xor - } - r4 = mulhi(r4, r1); // 12 - r7 = mulhi(r7, r3); // 13 - r3 = r3 + r2 + select(0x514f9ff4u, 0xc2828a42u, ((sel >> 8u) & 1u) != 0u); // 14 - r1 = r3 * r2 + r1; // 15 - r2 = rotl_imm(r2, 7u); // 16 - r1 = r1 ^ dataset[r2 & MASK]; // 17 - // per-load shadow sub-block 2 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 2 - for (uint sh2 = 0u; sh2 < 27u; ++sh2) { - r4 = r4 ^ r2; // s32 xor - r1 = r1 ^ simd_shuffle_xor(r2, (ushort)1); // s33 shfl - r5 = r5 ^ simd_shuffle_xor(r6, (ushort)1); // s34 shfl - r4 = r4 - r0; // s35 sub - r6 = r6 - r3; // s36 sub - r2 = r2 | r6; // s37 or - r2 = rotl_imm(r2, 30u); // s38 rotl - r4 = rotr_var(r4, r7); // s39 rotr - r0 = r0 | r7; // s40 or - r2 = rotr_var(r2, r5); // s41 rotr - r1 = r3 * r3 + r1; // s42 mad - r6 = r5 * r5 + r6; // s43 mad - r1 = mulhi(r1, r0); // s44 mulhi - r1 = r1 * r3; // s45 mul - r0 = r2 * r0 + r0; // s46 mad - r1 = r1 - r5; // s47 sub - } - r3 = r3 * r2; // 18 - r6 = rotl_imm(r6, 24u); // 19 - r6 = r6 ^ dataset[r3 & MASK]; // 20 - // per-load shadow sub-block 3 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 3 - for (uint sh3 = 0u; sh3 < 27u; ++sh3) { - r4 = r4 ^ r0; // s48 xor - r2 = rotr_var(r2, r0); // s49 rotr - r0 = mulhi(r0, r5); // s50 mulhi - r7 = r7 | r5; // s51 or - r4 = r0 * r3 + r4; // s52 mad - r0 = rotr_var(r0, r4); // s53 rotr - r6 = r3 * r3 + r6; // s54 mad - r6 = r4 * r2 + r6; // s55 mad - r5 = r5 ^ simd_shuffle_xor(r0, (ushort)2); // s56 shfl - r7 = rotl_imm(r7, 21u); // s57 rotl - r3 = r3 ^ r7; // s58 xor - r0 = r0 ^ simd_shuffle_xor(r4, (ushort)1); // s59 shfl - r5 = r5 ^ r6; // s60 xor - r4 = rotr_var(r4, r7); // s61 rotr - r1 = r1 + r6 + select(0xf7ccf1e9u, 0x31f87d74u, ((sel >> 10u) & 1u) != 0u); // s62 add - r5 = r5 + r2 + select(0x2f386099u, 0x9c2847f4u, ((sel >> 15u) & 1u) != 0u); // s63 add - } - r1 = r1 ^ dataset[r6 & MASK]; // 21 - // per-load shadow sub-block 4 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 4 - for (uint sh4 = 0u; sh4 < 27u; ++sh4) { - r7 = r7 + r0 + select(0x1b30ce7au, 0xd3241188u, ((sel >> 19u) & 1u) != 0u); // s64 add - r6 = r6 + r7 + select(0x02dc8349u, 0x9b42c3deu, ((sel >> 13u) & 1u) != 0u); // s65 add - r0 = rotl_imm(r0, 16u); // s66 rotl - r2 = rotl_imm(r2, 9u); // s67 rotl - r2 = r5 * r6 + r2; // s68 mad - r6 = r6 ^ simd_shuffle_xor(r1, (ushort)8); // s69 shfl - r2 = r2 ^ r5; // s70 xor - r5 = mulhi(r5, r1); // s71 mulhi - r6 = rotl_imm(r6, 29u); // s72 rotl - r0 = r0 - r7; // s73 sub - r5 = r2 * r2 + r5; // s74 mad - r0 = rotr_var(r0, r5); // s75 rotr - r7 = r7 + r2 + select(0x05d36679u, 0x0da1ce17u, ((sel >> 9u) & 1u) != 0u); // s76 add - r7 = r7 - r2; // s77 sub - r7 = rotr_var(r7, r0); // s78 rotr - r1 = r5 * r5 + r1; // s79 mad - } - r5 = r3 * r3 + r5; // 22 - r4 = r4 ^ dataset[r7 & MASK]; // 23 - // per-load shadow sub-block 5 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 5 - for (uint sh5 = 0u; sh5 < 27u; ++sh5) { - r3 = rotr_var(r3, r0); // s80 rotr - r6 = r6 + r5 + select(0xf14547bdu, 0xadf5ef88u, ((sel >> 2u) & 1u) != 0u); // s81 add - r0 = r0 | r6; // s82 or - r1 = mulhi(r1, r0); // s83 mulhi - r7 = r6 * r7 + r7; // s84 mad - r5 = rotl_imm(r5, 29u); // s85 rotl - r2 = r2 ^ r6; // s86 xor - r5 = r5 + r4 + select(0xba4947c2u, 0x35ff14aeu, ((sel >> 24u) & 1u) != 0u); // s87 add - r3 = r3 + r6 + select(0xf6878beeu, 0xf3900fc1u, ((sel >> 3u) & 1u) != 0u); // s88 add - r0 = r0 + r2 + select(0x926f3607u, 0xf7e8f59fu, ((sel >> 12u) & 1u) != 0u); // s89 add - r2 = r2 ^ r6; // s90 xor - r3 = r3 ^ simd_shuffle_xor(r2, (ushort)2); // s91 shfl - r6 = r6 - r1; // s92 sub - r5 = r5 + r1 + select(0xdab36c29u, 0x0e376f9cu, ((sel >> 13u) & 1u) != 0u); // s93 add - r6 = r1 * r1 + r6; // s94 mad - r3 = r3 ^ r4; // s95 xor - } - r6 = r6 ^ dataset[r4 & MASK]; // 24 - // per-load shadow sub-block 6 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 6 - for (uint sh6 = 0u; sh6 < 27u; ++sh6) { - r6 = r5 * r2 + r6; // s96 mad - r7 = r7 + r3 + select(0x204129e9u, 0x9f917747u, ((sel >> 23u) & 1u) != 0u); // s97 add - r5 = r5 ^ simd_shuffle_xor(r1, (ushort)16); // s98 shfl - r2 = r2 * r6; // s99 mul - r7 = r7 ^ simd_shuffle_xor(r3, (ushort)8); // s100 shfl - r6 = r6 + r1 + select(0xe3b596a0u, 0xe42fe974u, ((sel >> 10u) & 1u) != 0u); // s101 add - r5 = rotl_imm(r5, 3u); // s102 rotl - r5 = r5 ^ r1; // s103 xor - r4 = r4 + r1 + select(0x705c3f94u, 0xc3d77ffbu, ((sel >> 27u) & 1u) != 0u); // s104 add - r0 = r0 + r6 + select(0xbc6fb42bu, 0xea02a4cau, ((sel >> 25u) & 1u) != 0u); // s105 add - r3 = r3 | r5; // s106 or - r0 = r0 - r6; // s107 sub - r0 = rotr_var(r0, r3); // s108 rotr - r5 = r5 + r4 + select(0xcb37ea87u, 0x0357e63eu, ((sel >> 4u) & 1u) != 0u); // s109 add - r7 = rotl_imm(r7, 5u); // s110 rotl - r4 = r4 ^ r1; // s111 xor - } - r5 = r5 * r7; // 25 - r0 = r0 * r3; // 26 - r0 = r0 | r3; // 27 - r1 = r3 * r4 + r1; // 28 - r0 = r0 ^ simd_shuffle_xor(r2, (ushort)8); // 29 - r7 = r7 - r3; // 30 - r4 = r4 ^ r1; // 31 - r4 = r4 | r5; // 32 - r3 = rotr_var(r3, r5); // 33 - r4 = r4 ^ dataset[r5 & MASK]; // 34 - // per-load shadow sub-block 7 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 7 - for (uint sh7 = 0u; sh7 < 27u; ++sh7) { - r6 = r6 ^ r5; // s112 xor - r1 = r1 | r3; // s113 or - r0 = r0 + r5 + select(0x9aab0297u, 0x7f8b8cd2u, ((sel >> 3u) & 1u) != 0u); // s114 add - r3 = r3 - r5; // s115 sub - r4 = rotr_var(r4, r2); // s116 rotr - r4 = r4 + r6 + select(0xc511183fu, 0x59400b57u, ((sel >> 6u) & 1u) != 0u); // s117 add - r6 = r6 - r1; // s118 sub - r3 = r3 ^ simd_shuffle_xor(r1, (ushort)4); // s119 shfl - r6 = r4 * r4 + r6; // s120 mad - r2 = rotl_imm(r2, 8u); // s121 rotl - r5 = r0 * r3 + r5; // s122 mad - r7 = r7 + r6 + select(0x5200c242u, 0x27c70dc0u, ((sel >> 27u) & 1u) != 0u); // s123 add - r3 = r3 ^ simd_shuffle_xor(r5, (ushort)4); // s124 shfl - r0 = r0 + r1 + select(0x2d5028ceu, 0xec2f5997u, ((sel >> 17u) & 1u) != 0u); // s125 add - r3 = r3 ^ simd_shuffle_xor(r6, (ushort)16); // s126 shfl - r5 = r5 ^ r0; // s127 xor - } - r0 = r5 * r3 + r0; // 35 - r6 = r6 ^ dataset[r3 & MASK]; // 36 - // per-load shadow sub-block 8 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 8 - for (uint sh8 = 0u; sh8 < 27u; ++sh8) { - r7 = r7 + r1 + select(0x432f6c0du, 0x09e8ede2u, ((sel >> 10u) & 1u) != 0u); // s128 add - r2 = r2 ^ r4; // s129 xor - r0 = r0 * r2; // s130 mul - r4 = r4 | r3; // s131 or - r3 = r3 | r7; // s132 or - r7 = r7 ^ simd_shuffle_xor(r3, (ushort)1); // s133 shfl - r5 = r5 + r2 + select(0xcf5ecc75u, 0xce4fdf8eu, ((sel >> 11u) & 1u) != 0u); // s134 add - r3 = r2 * r3 + r3; // s135 mad - r1 = rotl_imm(r1, 11u); // s136 rotl - r2 = r2 | r0; // s137 or - r3 = r3 ^ r4; // s138 xor - r2 = r2 ^ simd_shuffle_xor(r4, (ushort)1); // s139 shfl - r2 = rotl_imm(r2, 6u); // s140 rotl - r0 = r0 * r3; // s141 mul - r1 = r1 * r7; // s142 mul - r0 = r0 ^ r1; // s143 xor - } - r5 = r5 * r3; // 37 - r5 = r5 ^ r4; // 38 - r6 = r6 ^ r0; // 39 - r4 = rotr_var(r4, r0); // 40 - r7 = r7 ^ r6; // 41 - r1 = r1 ^ dataset[r7 & MASK]; // 42 - // per-load shadow sub-block 9 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 9 - for (uint sh9 = 0u; sh9 < 27u; ++sh9) { - r6 = r2 * r0 + r6; // s144 mad - r1 = rotl_imm(r1, 15u); // s145 rotl - r1 = rotl_imm(r1, 30u); // s146 rotl - r6 = r6 ^ r0; // s147 xor - r3 = r3 - r2; // s148 sub - r3 = r3 + r0 + select(0xaa002f15u, 0x11bbdeefu, ((sel >> 5u) & 1u) != 0u); // s149 add - r6 = r6 + r5 + select(0xbe64b2e2u, 0xff9d4b0eu, ((sel >> 9u) & 1u) != 0u); // s150 add - r1 = r1 * r4; // s151 mul - r5 = rotl_imm(r5, 3u); // s152 rotl - r5 = r5 * r2; // s153 mul - r2 = rotl_imm(r2, 21u); // s154 rotl - r5 = rotl_imm(r5, 18u); // s155 rotl - r1 = r1 ^ r5; // s156 xor - r1 = mulhi(r1, r6); // s157 mulhi - r4 = r7 * r3 + r4; // s158 mad - r4 = r4 - r7; // s159 sub - } - r2 = r2 - r4; // 43 - r6 = mulhi(r6, r7); // 44 - r3 = rotl_imm(r3, 13u); // 45 - r7 = r7 ^ simd_shuffle_xor(r6, (ushort)16); // 46 - r2 = rotl_imm(r2, 26u); // 47 - r6 = r6 * r2; // 48 - r2 = r2 ^ dataset[r3 & MASK]; // 49 - // per-load shadow sub-block 10 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 10 - for (uint sh10 = 0u; sh10 < 27u; ++sh10) { - r3 = r3 ^ r2; // s160 xor - r3 = mulhi(r3, r4); // s161 mulhi - r3 = r3 ^ simd_shuffle_xor(r2, (ushort)1); // s162 shfl - r0 = r0 - r1; // s163 sub - r6 = r6 ^ simd_shuffle_xor(r0, (ushort)8); // s164 shfl - r4 = r4 ^ simd_shuffle_xor(r2, (ushort)2); // s165 shfl - r3 = r3 | r6; // s166 or - r6 = r6 ^ simd_shuffle_xor(r3, (ushort)4); // s167 shfl - r2 = r2 ^ r1; // s168 xor - r5 = r5 ^ r1; // s169 xor - r5 = r5 ^ r0; // s170 xor - r3 = r6 * r0 + r3; // s171 mad - r3 = r3 - r0; // s172 sub - r6 = r6 + r0 + select(0x3b90694bu, 0xc72dc2a0u, ((sel >> 27u) & 1u) != 0u); // s173 add - r7 = r7 ^ simd_shuffle_xor(r2, (ushort)8); // s174 shfl - r5 = mulhi(r5, r2); // s175 mulhi - } - r5 = r5 ^ dataset[r1 & MASK]; // 50 - // per-load shadow sub-block 11 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 11 - for (uint sh11 = 0u; sh11 < 27u; ++sh11) { - r3 = r3 + r6 + select(0x9e65cebdu, 0x4eb75843u, ((sel >> 5u) & 1u) != 0u); // s176 add - r0 = r4 * r1 + r0; // s177 mad - r6 = rotr_var(r6, r7); // s178 rotr - r0 = r0 + r7 + select(0x01e5b250u, 0x7001d036u, ((sel >> 31u) & 1u) != 0u); // s179 add - r5 = r5 ^ simd_shuffle_xor(r2, (ushort)16); // s180 shfl - r3 = r3 * r0; // s181 mul - r0 = rotl_imm(r0, 23u); // s182 rotl - r7 = mulhi(r7, r0); // s183 mulhi - r0 = r0 * r4; // s184 mul - r1 = r1 + r4 + select(0xbef14988u, 0x91736711u, ((sel >> 19u) & 1u) != 0u); // s185 add - r7 = r7 + r2 + select(0xf5d741beu, 0x15e0cdf3u, ((sel >> 6u) & 1u) != 0u); // s186 add - r1 = r1 ^ r7; // s187 xor - r1 = r1 + r3 + select(0x32be33c6u, 0x7549bc3eu, ((sel >> 21u) & 1u) != 0u); // s188 add - r1 = r0 * r6 + r1; // s189 mad - r0 = rotr_var(r0, r5); // s190 rotr - r4 = r0 * r1 + r4; // s191 mad - } - r0 = rotl_imm(r0, 20u); // 51 - r1 = r1 ^ r2; // 52 - r2 = r2 ^ dataset[r1 & MASK]; // 53 - // per-load shadow sub-block 12 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 12 - for (uint sh12 = 0u; sh12 < 27u; ++sh12) { - r3 = r6 * r1 + r3; // s192 mad - r6 = rotl_imm(r6, 31u); // s193 rotl - r5 = rotl_imm(r5, 28u); // s194 rotl - r4 = r4 * r3; // s195 mul - r2 = r2 + r3 + select(0x9d7aebbbu, 0x7c3d6253u, ((sel >> 4u) & 1u) != 0u); // s196 add - r3 = r3 | r5; // s197 or - r6 = r6 - r0; // s198 sub - r7 = r7 * r4; // s199 mul - r3 = rotr_var(r3, r2); // s200 rotr - r4 = r4 ^ simd_shuffle_xor(r0, (ushort)16); // s201 shfl - r7 = r7 ^ r3; // s202 xor - r4 = r4 | r3; // s203 or - r3 = rotr_var(r3, r6); // s204 rotr - r0 = rotr_var(r0, r1); // s205 rotr - r2 = r5 * r2 + r2; // s206 mad - r1 = mulhi(r1, r5); // s207 mulhi - } - r4 = rotl_imm(r4, 20u); // 54 - r3 = r3 ^ simd_shuffle_xor(r1, (ushort)2); // 55 - r1 = r1 ^ dataset[r0 & MASK]; // 56 - // per-load shadow sub-block 13 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 13 - for (uint sh13 = 0u; sh13 < 27u; ++sh13) { - r4 = r4 ^ simd_shuffle_xor(r3, (ushort)1); // s208 shfl - r7 = r5 * r7 + r7; // s209 mad - r6 = r6 ^ r3; // s210 xor - r3 = r3 ^ r2; // s211 xor - r5 = r5 ^ simd_shuffle_xor(r6, (ushort)16); // s212 shfl - r4 = rotl_imm(r4, 10u); // s213 rotl - r5 = r5 ^ simd_shuffle_xor(r6, (ushort)16); // s214 shfl - r3 = r3 | r0; // s215 or - r4 = r4 * r5; // s216 mul - r0 = r0 ^ simd_shuffle_xor(r3, (ushort)16); // s217 shfl - r0 = r0 * r5; // s218 mul - r0 = rotl_imm(r0, 13u); // s219 rotl - r1 = mulhi(r1, r0); // s220 mulhi - r4 = r4 ^ r1; // s221 xor - r2 = r2 | r3; // s222 or - r0 = r0 ^ r4; // s223 xor - } - r3 = r3 ^ dataset[r5 & MASK]; // 57 - // per-load shadow sub-block 14 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 14 - for (uint sh14 = 0u; sh14 < 27u; ++sh14) { - r5 = r6 * r1 + r5; // s224 mad - r4 = r4 ^ r3; // s225 xor - r5 = r5 + r0 + select(0xfe960971u, 0xe4e51c75u, ((sel >> 3u) & 1u) != 0u); // s226 add - r3 = mulhi(r3, r2); // s227 mulhi - r2 = r2 ^ r6; // s228 xor - r1 = mulhi(r1, r5); // s229 mulhi - r3 = r3 + r4 + select(0x48b3ce0au, 0x4d597c08u, ((sel >> 25u) & 1u) != 0u); // s230 add - r2 = rotr_var(r2, r3); // s231 rotr - r2 = r2 - r7; // s232 sub - r6 = r6 | r2; // s233 or - r0 = rotr_var(r0, r3); // s234 rotr - r4 = r4 + r3 + select(0x2ed8c878u, 0xc9f1d54cu, ((sel >> 13u) & 1u) != 0u); // s235 add - r0 = r3 * r7 + r0; // s236 mad - r3 = r3 + r5 + select(0x22e8b90au, 0xc572bd00u, ((sel >> 29u) & 1u) != 0u); // s237 add - r0 = mulhi(r0, r4); // s238 mulhi - r6 = r6 * r5; // s239 mul - } - r1 = r1 ^ simd_shuffle_xor(r2, (ushort)2); // 58 - r6 = r6 ^ dataset[r4 & MASK]; // 59 - // per-load shadow sub-block 15 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 15 - for (uint sh15 = 0u; sh15 < 27u; ++sh15) { - r5 = r5 + r2 + select(0xc6b790e6u, 0xb233f94fu, ((sel >> 20u) & 1u) != 0u); // s240 add - r0 = r0 - r3; // s241 sub - r4 = r4 + r7 + select(0x0f918d3bu, 0x534d924bu, ((sel >> 4u) & 1u) != 0u); // s242 add - r5 = rotr_var(r5, r0); // s243 rotr - r1 = rotr_var(r1, r7); // s244 rotr - r2 = r2 | r0; // s245 or - r4 = r4 * r7; // s246 mul - r1 = r2 * r5 + r1; // s247 mad - r2 = r2 * r5; // s248 mul - r4 = r4 ^ r2; // s249 xor - r3 = r3 ^ r1; // s250 xor - r5 = rotr_var(r5, r7); // s251 rotr - r2 = r7 * r7 + r2; // s252 mad - r7 = rotr_var(r7, r3); // s253 rotr - r6 = r6 + r1 + select(0xb8a27d95u, 0x86d27169u, ((sel >> 21u) & 1u) != 0u); // s254 add - r6 = r6 | r7; // s255 or - } - r7 = r7 ^ simd_shuffle_xor(r3, (ushort)4); // 60 - r5 = rotr_var(r5, r0); // 61 - r0 = r0 + r6 + select(0x8c2e5c24u, 0xb13a5391u, ((sel >> 14u) & 1u) != 0u); // 62 - r2 = r2 + r1 + select(0xb225b762u, 0xf82fc8b5u, ((sel >> 21u) & 1u) != 0u); // 63 - } - uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((ulong)hi << 32) | (ulong)lo; -} diff --git a/proto-cuda/packs-ca4/mx8_shl256x27_v2/program_bound.metal b/proto-cuda/packs-ca4/mx8_shl256x27_v2/program_bound.metal deleted file mode 100644 index 8d57d2447..000000000 --- a/proto-cuda/packs-ca4/mx8_shl256x27_v2/program_bound.metal +++ /dev/null @@ -1,415 +0,0 @@ -#include -using namespace metal; - -#define MASK 0x0fffffffu -constant uint SEEDW[8] = { 0xf71aee9fu, 0xad930c88u, 0x7f982573u, 0xa41f9137u, 0x76d803d6u, 0x37b4a534u, 0x4d3fb826u, 0xff614dcbu }; - -inline uint splitmix32(uint x) { - x ^= x >> 16; x *= 0x7feb352du; - x ^= x >> 15; x *= 0x846ca68bu; - x ^= x >> 16; - return x; -} -inline uint rotl_imm(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 -inline uint rotr_var(uint x, uint n) { n &= 31u; return (x >> n) | (x << ((32u - n) & 31u)); } -inline uint ds_elem(uint i, uint d0, uint d1) { - uint x = i ^ d0; - x *= 0x9E3779B1u; x ^= x >> 15; - x += d1; - x *= 0x85EBCA77u; x ^= x >> 13; - x *= 0xC2B2AE3Du; x ^= x >> 16; - return x; -} - -// Header-bound variant: the init words come from buffer 3 (bind.rs), not from SEEDW. -kernel void igneum_hash_bound(device const uint* dataset [[buffer(0)]], - device ulong* out [[buffer(1)]], - constant uint& baseNonce [[buffer(2)]], - constant uint* initw [[buffer(3)]], - uint gid [[thread_position_in_grid]]) { - uint nonce = baseNonce + gid; - uint r0, r1, r2, r3, r4, r5, r6, r7; - { uint x = nonce ^ initw[0]; x += 0x9e3779b9u * 1u; x = splitmix32(x); r0 = x ^ initw[1]; } - { uint x = nonce ^ initw[1]; x += 0x9e3779b9u * 2u; x = splitmix32(x); r1 = x ^ initw[2]; } - { uint x = nonce ^ initw[2]; x += 0x9e3779b9u * 3u; x = splitmix32(x); r2 = x ^ initw[3]; } - { uint x = nonce ^ initw[3]; x += 0x9e3779b9u * 4u; x = splitmix32(x); r3 = x ^ initw[4]; } - { uint x = nonce ^ initw[4]; x += 0x9e3779b9u * 5u; x = splitmix32(x); r4 = x ^ initw[5]; } - { uint x = nonce ^ initw[5]; x += 0x9e3779b9u * 6u; x = splitmix32(x); r5 = x ^ initw[6]; } - { uint x = nonce ^ initw[6]; x += 0x9e3779b9u * 7u; x = splitmix32(x); r6 = x ^ initw[7]; } - { uint x = nonce ^ initw[7]; x += 0x9e3779b9u * 8u; x = splitmix32(x); r7 = x ^ initw[0]; } - - for (uint it = 0u; it < 8u; ++it) { - uint sel = r0; - r2 = r2 + r5 + select(0xe3e2ed7du, 0x894e457du, ((sel >> 2u) & 1u) != 0u); // 0 - r7 = r4 * r0 + r7; // 1 - r3 = r3 - r6; // 2 - r5 = rotr_var(r5, r1); // 3 - r6 = r6 ^ simd_shuffle_xor(r2, (ushort)1); // 4 - r0 = r0 | r3; // 5 - r0 = r0 * r1; // 6 - r7 = r7 * r6; // 7 - r3 = r3 ^ dataset[r0 & MASK]; // 8 - // per-load shadow sub-block 0 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 0 - for (uint sh0 = 0u; sh0 < 27u; ++sh0) { - r5 = rotr_var(r5, r4); // s0 rotr - r6 = r6 - r5; // s1 sub - r7 = r7 - r6; // s2 sub - r5 = r5 * r4; // s3 mul - r7 = r7 ^ r6; // s4 xor - r2 = rotl_imm(r2, 23u); // s5 rotl - r7 = r7 ^ r4; // s6 xor - r0 = r0 + r7 + select(0xdaef8862u, 0x3051c491u, ((sel >> 26u) & 1u) != 0u); // s7 add - r6 = r6 ^ simd_shuffle_xor(r1, (ushort)4); // s8 shfl - r5 = r5 + r1 + select(0xc20e7045u, 0x5170d0b3u, ((sel >> 26u) & 1u) != 0u); // s9 add - r0 = r0 | r3; // s10 or - r5 = mulhi(r5, r4); // s11 mulhi - r7 = r7 ^ r0; // s12 xor - r5 = r5 - r2; // s13 sub - r0 = r0 + r6 + select(0xc86d98a4u, 0x2ead087fu, ((sel >> 0u) & 1u) != 0u); // s14 add - r5 = r5 ^ r7; // s15 xor - } - r4 = r4 + r3 + select(0xce9bea84u, 0xd26d3573u, ((sel >> 22u) & 1u) != 0u); // 9 - r2 = r2 * r4; // 10 - r3 = r3 ^ dataset[r2 & MASK]; // 11 - // per-load shadow sub-block 1 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 1 - for (uint sh1 = 0u; sh1 < 27u; ++sh1) { - r7 = r7 ^ simd_shuffle_xor(r6, (ushort)8); // s16 shfl - r5 = r5 - r7; // s17 sub - r0 = r0 | r6; // s18 or - r0 = r0 + r5 + select(0x4248b651u, 0xc47c70a8u, ((sel >> 13u) & 1u) != 0u); // s19 add - r4 = r4 ^ simd_shuffle_xor(r5, (ushort)16); // s20 shfl - r0 = r0 + r4 + select(0x1e35684fu, 0x718c1008u, ((sel >> 21u) & 1u) != 0u); // s21 add - r0 = mulhi(r0, r3); // s22 mulhi - r4 = r4 ^ r2; // s23 xor - r0 = r0 - r2; // s24 sub - r5 = rotl_imm(r5, 13u); // s25 rotl - r7 = r7 - r0; // s26 sub - r2 = r2 ^ r3; // s27 xor - r2 = r3 * r2 + r2; // s28 mad - r2 = r2 ^ r1; // s29 xor - r4 = mulhi(r4, r0); // s30 mulhi - r6 = r6 ^ r4; // s31 xor - } - r4 = mulhi(r4, r1); // 12 - r7 = mulhi(r7, r3); // 13 - r3 = r3 + r2 + select(0x514f9ff4u, 0xc2828a42u, ((sel >> 8u) & 1u) != 0u); // 14 - r1 = r3 * r2 + r1; // 15 - r2 = rotl_imm(r2, 7u); // 16 - r1 = r1 ^ dataset[r2 & MASK]; // 17 - // per-load shadow sub-block 2 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 2 - for (uint sh2 = 0u; sh2 < 27u; ++sh2) { - r4 = r4 ^ r2; // s32 xor - r1 = r1 ^ simd_shuffle_xor(r2, (ushort)1); // s33 shfl - r5 = r5 ^ simd_shuffle_xor(r6, (ushort)1); // s34 shfl - r4 = r4 - r0; // s35 sub - r6 = r6 - r3; // s36 sub - r2 = r2 | r6; // s37 or - r2 = rotl_imm(r2, 30u); // s38 rotl - r4 = rotr_var(r4, r7); // s39 rotr - r0 = r0 | r7; // s40 or - r2 = rotr_var(r2, r5); // s41 rotr - r1 = r3 * r3 + r1; // s42 mad - r6 = r5 * r5 + r6; // s43 mad - r1 = mulhi(r1, r0); // s44 mulhi - r1 = r1 * r3; // s45 mul - r0 = r2 * r0 + r0; // s46 mad - r1 = r1 - r5; // s47 sub - } - r3 = r3 * r2; // 18 - r6 = rotl_imm(r6, 24u); // 19 - r6 = r6 ^ dataset[r3 & MASK]; // 20 - // per-load shadow sub-block 3 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 3 - for (uint sh3 = 0u; sh3 < 27u; ++sh3) { - r4 = r4 ^ r0; // s48 xor - r2 = rotr_var(r2, r0); // s49 rotr - r0 = mulhi(r0, r5); // s50 mulhi - r7 = r7 | r5; // s51 or - r4 = r0 * r3 + r4; // s52 mad - r0 = rotr_var(r0, r4); // s53 rotr - r6 = r3 * r3 + r6; // s54 mad - r6 = r4 * r2 + r6; // s55 mad - r5 = r5 ^ simd_shuffle_xor(r0, (ushort)2); // s56 shfl - r7 = rotl_imm(r7, 21u); // s57 rotl - r3 = r3 ^ r7; // s58 xor - r0 = r0 ^ simd_shuffle_xor(r4, (ushort)1); // s59 shfl - r5 = r5 ^ r6; // s60 xor - r4 = rotr_var(r4, r7); // s61 rotr - r1 = r1 + r6 + select(0xf7ccf1e9u, 0x31f87d74u, ((sel >> 10u) & 1u) != 0u); // s62 add - r5 = r5 + r2 + select(0x2f386099u, 0x9c2847f4u, ((sel >> 15u) & 1u) != 0u); // s63 add - } - r1 = r1 ^ dataset[r6 & MASK]; // 21 - // per-load shadow sub-block 4 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 4 - for (uint sh4 = 0u; sh4 < 27u; ++sh4) { - r7 = r7 + r0 + select(0x1b30ce7au, 0xd3241188u, ((sel >> 19u) & 1u) != 0u); // s64 add - r6 = r6 + r7 + select(0x02dc8349u, 0x9b42c3deu, ((sel >> 13u) & 1u) != 0u); // s65 add - r0 = rotl_imm(r0, 16u); // s66 rotl - r2 = rotl_imm(r2, 9u); // s67 rotl - r2 = r5 * r6 + r2; // s68 mad - r6 = r6 ^ simd_shuffle_xor(r1, (ushort)8); // s69 shfl - r2 = r2 ^ r5; // s70 xor - r5 = mulhi(r5, r1); // s71 mulhi - r6 = rotl_imm(r6, 29u); // s72 rotl - r0 = r0 - r7; // s73 sub - r5 = r2 * r2 + r5; // s74 mad - r0 = rotr_var(r0, r5); // s75 rotr - r7 = r7 + r2 + select(0x05d36679u, 0x0da1ce17u, ((sel >> 9u) & 1u) != 0u); // s76 add - r7 = r7 - r2; // s77 sub - r7 = rotr_var(r7, r0); // s78 rotr - r1 = r5 * r5 + r1; // s79 mad - } - r5 = r3 * r3 + r5; // 22 - r4 = r4 ^ dataset[r7 & MASK]; // 23 - // per-load shadow sub-block 5 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 5 - for (uint sh5 = 0u; sh5 < 27u; ++sh5) { - r3 = rotr_var(r3, r0); // s80 rotr - r6 = r6 + r5 + select(0xf14547bdu, 0xadf5ef88u, ((sel >> 2u) & 1u) != 0u); // s81 add - r0 = r0 | r6; // s82 or - r1 = mulhi(r1, r0); // s83 mulhi - r7 = r6 * r7 + r7; // s84 mad - r5 = rotl_imm(r5, 29u); // s85 rotl - r2 = r2 ^ r6; // s86 xor - r5 = r5 + r4 + select(0xba4947c2u, 0x35ff14aeu, ((sel >> 24u) & 1u) != 0u); // s87 add - r3 = r3 + r6 + select(0xf6878beeu, 0xf3900fc1u, ((sel >> 3u) & 1u) != 0u); // s88 add - r0 = r0 + r2 + select(0x926f3607u, 0xf7e8f59fu, ((sel >> 12u) & 1u) != 0u); // s89 add - r2 = r2 ^ r6; // s90 xor - r3 = r3 ^ simd_shuffle_xor(r2, (ushort)2); // s91 shfl - r6 = r6 - r1; // s92 sub - r5 = r5 + r1 + select(0xdab36c29u, 0x0e376f9cu, ((sel >> 13u) & 1u) != 0u); // s93 add - r6 = r1 * r1 + r6; // s94 mad - r3 = r3 ^ r4; // s95 xor - } - r6 = r6 ^ dataset[r4 & MASK]; // 24 - // per-load shadow sub-block 6 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 6 - for (uint sh6 = 0u; sh6 < 27u; ++sh6) { - r6 = r5 * r2 + r6; // s96 mad - r7 = r7 + r3 + select(0x204129e9u, 0x9f917747u, ((sel >> 23u) & 1u) != 0u); // s97 add - r5 = r5 ^ simd_shuffle_xor(r1, (ushort)16); // s98 shfl - r2 = r2 * r6; // s99 mul - r7 = r7 ^ simd_shuffle_xor(r3, (ushort)8); // s100 shfl - r6 = r6 + r1 + select(0xe3b596a0u, 0xe42fe974u, ((sel >> 10u) & 1u) != 0u); // s101 add - r5 = rotl_imm(r5, 3u); // s102 rotl - r5 = r5 ^ r1; // s103 xor - r4 = r4 + r1 + select(0x705c3f94u, 0xc3d77ffbu, ((sel >> 27u) & 1u) != 0u); // s104 add - r0 = r0 + r6 + select(0xbc6fb42bu, 0xea02a4cau, ((sel >> 25u) & 1u) != 0u); // s105 add - r3 = r3 | r5; // s106 or - r0 = r0 - r6; // s107 sub - r0 = rotr_var(r0, r3); // s108 rotr - r5 = r5 + r4 + select(0xcb37ea87u, 0x0357e63eu, ((sel >> 4u) & 1u) != 0u); // s109 add - r7 = rotl_imm(r7, 5u); // s110 rotl - r4 = r4 ^ r1; // s111 xor - } - r5 = r5 * r7; // 25 - r0 = r0 * r3; // 26 - r0 = r0 | r3; // 27 - r1 = r3 * r4 + r1; // 28 - r0 = r0 ^ simd_shuffle_xor(r2, (ushort)8); // 29 - r7 = r7 - r3; // 30 - r4 = r4 ^ r1; // 31 - r4 = r4 | r5; // 32 - r3 = rotr_var(r3, r5); // 33 - r4 = r4 ^ dataset[r5 & MASK]; // 34 - // per-load shadow sub-block 7 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 7 - for (uint sh7 = 0u; sh7 < 27u; ++sh7) { - r6 = r6 ^ r5; // s112 xor - r1 = r1 | r3; // s113 or - r0 = r0 + r5 + select(0x9aab0297u, 0x7f8b8cd2u, ((sel >> 3u) & 1u) != 0u); // s114 add - r3 = r3 - r5; // s115 sub - r4 = rotr_var(r4, r2); // s116 rotr - r4 = r4 + r6 + select(0xc511183fu, 0x59400b57u, ((sel >> 6u) & 1u) != 0u); // s117 add - r6 = r6 - r1; // s118 sub - r3 = r3 ^ simd_shuffle_xor(r1, (ushort)4); // s119 shfl - r6 = r4 * r4 + r6; // s120 mad - r2 = rotl_imm(r2, 8u); // s121 rotl - r5 = r0 * r3 + r5; // s122 mad - r7 = r7 + r6 + select(0x5200c242u, 0x27c70dc0u, ((sel >> 27u) & 1u) != 0u); // s123 add - r3 = r3 ^ simd_shuffle_xor(r5, (ushort)4); // s124 shfl - r0 = r0 + r1 + select(0x2d5028ceu, 0xec2f5997u, ((sel >> 17u) & 1u) != 0u); // s125 add - r3 = r3 ^ simd_shuffle_xor(r6, (ushort)16); // s126 shfl - r5 = r5 ^ r0; // s127 xor - } - r0 = r5 * r3 + r0; // 35 - r6 = r6 ^ dataset[r3 & MASK]; // 36 - // per-load shadow sub-block 8 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 8 - for (uint sh8 = 0u; sh8 < 27u; ++sh8) { - r7 = r7 + r1 + select(0x432f6c0du, 0x09e8ede2u, ((sel >> 10u) & 1u) != 0u); // s128 add - r2 = r2 ^ r4; // s129 xor - r0 = r0 * r2; // s130 mul - r4 = r4 | r3; // s131 or - r3 = r3 | r7; // s132 or - r7 = r7 ^ simd_shuffle_xor(r3, (ushort)1); // s133 shfl - r5 = r5 + r2 + select(0xcf5ecc75u, 0xce4fdf8eu, ((sel >> 11u) & 1u) != 0u); // s134 add - r3 = r2 * r3 + r3; // s135 mad - r1 = rotl_imm(r1, 11u); // s136 rotl - r2 = r2 | r0; // s137 or - r3 = r3 ^ r4; // s138 xor - r2 = r2 ^ simd_shuffle_xor(r4, (ushort)1); // s139 shfl - r2 = rotl_imm(r2, 6u); // s140 rotl - r0 = r0 * r3; // s141 mul - r1 = r1 * r7; // s142 mul - r0 = r0 ^ r1; // s143 xor - } - r5 = r5 * r3; // 37 - r5 = r5 ^ r4; // 38 - r6 = r6 ^ r0; // 39 - r4 = rotr_var(r4, r0); // 40 - r7 = r7 ^ r6; // 41 - r1 = r1 ^ dataset[r7 & MASK]; // 42 - // per-load shadow sub-block 9 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 9 - for (uint sh9 = 0u; sh9 < 27u; ++sh9) { - r6 = r2 * r0 + r6; // s144 mad - r1 = rotl_imm(r1, 15u); // s145 rotl - r1 = rotl_imm(r1, 30u); // s146 rotl - r6 = r6 ^ r0; // s147 xor - r3 = r3 - r2; // s148 sub - r3 = r3 + r0 + select(0xaa002f15u, 0x11bbdeefu, ((sel >> 5u) & 1u) != 0u); // s149 add - r6 = r6 + r5 + select(0xbe64b2e2u, 0xff9d4b0eu, ((sel >> 9u) & 1u) != 0u); // s150 add - r1 = r1 * r4; // s151 mul - r5 = rotl_imm(r5, 3u); // s152 rotl - r5 = r5 * r2; // s153 mul - r2 = rotl_imm(r2, 21u); // s154 rotl - r5 = rotl_imm(r5, 18u); // s155 rotl - r1 = r1 ^ r5; // s156 xor - r1 = mulhi(r1, r6); // s157 mulhi - r4 = r7 * r3 + r4; // s158 mad - r4 = r4 - r7; // s159 sub - } - r2 = r2 - r4; // 43 - r6 = mulhi(r6, r7); // 44 - r3 = rotl_imm(r3, 13u); // 45 - r7 = r7 ^ simd_shuffle_xor(r6, (ushort)16); // 46 - r2 = rotl_imm(r2, 26u); // 47 - r6 = r6 * r2; // 48 - r2 = r2 ^ dataset[r3 & MASK]; // 49 - // per-load shadow sub-block 10 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 10 - for (uint sh10 = 0u; sh10 < 27u; ++sh10) { - r3 = r3 ^ r2; // s160 xor - r3 = mulhi(r3, r4); // s161 mulhi - r3 = r3 ^ simd_shuffle_xor(r2, (ushort)1); // s162 shfl - r0 = r0 - r1; // s163 sub - r6 = r6 ^ simd_shuffle_xor(r0, (ushort)8); // s164 shfl - r4 = r4 ^ simd_shuffle_xor(r2, (ushort)2); // s165 shfl - r3 = r3 | r6; // s166 or - r6 = r6 ^ simd_shuffle_xor(r3, (ushort)4); // s167 shfl - r2 = r2 ^ r1; // s168 xor - r5 = r5 ^ r1; // s169 xor - r5 = r5 ^ r0; // s170 xor - r3 = r6 * r0 + r3; // s171 mad - r3 = r3 - r0; // s172 sub - r6 = r6 + r0 + select(0x3b90694bu, 0xc72dc2a0u, ((sel >> 27u) & 1u) != 0u); // s173 add - r7 = r7 ^ simd_shuffle_xor(r2, (ushort)8); // s174 shfl - r5 = mulhi(r5, r2); // s175 mulhi - } - r5 = r5 ^ dataset[r1 & MASK]; // 50 - // per-load shadow sub-block 11 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 11 - for (uint sh11 = 0u; sh11 < 27u; ++sh11) { - r3 = r3 + r6 + select(0x9e65cebdu, 0x4eb75843u, ((sel >> 5u) & 1u) != 0u); // s176 add - r0 = r4 * r1 + r0; // s177 mad - r6 = rotr_var(r6, r7); // s178 rotr - r0 = r0 + r7 + select(0x01e5b250u, 0x7001d036u, ((sel >> 31u) & 1u) != 0u); // s179 add - r5 = r5 ^ simd_shuffle_xor(r2, (ushort)16); // s180 shfl - r3 = r3 * r0; // s181 mul - r0 = rotl_imm(r0, 23u); // s182 rotl - r7 = mulhi(r7, r0); // s183 mulhi - r0 = r0 * r4; // s184 mul - r1 = r1 + r4 + select(0xbef14988u, 0x91736711u, ((sel >> 19u) & 1u) != 0u); // s185 add - r7 = r7 + r2 + select(0xf5d741beu, 0x15e0cdf3u, ((sel >> 6u) & 1u) != 0u); // s186 add - r1 = r1 ^ r7; // s187 xor - r1 = r1 + r3 + select(0x32be33c6u, 0x7549bc3eu, ((sel >> 21u) & 1u) != 0u); // s188 add - r1 = r0 * r6 + r1; // s189 mad - r0 = rotr_var(r0, r5); // s190 rotr - r4 = r0 * r1 + r4; // s191 mad - } - r0 = rotl_imm(r0, 20u); // 51 - r1 = r1 ^ r2; // 52 - r2 = r2 ^ dataset[r1 & MASK]; // 53 - // per-load shadow sub-block 12 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 12 - for (uint sh12 = 0u; sh12 < 27u; ++sh12) { - r3 = r6 * r1 + r3; // s192 mad - r6 = rotl_imm(r6, 31u); // s193 rotl - r5 = rotl_imm(r5, 28u); // s194 rotl - r4 = r4 * r3; // s195 mul - r2 = r2 + r3 + select(0x9d7aebbbu, 0x7c3d6253u, ((sel >> 4u) & 1u) != 0u); // s196 add - r3 = r3 | r5; // s197 or - r6 = r6 - r0; // s198 sub - r7 = r7 * r4; // s199 mul - r3 = rotr_var(r3, r2); // s200 rotr - r4 = r4 ^ simd_shuffle_xor(r0, (ushort)16); // s201 shfl - r7 = r7 ^ r3; // s202 xor - r4 = r4 | r3; // s203 or - r3 = rotr_var(r3, r6); // s204 rotr - r0 = rotr_var(r0, r1); // s205 rotr - r2 = r5 * r2 + r2; // s206 mad - r1 = mulhi(r1, r5); // s207 mulhi - } - r4 = rotl_imm(r4, 20u); // 54 - r3 = r3 ^ simd_shuffle_xor(r1, (ushort)2); // 55 - r1 = r1 ^ dataset[r0 & MASK]; // 56 - // per-load shadow sub-block 13 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 13 - for (uint sh13 = 0u; sh13 < 27u; ++sh13) { - r4 = r4 ^ simd_shuffle_xor(r3, (ushort)1); // s208 shfl - r7 = r5 * r7 + r7; // s209 mad - r6 = r6 ^ r3; // s210 xor - r3 = r3 ^ r2; // s211 xor - r5 = r5 ^ simd_shuffle_xor(r6, (ushort)16); // s212 shfl - r4 = rotl_imm(r4, 10u); // s213 rotl - r5 = r5 ^ simd_shuffle_xor(r6, (ushort)16); // s214 shfl - r3 = r3 | r0; // s215 or - r4 = r4 * r5; // s216 mul - r0 = r0 ^ simd_shuffle_xor(r3, (ushort)16); // s217 shfl - r0 = r0 * r5; // s218 mul - r0 = rotl_imm(r0, 13u); // s219 rotl - r1 = mulhi(r1, r0); // s220 mulhi - r4 = r4 ^ r1; // s221 xor - r2 = r2 | r3; // s222 or - r0 = r0 ^ r4; // s223 xor - } - r3 = r3 ^ dataset[r5 & MASK]; // 57 - // per-load shadow sub-block 14 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 14 - for (uint sh14 = 0u; sh14 < 27u; ++sh14) { - r5 = r6 * r1 + r5; // s224 mad - r4 = r4 ^ r3; // s225 xor - r5 = r5 + r0 + select(0xfe960971u, 0xe4e51c75u, ((sel >> 3u) & 1u) != 0u); // s226 add - r3 = mulhi(r3, r2); // s227 mulhi - r2 = r2 ^ r6; // s228 xor - r1 = mulhi(r1, r5); // s229 mulhi - r3 = r3 + r4 + select(0x48b3ce0au, 0x4d597c08u, ((sel >> 25u) & 1u) != 0u); // s230 add - r2 = rotr_var(r2, r3); // s231 rotr - r2 = r2 - r7; // s232 sub - r6 = r6 | r2; // s233 or - r0 = rotr_var(r0, r3); // s234 rotr - r4 = r4 + r3 + select(0x2ed8c878u, 0xc9f1d54cu, ((sel >> 13u) & 1u) != 0u); // s235 add - r0 = r3 * r7 + r0; // s236 mad - r3 = r3 + r5 + select(0x22e8b90au, 0xc572bd00u, ((sel >> 29u) & 1u) != 0u); // s237 add - r0 = mulhi(r0, r4); // s238 mulhi - r6 = r6 * r5; // s239 mul - } - r1 = r1 ^ simd_shuffle_xor(r2, (ushort)2); // 58 - r6 = r6 ^ dataset[r4 & MASK]; // 59 - // per-load shadow sub-block 15 (Counter ASIC 4.0 research): 16 ALU instructions x 27 passes after load 15 - for (uint sh15 = 0u; sh15 < 27u; ++sh15) { - r5 = r5 + r2 + select(0xc6b790e6u, 0xb233f94fu, ((sel >> 20u) & 1u) != 0u); // s240 add - r0 = r0 - r3; // s241 sub - r4 = r4 + r7 + select(0x0f918d3bu, 0x534d924bu, ((sel >> 4u) & 1u) != 0u); // s242 add - r5 = rotr_var(r5, r0); // s243 rotr - r1 = rotr_var(r1, r7); // s244 rotr - r2 = r2 | r0; // s245 or - r4 = r4 * r7; // s246 mul - r1 = r2 * r5 + r1; // s247 mad - r2 = r2 * r5; // s248 mul - r4 = r4 ^ r2; // s249 xor - r3 = r3 ^ r1; // s250 xor - r5 = rotr_var(r5, r7); // s251 rotr - r2 = r7 * r7 + r2; // s252 mad - r7 = rotr_var(r7, r3); // s253 rotr - r6 = r6 + r1 + select(0xb8a27d95u, 0x86d27169u, ((sel >> 21u) & 1u) != 0u); // s254 add - r6 = r6 | r7; // s255 or - } - r7 = r7 ^ simd_shuffle_xor(r3, (ushort)4); // 60 - r5 = rotr_var(r5, r0); // 61 - r0 = r0 + r6 + select(0x8c2e5c24u, 0xb13a5391u, ((sel >> 14u) & 1u) != 0u); // 62 - r2 = r2 + r1 + select(0xb225b762u, 0xf82fc8b5u, ((sel >> 21u) & 1u) != 0u); // 63 - } - uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u); - uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u); - out[gid] = ((ulong)hi << 32) | (ulong)lo; -} diff --git a/proto-cuda/packs-ca4/mx8_shl256x27_v2/vectors.h b/proto-cuda/packs-ca4/mx8_shl256x27_v2/vectors.h deleted file mode 100644 index cd482f7fe..000000000 --- a/proto-cuda/packs-ca4/mx8_shl256x27_v2/vectors.h +++ /dev/null @@ -1,57 +0,0 @@ -// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand. -// Expected outputs: igneum-pow (Rust) CPU interpreter, generator v2, memory-hard dataset -#pragma once -#ifdef __cplusplus -#include -#else -#include -#endif - -#define IGNEUM_VEC_WARPS 3 -static const uint32_t IGNEUM_VEC_BASE[IGNEUM_VEC_WARPS] = { 0u, 4096u, 1000000u }; -static const uint64_t IGNEUM_VEC_OUT[IGNEUM_VEC_WARPS][32] = { - { // base nonce 0 - 0xefd63997a0441711ull, 0x33de48f246942bbdull, 0x75d11c0462d5d88eull, 0x339027e7a81edd02ull, 0x44ea8055287924c8ull, 0x94695edae512b2fdull, 0x10e849a40976a1f9ull, 0xe359cb57411223cdull, - 0x9f87f6349d272c37ull, 0x9b014e403a50b9d8ull, 0x26fc7cf39d2abb3full, 0x46a8c54601339b19ull, 0x24358a4463be7912ull, 0x9114765e85734923ull, 0x6fe5559c34419006ull, 0x6e0084f9fb52a815ull, - 0xcb00c0ae818d85adull, 0x523ff275ef11bba8ull, 0x18919bc6daeeecc9ull, 0x049eb1764feed566ull, 0xb10204b89ad283fcull, 0xb01bd8c897ec6edaull, 0x37566026b58c0a06ull, 0x52dd03de72a2c1c1ull, - 0xfdb39e45434da769ull, 0x65aa545737c3d5a8ull, 0x1075f64ffd1240fbull, 0x4c025b333a619365ull, 0x370d220ec4a57f28ull, 0x65279249eac9e65bull, 0x733a248e4f1d06b0ull, 0xc8b37f453592452bull - }, - { // base nonce 4096 - 0x918005a6d3fbc63eull, 0xb96b1dd6712afa86ull, 0xd76c1007676d61c2ull, 0x05390f3bd5062f59ull, 0x73aa6903da6f4356ull, 0xc494259068e25752ull, 0xf59c83f92e53c420ull, 0xd218528573a52adeull, - 0x3978767d74c7d605ull, 0xeaab672b16d91b5aull, 0x7c63cb2f17d0e460ull, 0x259ed48b2ecf354cull, 0x39086dbce0897309ull, 0x77a4dd060aaaf1bcull, 0xac407a7c7bea2ebcull, 0xfd818079715aa327ull, - 0x7b9db28bfbac80d0ull, 0xecd209239058699eull, 0x5ed5239644b890f6ull, 0x1671a35df0cdf469ull, 0xbcddda57058609cfull, 0xd8ae74e8f95b1ef1ull, 0x75764d60cfd2ac82ull, 0x79e1e7461d8178bdull, - 0x201bd093799b88e7ull, 0x7ec60d524372883full, 0x79d63de696496167ull, 0x8e449b589811c497ull, 0x52adf459f981f0b8ull, 0x596742f5dd8b43f3ull, 0x86a5fcd2631ae5deull, 0xd22446fc73434c7cull - }, - { // base nonce 1000000 - 0x95de23ae63046a5eull, 0x1d08d2a4b1cb3161ull, 0x6a940c9bef843fd0ull, 0xc5ce48f84aadb294ull, 0xba82215f36795b42ull, 0x8468642d1bf89decull, 0xe898eb1ae43e41a7ull, 0x4406325dcf229715ull, - 0x5bdef83a567bf872ull, 0x7148d685ed7aea55ull, 0xa1ade4af843a6cf6ull, 0xb557e23607845239ull, 0x3e8ed61763fc9602ull, 0x59088c92f20eae50ull, 0xd0ccb1c1189923c9ull, 0x6e43ec976d49b534ull, - 0xa895822ad9282927ull, 0x3eee289557cce518ull, 0x3004d873b41ef15bull, 0xbd029727b3853e96ull, 0x55c208f4320d0c37ull, 0x49fbcab22b92a4baull, 0xddc3fca332df8c3full, 0x288753718fd99e01ull, - 0x31d83163a6fe8b98ull, 0x85bf6f0b5397a63eull, 0x22beb8e9b3aff83dull, 0x9c52af026d8f81ecull, 0xcf4db829220c248dull, 0xf200f1e97bf4fe6cull, 0xf4475fbe74941055ull, 0xdb1af4be8ec353d4ull - } -}; - -// Dataset self-test: dataset[0..15] and dataset[IGNEUM_MASK] (268435455). -static const uint32_t IGNEUM_DS_HEAD[16] = { - 0xfdad4319u, 0x1a7b68e1u, 0xde6db608u, 0x13d73892u, 0xd17f447au, 0xb2221ccfu, 0x9db004bdu, 0x57d7d367u, - 0xdbc4cf34u, 0x697c009au, 0xc43af1d4u, 0x97f12b2eu, 0x74c37cd0u, 0xc651ea15u, 0x665a6d29u, 0x22330a2du -}; -static const uint32_t IGNEUM_DS_LAST_INDEX = 268435455u; -static const uint32_t IGNEUM_DS_LAST = 0xa83e7aa6u; -// 64 sampled dataset words (index, value) computed on the Mac. -#define IGNEUM_DS_SAMPLES 64 -static const uint32_t IGNEUM_DS_SAMPLE_INDEX[IGNEUM_DS_SAMPLES] = { - 59471966u, 217795994u, 208353206u, 42483309u, 172547758u, 148076330u, 183853158u, 214389424u, 267488061u, 169781097u, 184093494u, 153880993u, 84977930u, 46426879u, 3093825u, 225364072u, 44593546u, 260713159u, 168250303u, 52384140u, 223401610u, 45554030u, 95410555u, 175039924u, 79171087u, 267580473u, 24168642u, 37981670u, 171551130u, 195559979u, 204611762u, 140997658u, 138925853u, 86637313u, 20736778u, 219665210u, 160430336u, 264654675u, 8013395u, 228945585u, 213884386u, 104419827u, 44185464u, 142737231u, 99284897u, 132475900u, 61861762u, 132056166u, 262388043u, 91878046u, 117353561u, 124768597u, 71352993u, 190698941u, 46055428u, 55281366u, 165145231u, 106810753u, 171985651u, 232085256u, 159510492u, 40072060u, 209107596u, 39023794u -}; -static const uint32_t IGNEUM_DS_SAMPLE_VALUE[IGNEUM_DS_SAMPLES] = { - 0x3230bc7bu, 0x7fbfe2c9u, 0xb2690991u, 0x1745c7c5u, 0x0ab0ccafu, 0x1bf87d6bu, 0x160139fdu, 0x719817acu, 0x0155df4bu, 0xbe1e86c3u, 0x680bcd6cu, 0x79c3dc6cu, 0x181e7e5fu, 0x0713a109u, 0xc705dd9fu, 0x3933b7a8u, 0xdd1c0431u, 0x50522b30u, 0xa0020b38u, 0xbff39e96u, 0x21b67e18u, 0x740f8db3u, 0x2baba568u, 0x2c9bef83u, 0x0ad9b671u, 0xc4327869u, 0x7b4fd7d0u, 0x2c29965fu, 0xec56f15fu, 0x61111746u, 0x303a1d6eu, 0xbddcfd1au, 0xf829a355u, 0x6d5df2a9u, 0x01ab8e44u, 0x06d13507u, 0xda8dcfc6u, 0x01a703e1u, 0xafe7d2c1u, 0xc091c3a2u, 0xac1814feu, 0x6e6ff62au, 0x8fdf01bau, 0xdd3f7159u, 0xdfa0d75cu, 0x26684c35u, 0x7f441e63u, 0x88df2570u, 0x8aa4d5ebu, 0xcc816c05u, 0x434df890u, 0xcd392ad6u, 0x1ab4cb63u, 0x595926fau, 0x7cd76b41u, 0x20cb95c4u, 0x13cf823fu, 0xf9daf901u, 0xff9af40au, 0x2c7dfa51u, 0x871206dbu, 0x938c116cu, 0xb64bf199u, 0x5751f874u -}; -// Cache self-test (memory-hard mode): cache[0..15], the last 16 words, and FNV-1a 64 over all 2^26 words. -static const uint32_t IGNEUM_CACHE_HEAD[16] = { - 0x355a86d2u, 0x7957db1cu, 0xd21772afu, 0x6fc1e09bu, 0xd55ce61du, 0x6e6a278bu, 0xd3f543ceu, 0x223d8e82u, - 0x143ab337u, 0x2e9f05bdu, 0x2eb389bfu, 0x0c6e449eu, 0x5cfa4222u, 0xba6560feu, 0x8e3e1aa4u, 0xdbcc1d53u -}; -static const uint32_t IGNEUM_CACHE_LAST[16] = { - 0x41190d91u, 0xbd277957u, 0x22ddbb49u, 0x6986f207u, 0xdf69a4d6u, 0x26401a3au, 0x818230fbu, 0xc417122du, - 0x3597b211u, 0xb553ce55u, 0xcf39cc0du, 0x3b7fc43au, 0x3fd43b00u, 0x67e1c80eu, 0xffa7ea7du, 0xca2960abu -}; -static const uint64_t IGNEUM_CACHE_FNV64 = 0x48c4f5bf24166b2eull; diff --git a/proto-cuda/packs-ca4/mx8_shl256x27_v2/vectors.json b/proto-cuda/packs-ca4/mx8_shl256x27_v2/vectors.json deleted file mode 100644 index 1d714e8c9..000000000 --- a/proto-cuda/packs-ca4/mx8_shl256x27_v2/vectors.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "seed": "igneum-genesis", - "day": "2026-10-03", - "dataset_mode": "memory-hard", - "dataset_log2_words": 28, - "mask": "0x0fffffff", - "lanes": 32, - "source": "igneum-pow (Rust) CPU interpreter, generator v2, memory-hard dataset", - "warps": [ - {"base_nonce": 0, "expected": [ - "0xefd63997a0441711", "0x33de48f246942bbd", "0x75d11c0462d5d88e", "0x339027e7a81edd02", "0x44ea8055287924c8", "0x94695edae512b2fd", "0x10e849a40976a1f9", "0xe359cb57411223cd", - "0x9f87f6349d272c37", "0x9b014e403a50b9d8", "0x26fc7cf39d2abb3f", "0x46a8c54601339b19", "0x24358a4463be7912", "0x9114765e85734923", "0x6fe5559c34419006", "0x6e0084f9fb52a815", - "0xcb00c0ae818d85ad", "0x523ff275ef11bba8", "0x18919bc6daeeecc9", "0x049eb1764feed566", "0xb10204b89ad283fc", "0xb01bd8c897ec6eda", "0x37566026b58c0a06", "0x52dd03de72a2c1c1", - "0xfdb39e45434da769", "0x65aa545737c3d5a8", "0x1075f64ffd1240fb", "0x4c025b333a619365", "0x370d220ec4a57f28", "0x65279249eac9e65b", "0x733a248e4f1d06b0", "0xc8b37f453592452b" - ]}, - {"base_nonce": 4096, "expected": [ - "0x918005a6d3fbc63e", "0xb96b1dd6712afa86", "0xd76c1007676d61c2", "0x05390f3bd5062f59", "0x73aa6903da6f4356", "0xc494259068e25752", "0xf59c83f92e53c420", "0xd218528573a52ade", - "0x3978767d74c7d605", "0xeaab672b16d91b5a", "0x7c63cb2f17d0e460", "0x259ed48b2ecf354c", "0x39086dbce0897309", "0x77a4dd060aaaf1bc", "0xac407a7c7bea2ebc", "0xfd818079715aa327", - "0x7b9db28bfbac80d0", "0xecd209239058699e", "0x5ed5239644b890f6", "0x1671a35df0cdf469", "0xbcddda57058609cf", "0xd8ae74e8f95b1ef1", "0x75764d60cfd2ac82", "0x79e1e7461d8178bd", - "0x201bd093799b88e7", "0x7ec60d524372883f", "0x79d63de696496167", "0x8e449b589811c497", "0x52adf459f981f0b8", "0x596742f5dd8b43f3", "0x86a5fcd2631ae5de", "0xd22446fc73434c7c" - ]}, - {"base_nonce": 1000000, "expected": [ - "0x95de23ae63046a5e", "0x1d08d2a4b1cb3161", "0x6a940c9bef843fd0", "0xc5ce48f84aadb294", "0xba82215f36795b42", "0x8468642d1bf89dec", "0xe898eb1ae43e41a7", "0x4406325dcf229715", - "0x5bdef83a567bf872", "0x7148d685ed7aea55", "0xa1ade4af843a6cf6", "0xb557e23607845239", "0x3e8ed61763fc9602", "0x59088c92f20eae50", "0xd0ccb1c1189923c9", "0x6e43ec976d49b534", - "0xa895822ad9282927", "0x3eee289557cce518", "0x3004d873b41ef15b", "0xbd029727b3853e96", "0x55c208f4320d0c37", "0x49fbcab22b92a4ba", "0xddc3fca332df8c3f", "0x288753718fd99e01", - "0x31d83163a6fe8b98", "0x85bf6f0b5397a63e", "0x22beb8e9b3aff83d", "0x9c52af026d8f81ec", "0xcf4db829220c248d", "0xf200f1e97bf4fe6c", "0xf4475fbe74941055", "0xdb1af4be8ec353d4" - ]} - ], - "dataset_head": ["0xfdad4319", "0x1a7b68e1", "0xde6db608", "0x13d73892", "0xd17f447a", "0xb2221ccf", "0x9db004bd", "0x57d7d367", "0xdbc4cf34", "0x697c009a", "0xc43af1d4", "0x97f12b2e", "0x74c37cd0", "0xc651ea15", "0x665a6d29", "0x22330a2d"], - "dataset_last_index": 268435455, - "dataset_last": "0xa83e7aa6", - "dataset_samples": [{"index": 59471966, "value": "0x3230bc7b"}, {"index": 217795994, "value": "0x7fbfe2c9"}, {"index": 208353206, "value": "0xb2690991"}, {"index": 42483309, "value": "0x1745c7c5"}, {"index": 172547758, "value": "0x0ab0ccaf"}, {"index": 148076330, "value": "0x1bf87d6b"}, {"index": 183853158, "value": "0x160139fd"}, {"index": 214389424, "value": "0x719817ac"}, {"index": 267488061, "value": "0x0155df4b"}, {"index": 169781097, "value": "0xbe1e86c3"}, {"index": 184093494, "value": "0x680bcd6c"}, {"index": 153880993, "value": "0x79c3dc6c"}, {"index": 84977930, "value": "0x181e7e5f"}, {"index": 46426879, "value": "0x0713a109"}, {"index": 3093825, "value": "0xc705dd9f"}, {"index": 225364072, "value": "0x3933b7a8"}, {"index": 44593546, "value": "0xdd1c0431"}, {"index": 260713159, "value": "0x50522b30"}, {"index": 168250303, "value": "0xa0020b38"}, {"index": 52384140, "value": "0xbff39e96"}, {"index": 223401610, "value": "0x21b67e18"}, {"index": 45554030, "value": "0x740f8db3"}, {"index": 95410555, "value": "0x2baba568"}, {"index": 175039924, "value": "0x2c9bef83"}, {"index": 79171087, "value": "0x0ad9b671"}, {"index": 267580473, "value": "0xc4327869"}, {"index": 24168642, "value": "0x7b4fd7d0"}, {"index": 37981670, "value": "0x2c29965f"}, {"index": 171551130, "value": "0xec56f15f"}, {"index": 195559979, "value": "0x61111746"}, {"index": 204611762, "value": "0x303a1d6e"}, {"index": 140997658, "value": "0xbddcfd1a"}, {"index": 138925853, "value": "0xf829a355"}, {"index": 86637313, "value": "0x6d5df2a9"}, {"index": 20736778, "value": "0x01ab8e44"}, {"index": 219665210, "value": "0x06d13507"}, {"index": 160430336, "value": "0xda8dcfc6"}, {"index": 264654675, "value": "0x01a703e1"}, {"index": 8013395, "value": "0xafe7d2c1"}, {"index": 228945585, "value": "0xc091c3a2"}, {"index": 213884386, "value": "0xac1814fe"}, {"index": 104419827, "value": "0x6e6ff62a"}, {"index": 44185464, "value": "0x8fdf01ba"}, {"index": 142737231, "value": "0xdd3f7159"}, {"index": 99284897, "value": "0xdfa0d75c"}, {"index": 132475900, "value": "0x26684c35"}, {"index": 61861762, "value": "0x7f441e63"}, {"index": 132056166, "value": "0x88df2570"}, {"index": 262388043, "value": "0x8aa4d5eb"}, {"index": 91878046, "value": "0xcc816c05"}, {"index": 117353561, "value": "0x434df890"}, {"index": 124768597, "value": "0xcd392ad6"}, {"index": 71352993, "value": "0x1ab4cb63"}, {"index": 190698941, "value": "0x595926fa"}, {"index": 46055428, "value": "0x7cd76b41"}, {"index": 55281366, "value": "0x20cb95c4"}, {"index": 165145231, "value": "0x13cf823f"}, {"index": 106810753, "value": "0xf9daf901"}, {"index": 171985651, "value": "0xff9af40a"}, {"index": 232085256, "value": "0x2c7dfa51"}, {"index": 159510492, "value": "0x871206db"}, {"index": 40072060, "value": "0x938c116c"}, {"index": 209107596, "value": "0xb64bf199"}, {"index": 39023794, "value": "0x5751f874"}], - "cache_head": ["0x355a86d2", "0x7957db1c", "0xd21772af", "0x6fc1e09b", "0xd55ce61d", "0x6e6a278b", "0xd3f543ce", "0x223d8e82", "0x143ab337", "0x2e9f05bd", "0x2eb389bf", "0x0c6e449e", "0x5cfa4222", "0xba6560fe", "0x8e3e1aa4", "0xdbcc1d53"], - "cache_last_line": ["0x41190d91", "0xbd277957", "0x22ddbb49", "0x6986f207", "0xdf69a4d6", "0x26401a3a", "0x818230fb", "0xc417122d", "0x3597b211", "0xb553ce55", "0xcf39cc0d", "0x3b7fc43a", "0x3fd43b00", "0x67e1c80e", "0xffa7ea7d", "0xca2960ab"], - "cache_fnv1a64": "0x48c4f5bf24166b2e" -}