diff --git a/common/speculative.cpp b/common/speculative.cpp index 2d73e48bc8cc..23843abe0139 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -3490,6 +3490,15 @@ common_params common_base_params_to_speculative(const common_params & params) { result.cache_type_k = params_spec.cache_type_k; result.cache_type_v = params_spec.cache_type_v; + // ARC-LAB: the draft context inherits the target's physical batch (2048 on the Arc server = a 784 MiB compute + // buffer for an MTP layer that drafts 1-3 tokens); its prompt catch-up already runs in n_ubatch chunks. + if (const char * ub = getenv("LLAMA_ARG_SPEC_DRAFT_UBATCH")) { + const int32_t n = atoi(ub); + if (n > 0 && n < result.n_ubatch) { + result.n_ubatch = n; + } + } + // the K-cache mean-center bias is calibrated for the target model // (per-head/channel K layout); the draft model has a different K // geometry, so it must not inherit the bias diff --git a/ggml/src/ggml-sycl/CMakeLists.txt b/ggml/src/ggml-sycl/CMakeLists.txt index bd4dc57101af..f60ec4c537f2 100644 --- a/ggml/src/ggml-sycl/CMakeLists.txt +++ b/ggml/src/ggml-sycl/CMakeLists.txt @@ -105,6 +105,15 @@ endif() target_compile_options(ggml-sycl PRIVATE "-Wno-narrowing") +# ARC-LAB: one device image per kernel. The Xe2 XMX kernels (DPAS) then only get JIT-compiled when they are first launched, +# so a GPU without DPAS (Xe-LP, Gen9) never compiles them; with one image per source file, compiling any kernel of +# fattn-dec / mmvq / ptq1-t2 built the DPAS ones too and the shader compiler crashed (UHD 630). OFF = compiler default. +option(GGML_SYCL_DEVICE_CODE_SPLIT_PER_KERNEL "ggml: SYCL device code split per kernel" ON) +if (GGML_SYCL_DEVICE_CODE_SPLIT_PER_KERNEL) + target_compile_options(ggml-sycl PRIVATE "-fsycl-device-code-split=per_kernel") + target_link_options(ggml-sycl PRIVATE "-fsycl-device-code-split=per_kernel") +endif() + message(STATUS "GGML_SYCL_SUPPORT_LEVEL_ZERO_API ${GGML_SYCL_SUPPORT_LEVEL_ZERO_API}") if (GGML_SYCL_SUPPORT_LEVEL_ZERO_API) # Link against Level Zero loader for direct device memory allocation. diff --git a/ggml/src/ggml-sycl/common.hpp b/ggml/src/ggml-sycl/common.hpp index 8f2106d83088..a73b2261daa6 100644 --- a/ggml/src/ggml-sycl/common.hpp +++ b/ggml/src/ggml-sycl/common.hpp @@ -60,6 +60,8 @@ void ggml_sycl_host_free(void* ptr); extern int g_ggml_sycl_debug; extern int g_ggml_sycl_enable_optimize; +extern int g_ggml_sycl_quant_f16_gemm; +extern int g_ggml_sycl_quant_s4_gemm; extern int g_ggml_sycl_enable_fusion; extern int g_ggml_sycl_enable_esimd; extern int g_ggml_sycl_prioritize_dmmv; @@ -225,6 +227,7 @@ inline dpct::err0 ggml_sycl_set_device(const int device) try { struct optimize_feature { bool reorder=false; bool xmx_pq2=false; // PQ2_0 rewritten into the XMX layout (pq2_xmx.hpp); only that path can read it + bool t2=false; // ARC-LAB: PTQ1_0 repacked to the TernSYCL 2-bit layout (ptq1-t2.cpp) }; struct sycl_device_info { @@ -1041,4 +1044,10 @@ static __dpct_inline__ float ggml_sycl_ue4m3_to_fp32(uint8_t x) { return raw * 0.5f; } +// ARC-LAB lab knobs: integer tuning switches read through here can be changed while the process runs. With +// GGML_SYCL_KNOBS_FILE= set, the file (NAME=VALUE lines) is re-read at the start of a graph compute when its +// mtime changes and its values override the environment; without it, each knob is read from the environment once. +int ggml_sycl_knob(const char * name, int def); +void ggml_sycl_knobs_refresh(); + #endif // GGML_SYCL_COMMON_HPP diff --git a/ggml/src/ggml-sycl/concat.cpp b/ggml/src/ggml-sycl/concat.cpp index bd5f3b2ceb30..2392e81cb78e 100644 --- a/ggml/src/ggml-sycl/concat.cpp +++ b/ggml/src/ggml-sycl/concat.cpp @@ -94,6 +94,23 @@ template static void concat_T_sycl(const T *x, const T *y, T *dst, int ne00, int ne01, int ne02, int ne0, int ne1, int ne2, int dim, queue_ptr stream) { + if (dim == 0 && ne0 < SYCL_CONCAT_BLOCK_SIZE) { + // ARC-LAB: narrow rows (delta-net conv state: 3 + 1 columns x 10240 channels) launched one work-group of 256 per + // row with 4 active lanes (12.5 us per call on the B580); one lane per element instead. + const int total = ne0 * ne1 * ne2; + const int nblk = (total + SYCL_CONCAT_BLOCK_SIZE - 1) / SYCL_CONCAT_BLOCK_SIZE; + stream->parallel_for(sycl::nd_range<1>(nblk * SYCL_CONCAT_BLOCK_SIZE, SYCL_CONCAT_BLOCK_SIZE), + [=](sycl::nd_item<1> it) { + const int i = it.get_global_id(0); + if (i >= total) { + return; + } + const int i0 = i % ne0; + const int row = i / ne0; // i1 + i2 * ne1 + dst[i] = i0 < ne00 ? x[row * ne00 + i0] : y[row * (ne0 - ne00) + i0 - ne00]; + }); + return; + } int num_blocks = (ne0 + SYCL_CONCAT_BLOCK_SIZE - 1) / SYCL_CONCAT_BLOCK_SIZE; sycl::range<3> gridDim(ne2, ne1, num_blocks); switch (dim) { diff --git a/ggml/src/ggml-sycl/convert.cpp b/ggml/src/ggml-sycl/convert.cpp index b046091512ea..f30625fe8b11 100644 --- a/ggml/src/ggml-sycl/convert.cpp +++ b/ggml/src/ggml-sycl/convert.cpp @@ -170,6 +170,35 @@ static void dequantize_row_q4_0_sycl_reorder(const void *vx, dst_t *y, const int } +template +static void dequantize_row_pq2_0_sycl_reorder(const void * vx, dst_t * y, const int64_t k, dpct::queue_ptr stream) { + GGML_ASSERT(k % QK_PQ2_0 == 0); + const uint8_t * qs = (const uint8_t *) vx; + const sycl::half * d = (const sycl::half *) (qs + k / 4); + stream->parallel_for(sycl::range<1>(k / 4), [=](sycl::id<1> i) { + const float dv = d[i / (QK_PQ2_0 / 4)]; + const uint8_t q = qs[i]; +#pragma unroll + for (int j = 0; j < 4; ++j) { + y[i * 4 + j] = (float) (((q >> (2 * j)) & 3) - 1) * dv; + } + }); +} + +template +static void dequantize_row_ptq1_0_sycl_reorder(const void * vx, dst_t * y, const int64_t k, dpct::queue_ptr stream) { + GGML_ASSERT(k % QK_PTQ1_0 == 0); + const int64_t nb = k / QK_PTQ1_0; + const uint8_t * qs = (const uint8_t *) vx; + const uint8_t * qh = qs + nb * 24; + const sycl::half * d = (const sycl::half *) (qs + nb * 26); + stream->parallel_for(sycl::range<1>(k), [=](sycl::id<1> id) { + const int64_t i = id[0]; + const int64_t ib = i / QK_PTQ1_0; + y[i] = (float) (ptq1_0_trit(qs + ib * 24, qh + ib * 2, (int) (i % QK_PTQ1_0)) - 1) * (float) d[ib]; + }); +} + template static void dequantize_row_q8_0_sycl_reorder(const void *vx, dst_t *y, const int64_t k, dpct::queue_ptr stream) { @@ -657,12 +686,20 @@ to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type, ggml_tensor * dst) { switch (type) { case GGML_TYPE_Q1_0: return dequantize_block_sycl; + case GGML_TYPE_Q2_0: + return dequantize_block_sycl; case GGML_TYPE_PTQ1_0: + if (dst->src[0]->extra && + ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { + return dequantize_row_ptq1_0_sycl_reorder; + } return dequantize_block_sycl; case GGML_TYPE_PQ2_0: + if (dst->src[0]->extra && + ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { + return dequantize_row_pq2_0_sycl_reorder; + } return dequantize_block_sycl; - case GGML_TYPE_Q2_0: - return dequantize_block_sycl; case GGML_TYPE_Q4_0: if (dst->src[0]->extra && ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) { @@ -751,12 +788,20 @@ to_fp32_sycl_t ggml_get_to_fp32_sycl(ggml_type type, ggml_tensor *dst) { switch (type) { case GGML_TYPE_Q1_0: return dequantize_block_sycl; + case GGML_TYPE_Q2_0: + return dequantize_block_sycl; case GGML_TYPE_PTQ1_0: + if (dst->src[0]->extra && + ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { + return dequantize_row_ptq1_0_sycl_reorder; + } return dequantize_block_sycl; case GGML_TYPE_PQ2_0: + if (dst->src[0]->extra && + ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { + return dequantize_row_pq2_0_sycl_reorder; + } return dequantize_block_sycl; - case GGML_TYPE_Q2_0: - return dequantize_block_sycl; case GGML_TYPE_Q4_0: if (dst->src[0]->extra && ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) { @@ -887,3 +932,68 @@ to_fp16_nc_sycl_t ggml_get_to_fp16_nc_sycl(ggml_type type) { return nullptr; } } + +void ggml_sycl_quant_to_s4(ggml_type type, bool reordered, const void * vx, uint8_t * w4, float * scales, + int64_t nrows, int64_t ncols, dpct::queue_ptr stream) { + GGML_ASSERT(type == GGML_TYPE_PQ2_0 || type == GGML_TYPE_PTQ1_0); + GGML_ASSERT(ncols % 128 == 0); + const int64_t nb = nrows * ncols / 128; + const uint8_t * base = (const uint8_t *) vx; + const bool ptq = type == GGML_TYPE_PTQ1_0; + // one work item = 8 elements = one uint32 of s4 + stream->parallel_for(sycl::range<1>(nb * 16), [=](sycl::id<1> id) { + const int64_t ib = id[0] / 16; + const int c = (int) (id[0] % 16); + int v[8]; + sycl::half d; + if (ptq) { + const uint8_t * qs; + const uint8_t * qh; + if (reordered) { + qs = base + ib * 24; + qh = base + nb * 24 + ib * 2; + d = ((const sycl::half *) (base + nb * 26))[ib]; + } else { + const block_ptq1_0 * x = (const block_ptq1_0 *) vx + ib; + qs = x->qs; + qh = x->qh; + d = x->d; + } + // trit t of byte b = ((b * 3^t) mod 256) * 3 >> 8 + constexpr uint32_t pow3[5] = { 1, 3, 9, 27, 81 }; + if (c < 10) { // elements t*16 + m, 8 consecutive m + const uint32_t p = pow3[c / 2]; + const int m = (c % 2) * 8; +#pragma unroll + for (int i = 0; i < 8; ++i) v[i] = (int) ((((qs[m + i] * p) & 0xFF) * 3) >> 8) - 1; + } else if (c < 15) { // elements 80 + t*8 + m, m = 0..7 + const uint32_t p = pow3[c - 10]; +#pragma unroll + for (int i = 0; i < 8; ++i) v[i] = (int) ((((qs[16 + i] * p) & 0xFF) * 3) >> 8) - 1; + } else { // elements 120 + t*2 + h +#pragma unroll + for (int i = 0; i < 8; ++i) v[i] = (int) ((((qh[i % 2] * pow3[i / 2]) & 0xFF) * 3) >> 8) - 1; + } + } else { + const uint8_t * qs; + if (reordered) { + qs = base + ib * 32; + d = ((const sycl::half *) (base + nb * 32))[ib]; + } else { + const block_pq2_0 * x = (const block_pq2_0 *) vx + ib; + qs = x->qs; + d = x->d; + } +#pragma unroll + for (int i = 0; i < 8; ++i) v[i] = ((qs[2 * c + i / 4] >> (2 * (i % 4))) & 3) - 1; + } + uint32_t out = 0; +#pragma unroll + for (int i = 0; i < 8; ++i) out |= (uint32_t) (v[i] & 0xF) << (4 * i); + ((uint32_t *) w4)[id[0]] = out; + if (c == 0) { + const int64_t bpr = ncols / 128; + scales[(ib % bpr) * nrows + ib / bpr] = d; // [group][row] + } + }); +} diff --git a/ggml/src/ggml-sycl/convert.hpp b/ggml/src/ggml-sycl/convert.hpp index 8de79d10ff69..bbd9fe124184 100644 --- a/ggml/src/ggml-sycl/convert.hpp +++ b/ggml/src/ggml-sycl/convert.hpp @@ -21,6 +21,11 @@ typedef to_t_sycl_t to_fp32_sycl_t; typedef to_t_sycl_t to_fp16_sycl_t; to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type, ggml_tensor * dst); + +// PQ2_0 / PTQ1_0 rows -> packed s4 (low nibble = even element) + f32 scales laid out [group][row]. +// All their values (-1..2) are exact in s4. +void ggml_sycl_quant_to_s4(ggml_type type, bool reordered, const void * vx, uint8_t * w4, float * scales, + int64_t nrows, int64_t ncols, dpct::queue_ptr stream); to_fp32_sycl_t ggml_get_to_fp32_sycl(ggml_type type, ggml_tensor * dst); #ifdef GGML_SYCL_HAS_BF16 diff --git a/ggml/src/ggml-sycl/dequantize.hpp b/ggml/src/ggml-sycl/dequantize.hpp index 6bb4dd4515a9..076380eb05a9 100644 --- a/ggml/src/ggml-sycl/dequantize.hpp +++ b/ggml/src/ggml-sycl/dequantize.hpp @@ -47,6 +47,26 @@ static __dpct_inline__ void dequantize_q2_0(const void *vx, const int64_t ib, #endif // GGML_SYCL_F16 } +// Trit of element e (0..127) in a PTQ1_0 block: 16 bytes x 5, 8 bytes x 5, then 2 qh bytes x 4. +static __dpct_inline__ int ptq1_0_trit(const uint8_t * qs, const uint8_t * qh, const int e) { + uint32_t v; + int t; + if (e < 80) { + v = qs[e % 16]; + t = e / 16; + } else if (e < 120) { + v = qs[16 + (e - 80) % 8]; + t = (e - 80) / 8; + } else { + v = qh[(e - 120) % 2]; + t = (e - 120) / 2; + } + for (int i = 0; i < t; ++i) { + v = (v * 3) & 0xFF; + } + return (int) ((v * 3) >> 8); +} + static __dpct_inline__ void dequantize_q4_0(const void *vx, const int64_t ib, const int iqs, dfloat2 &v) { const block_q4_0 * x = (const block_q4_0 *) vx; diff --git a/ggml/src/ggml-sycl/dmmv.cpp b/ggml/src/ggml-sycl/dmmv.cpp index d47d6831a359..4995ffece14f 100644 --- a/ggml/src/ggml-sycl/dmmv.cpp +++ b/ggml/src/ggml-sycl/dmmv.cpp @@ -238,6 +238,34 @@ static void convert_mul_mat_vec_f16_sycl(const void *vx, const dfloat *y, } #ifdef GGML_SYCL_DMMV_HAS_BF16 +// ARC-LAB: small-output bf16 mat-vec (Bonsai's delta-net gate projections: 48 x 5120, 96 calls per token). +// The generic dequantize_mul_mat_vec gives each row one sub-group, so 48 rows used 48 sub-groups and reached +// ~25 GB/s on the B580 (~20 us per call). Here each row gets a 256-wide work-group that splits K, loads 8 bf16 +// at a time and reduces over the group. +static void bf16_gemv_rowsplit_sycl(const void * vx, const dfloat * y, float * dst, const int ncols, + const int nrows, dpct::queue_ptr stream) { + constexpr int WG = 256; + stream->parallel_for( + sycl::nd_range<1>(sycl::range<1>((size_t) nrows * WG), sycl::range<1>(WG)), + [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + const int row = it.get_group(0); + const int tid = it.get_local_id(0); + const uint16_t * x = (const uint16_t *) vx + (size_t) row * ncols; + float sum = 0.0f; + for (int i = tid * 8; i < ncols; i += WG * 8) { + const sycl::vec w = *(const sycl::vec *) (x + i); +#pragma unroll + for (int j = 0; j < 8; ++j) { + sum += sycl::bit_cast((uint32_t) w[j] << 16) * (float) y[i + j]; + } + } + sum = sycl::reduce_over_group(it.get_group(), sum, sycl::plus()); + if (tid == 0) { + dst[row] = sum; + } + }); +} + static void convert_mul_mat_vec_bf16_sycl(const void *vx, const dfloat *y, float *dst, const int ncols, const int nrows, @@ -2209,9 +2237,15 @@ void ggml_sycl_op_dequantize_mul_mat_vec( convert_mul_mat_vec_f16_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); break; #ifdef GGML_SYCL_DMMV_HAS_BF16 - case GGML_TYPE_BF16: - convert_mul_mat_vec_bf16_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); + case GGML_TYPE_BF16: { + static const bool rowsplit = getenv("GGML_SYCL_BF16_ROWSPLIT_OFF") == nullptr; + if (rowsplit && row_diff <= 512 && ne00 % 8 == 0 && ((uintptr_t) src0_dd_i % 16) == 0) { + bf16_gemv_rowsplit_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); + } else { + convert_mul_mat_vec_bf16_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); + } break; + } #endif default: printf("ggml_sycl_op_dequantize_mul_mat_vec unsupported GGML_TYPE %d\n", src0->type); diff --git a/ggml/src/ggml-sycl/fattn-dec.cpp b/ggml/src/ggml-sycl/fattn-dec.cpp new file mode 100644 index 000000000000..86b74caabb89 --- /dev/null +++ b/ggml/src/ggml-sycl/fattn-dec.cpp @@ -0,0 +1,1352 @@ +// ARC-LAB decode attention for a q4_0 KV cache (generation and MTP verify batches of 1-4 query tokens). +// +// The TILE path first converts the whole K and V cache to f16 on every call and then reads that copy +// (Bonsai 27B at 16K context: 5.9 + 11.4 ms per generated token over 16 attention layers). Here each +// work-group owns one KV head and a slice of the context, reads the q4_0 blocks directly, and serves all +// G query heads of that KV head (GQA) and all NQ query tokens from the one read (flash-decoding): +// - the scaled query rows (G * NQ of them) sit in SLM; +// - per tile of TK keys: thread (c, key) computes the G scores of query token c against its key, +// straight from the q4_0 nibbles; per-row online softmax with a finite running max (a fully masked +// tile or slice gets zero weight, never NaN); thread d accumulates P.V for head dim d of every row; +// - slices are merged by flash_attn_combine_results (same partial layout as the vec kernel). + +#include "xe2-dpas.hpp" +#include +#include +#include "dpct/helper.hpp" +#include "common.hpp" +#include "fattn-common.hpp" +#include "fattn-dec.hpp" +#include "ptq1-t2.hpp" +#include + +namespace { +constexpr int DEC_D = 256; // head size (K and V) +constexpr int DEC_WG = 256; // work-group size = DEC_D: thread d owns output dim d +constexpr float DEC_MINF = -1.0e30f; // finite "minus infinity" for the running max +} + +// v3 kernel, kept for 4-token batches: the pipelined kernel below is slower there (436 vs 508 us per layer at +// 16K; see README) and faster at 1-2 tokens (16K 273 -> 256, 48K 944 -> 742 us). +template +static void fattn_dec_q4_0_v3(const char * Q, const char * K, const char * V, const char * mask, float * dst, + float * parts, sycl::float2 * meta, float scale, int ne01, int ne02, int ne11, + int nkvh, int ne03, int64_t nb01, int64_t nb02, int64_t nb03, int64_t nb11, + int64_t nb12, int64_t nb13, int64_t nb21, int64_t nb22, int64_t nb23, int64_t nb31, + int64_t nb33, int ne33, int nsplit, int chunk, dpct::queue_ptr stream) { + constexpr int R = G * NQ; // query rows per KV head + constexpr int RP = (R + 3) / 4 * 4; // padded row stride of the transposed scores + constexpr int TK = DEC_WG / NQ; // keys per tile: thread (c, jj) = (tid / TK, tid % TK) + constexpr int NB = DEC_D / QK4_0; + constexpr int KW = DEC_D / QK4_0 * sizeof(block_q4_0) / 4; // 36 dwords per q4_0 K row (D 256) + constexpr int KWP = KW + 1; // padded SLM row stride (bank conflicts) + static_assert(TK % 16 == 0 && (DEC_D / QK4_0 * sizeof(block_q4_0)) % 4 == 0, "shape"); + + stream->submit([&](sycl::handler & cgh) { + sycl::local_accessor sQ(sycl::range<1>(R * DEC_D / 4), cgh); + sycl::local_accessor sK(sycl::range<1>(TK * KWP), cgh); + sycl::local_accessor sS(sycl::range<1>(TK * RP / 4), cgh); // [key][row], transposed + sycl::local_accessor sM(sycl::range<1>(R), cgh); + sycl::local_accessor sL(sycl::range<1>(R), cgh); + sycl::local_accessor sA(sycl::range<1>(RP), cgh); + cgh.parallel_for( + sycl::nd_range<3>(sycl::range<3>(ne03, nkvh, (size_t) nsplit * DEC_WG), sycl::range<3>(1, 1, DEC_WG)), + [=](sycl::nd_item<3> it) [[sycl::reqd_sub_group_size(16)]] { + const int tid = it.get_local_id(2); + const int split = it.get_group(2); + const int kvh = it.get_group(1); + const int seq = it.get_group(0); + auto sg = it.get_sub_group(); + const int w = tid / 16; + const int lane = tid % 16; + float * sSf = (float *) &sS[0]; + + // query rows r = c * G + g: token c, head kvh * G + g; pre-scaled like the other FA kernels + for (int e = tid; e < R * DEC_D / 4; e += DEC_WG) { + const int r = e / (DEC_D / 4), d4 = e % (DEC_D / 4), c = r / G, g = r % G; + sycl::float4 qv(0.0f); + if (c < ne01) { + qv = *(const sycl::float4 *) (Q + seq * nb03 + c * nb01 + (int64_t) (kvh * G + g) * nb02 + d4 * sizeof(sycl::float4)); + } + sQ[e] = qv * scale; + } + if (tid < R) { + sM[tid] = DEC_MINF; + sL[tid] = 0.0f; + } + if (tid < RP) { + sA[tid] = 1.0f; + } + + const int k_begin = split * chunk; + const int k_end = sycl::min(k_begin + chunk, ne11); + const char * Kh = K + seq * nb13 + kvh * nb12; + const char * Vh = V + seq * nb23 + kvh * nb22; + const sycl::half * mrow = mask ? (const sycl::half *) (mask + (seq % ne33) * nb33) : nullptr; + + float o[RP]; +#pragma unroll + for (int r = 0; r < RP; ++r) { + o[r] = 0.0f; + } + + const int c = tid / TK; + const int jj = tid % TK; + for (int t0 = k_begin; t0 < k_end; t0 += TK) { + // K tile -> SLM: raw q4_0 rows, coalesced aligned dwords + for (int e = tid; e < TK * KW; e += DEC_WG) { + const int kk = e / KW, wd = e % KW, j = t0 + kk; + sK[kk * KWP + wd] = j < k_end ? ((const uint32_t *) (Kh + j * nb11))[wd] : 0u; + } + it.barrier(sycl::access::fence_space::local_space); + + // scores: this thread's key against the G heads of query token c + { + const int j = t0 + jj; + float s[G]; +#pragma unroll + for (int g = 0; g < G; ++g) { + s[g] = 0.0f; + } + if (j < k_end && c < ne01) { + const int kr = jj * KWP; + const int q0 = c * G * (DEC_D / 4); +#pragma unroll 2 + for (int b = 0; b < NB; ++b) { + // block b = bytes [18b, 18b + 18): scale (2 bytes) then 16 bytes of nibbles + auto byte_at = [&](int o) -> uint32_t { + return (sK[kr + o / 4] >> (8 * (o % 4))) & 0xFFu; + }; + const int o0 = b * (int) sizeof(block_q4_0); + const uint16_t hb = (uint16_t) (byte_at(o0) | (byte_at(o0 + 1) << 8)); + const float dk = static_cast(sycl::bit_cast(hb)); +#pragma unroll + for (int i = 0; i < QK4_0 / 2; i += 4) { + sycl::float4 k0, k1; +#pragma unroll + for (int u = 0; u < 4; ++u) { + const uint32_t by = byte_at(o0 + 2 + i + u); + k0[u] = (float) ((int) (by & 0xF) - 8) * dk; + k1[u] = (float) ((int) (by >> 4) - 8) * dk; + } + const int d4 = (b * QK4_0 + i) / 4; +#pragma unroll + for (int g = 0; g < G; ++g) { + const sycl::float4 qa = sQ[q0 + g * (DEC_D / 4) + d4]; + const sycl::float4 qb = sQ[q0 + g * (DEC_D / 4) + d4 + QK4_0 / 8]; + s[g] += sycl::dot(qa, k0) + sycl::dot(qb, k1); + } + } + } + const float mv = mrow ? static_cast(mrow[(c * nb31) / (int64_t) sizeof(sycl::half) + j]) : 0.0f; +#pragma unroll + for (int g = 0; g < G; ++g) { + s[g] += mv; + } + } else { +#pragma unroll + for (int g = 0; g < G; ++g) { + s[g] = -INFINITY; + } + } +#pragma unroll + for (int g = 0; g < G; ++g) { + sSf[jj * RP + c * G + g] = s[g]; + } + } + it.barrier(sycl::access::fence_space::local_space); + + // V tile -> SLM (reusing the K tile buffer; scores are done with it) + static_assert(sizeof(block_q4_0) * NB % 4 == 0, "V rows are dword multiples"); + for (int e = tid; e < TK * KW; e += DEC_WG) { + const int kk = e / KW, wd = e % KW, j = t0 + kk; + sK[kk * KWP + wd] = j < k_end ? ((const uint32_t *) (Vh + j * nb21))[wd] : 0u; + } + + // online softmax, one row per subgroup at a time + for (int r = w; r < R; r += DEC_WG / 16) { + float mt = -INFINITY; + for (int k = lane; k < TK; k += 16) { + mt = sycl::fmax(mt, sSf[k * RP + r]); + } + mt = sycl::reduce_over_group(sg, mt, sycl::maximum()); + const float m_old = sM[r]; + const float m_new = sycl::fmax(m_old, mt); // finite: m_old starts at DEC_MINF + float lt = 0.0f; + for (int k = lane; k < TK; k += 16) { + const float p = sycl::native::exp(sSf[k * RP + r] - m_new); // exp(-inf) = 0 + sSf[k * RP + r] = p; + lt += p; + } + lt = sycl::reduce_over_group(sg, lt, sycl::plus()); + if (lane == 0) { + const float a = sycl::native::exp(m_old - m_new); + sA[r] = a; + sL[r] = sL[r] * a + lt; + sM[r] = m_new; + } + } + it.barrier(sycl::access::fence_space::local_space); + + // P.V for head dim d = tid (padded rows R..RP-1 have P = 0 from the -inf scores) + { + const int d = tid; + const int b = d / QK4_0; + const int wi = d % QK4_0; + const int nk = sycl::min(TK, k_end - t0); +#pragma unroll + for (int r = 0; r < RP; ++r) { + o[r] *= sA[r]; + } + const int ob = b * (int) sizeof(block_q4_0); // scale bytes ob, ob + 1 + const int oq = ob + 2 + wi % (QK4_0 / 2); // this dim's nibble byte + const int sh = wi < QK4_0 / 2 ? 0 : 4; +#pragma unroll 4 + for (int k = 0; k < nk; ++k) { + const int vr = k * KWP; + const uint32_t sw = sK[vr + ob / 4] >> (8 * (ob % 4)); // ob even: both scale bytes in one dword + const float dv = static_cast(sycl::bit_cast((uint16_t) (sw & 0xFFFFu))); + const int byte = (sK[vr + oq / 4] >> (8 * (oq % 4))) & 0xFF; + const float vv = (float) (((byte >> sh) & 0xF) - 8) * dv; +#pragma unroll + for (int r4 = 0; r4 < RP / 4; ++r4) { + const sycl::float4 p = sS[k * (RP / 4) + r4]; + o[4 * r4 + 0] += p[0] * vv; + o[4 * r4 + 1] += p[1] * vv; + o[4 * r4 + 2] += p[2] * vv; + o[4 * r4 + 3] += p[3] * vv; + } + } + } + it.barrier(sycl::access::fence_space::local_space); + } + +#pragma unroll + for (int r = 0; r < R; ++r) { + const int cr = r / G; + if (cr >= ne01) { + continue; + } + const int64_t jdu = ((int64_t) seq * ne01 + cr) * ne02 + kvh * G + r % G; + if (nsplit == 1) { + const float l = sL[r]; + dst[jdu * DEC_D + tid] = l > 0.0f ? o[r] / l : 0.0f; + } else { + parts[(jdu * nsplit + split) * DEC_D + tid] = o[r]; + if (tid == 0) { + meta[jdu * nsplit + split] = sycl::float2(sM[r], sL[r]); + } + } + } + }); + }); +} + +template +static void fattn_dec_q4_0(const char * Q, const char * K, const char * V, const char * mask, float * dst, + float * parts, sycl::float2 * meta, float scale, int ne01, int ne02, int ne11, + int nkvh, int ne03, int64_t nb01, int64_t nb02, int64_t nb03, int64_t nb11, + int64_t nb12, int64_t nb13, int64_t nb21, int64_t nb22, int64_t nb23, int64_t nb31, + int64_t nb33, int ne33, int nsplit, int chunk, dpct::queue_ptr stream) { + constexpr int R = G * NQ; // query rows per KV head + constexpr int RP = (R + 3) / 4 * 4; // padded row stride of the transposed scores + constexpr int NRG = DEC_WG / TK; // row groups in the score phase: thread (rg, jj) = (tid / TK, tid % TK) + constexpr int RPT = R / NRG; // rows per thread in the score phase + constexpr int NB = DEC_D / QK4_0; + constexpr int KW = DEC_D / QK4_0 * sizeof(block_q4_0) / 4; // 36 dwords per q4_0 row (D 256) + constexpr int KWP = KW + 1; // padded SLM row stride (bank conflicts) + constexpr int PKW = TK * KW / DEC_WG; // dwords of a K or V tile per thread + static_assert(TK % 16 == 0 && R % NRG == 0 && (TK * KW) % DEC_WG == 0 && G % RPT == 0, "shape"); // a thread's rows share one token + + stream->submit([&](sycl::handler & cgh) { + sycl::local_accessor sQ(sycl::range<1>(R * DEC_D / 4), cgh); + sycl::local_accessor sK(sycl::range<1>(TK * KWP), cgh); // K tile, then V tile + sycl::local_accessor sS(sycl::range<1>(TK * RP / 4), cgh); // [key][row], transposed + sycl::local_accessor sM(sycl::range<1>(R), cgh); + sycl::local_accessor sL(sycl::range<1>(R), cgh); + sycl::local_accessor sA(sycl::range<1>(RP), cgh); + cgh.parallel_for( + sycl::nd_range<3>(sycl::range<3>(ne03, nkvh, (size_t) nsplit * DEC_WG), sycl::range<3>(1, 1, DEC_WG)), + [=](sycl::nd_item<3> it) [[sycl::reqd_sub_group_size(16)]] { + const int tid = it.get_local_id(2); + const int split = it.get_group(2); + const int kvh = it.get_group(1); + const int seq = it.get_group(0); + auto sg = it.get_sub_group(); + const int w = tid / 16; + const int lane = tid % 16; + float * sSf = (float *) &sS[0]; + + const int k_begin = split * chunk; + const int k_end = sycl::min(k_begin + chunk, ne11); + const char * Kh = K + seq * nb13 + kvh * nb12; + const char * Vh = V + seq * nb23 + kvh * nb22; + const sycl::half * mrow = mask ? (const sycl::half *) (mask + (seq % ne33) * nb33) : nullptr; + + // software pipeline: a tile's raw rows are loaded into registers one phase before they are stored + // to SLM (V during Q.K, the next K during softmax + P.V), so the loads overlap compute + uint32_t pk[PKW], pv[PKW]; + auto load_rows = [&](const char * base, int64_t nb, int t0, uint32_t * reg) { +#pragma unroll + for (int i = 0; i < PKW; ++i) { + const int e = tid + i * DEC_WG, kk = e / KW, wd = e % KW, j = t0 + kk; + reg[i] = j < k_end ? ((const uint32_t *) (base + j * nb))[wd] : 0u; + } + }; + auto store_rows = [&](const uint32_t * reg) { +#pragma unroll + for (int i = 0; i < PKW; ++i) { + const int e = tid + i * DEC_WG, kk = e / KW, wd = e % KW; + sK[kk * KWP + wd] = reg[i]; + } + }; + if constexpr (PIPE) { + load_rows(Kh, nb11, k_begin, pk); + } + + // query rows r = c * G + g: token c, head kvh * G + g; pre-scaled like the other FA kernels + for (int e = tid; e < R * DEC_D / 4; e += DEC_WG) { + const int r = e / (DEC_D / 4), d4 = e % (DEC_D / 4), c = r / G, g = r % G; + sycl::float4 qv(0.0f); + if (c < ne01) { + qv = *(const sycl::float4 *) (Q + seq * nb03 + c * nb01 + (int64_t) (kvh * G + g) * nb02 + d4 * sizeof(sycl::float4)); + } + sQ[e] = qv * scale; + } + if (tid < R) { + sM[tid] = DEC_MINF; + sL[tid] = 0.0f; + } + if (tid < RP) { + sA[tid] = 1.0f; + } + + float o[RP]; +#pragma unroll + for (int r = 0; r < RP; ++r) { + o[r] = 0.0f; + } + + const int rg = tid / TK; + const int jj = tid % TK; + const int r0 = rg * RPT; + for (int t0 = k_begin; t0 < k_end; t0 += TK) { + if constexpr (!PIPE) { + load_rows(Kh, nb11, t0, pk); + } + store_rows(pk); // K tile t + it.barrier(sycl::access::fence_space::local_space); + if constexpr (PIPE) { + load_rows(Vh, nb21, t0, pv); // V tile t, in flight during Q.K + } + + // scores of rows r0 .. r0 + RPT - 1 against this thread's key + { + const int j = t0 + jj; + float s[RPT]; +#pragma unroll + for (int i = 0; i < RPT; ++i) { + s[i] = 0.0f; + } + if (j < k_end) { + const int kr = jj * KWP; + const int qb = r0 * (DEC_D / 4); // hoisted row base (v3 form): row rr at qb + rr * 64 +#pragma unroll 2 + for (int b = 0; b < NB; ++b) { + auto byte_at = [&](int o) -> uint32_t { + return (sK[kr + o / 4] >> (8 * (o % 4))) & 0xFFu; + }; + const int o0 = b * (int) sizeof(block_q4_0); + const uint16_t hb = (uint16_t) (byte_at(o0) | (byte_at(o0 + 1) << 8)); + const float dk = static_cast(sycl::bit_cast(hb)); +#pragma unroll + for (int i = 0; i < QK4_0 / 2; i += 4) { + sycl::float4 k0, k1; +#pragma unroll + for (int u = 0; u < 4; ++u) { + const uint32_t by = byte_at(o0 + 2 + i + u); + k0[u] = (float) ((int) (by & 0xF) - 8) * dk; + k1[u] = (float) ((int) (by >> 4) - 8) * dk; + } + const int d4 = (b * QK4_0 + i) / 4; +#pragma unroll + for (int rr = 0; rr < RPT; ++rr) { + const sycl::float4 qa = sQ[qb + rr * (DEC_D / 4) + d4]; + const sycl::float4 qc = sQ[qb + rr * (DEC_D / 4) + d4 + QK4_0 / 8]; + s[rr] += sycl::dot(qa, k0) + sycl::dot(qc, k1); + } + } + } + } + const int c = r0 / G; // all RPT rows of this thread belong to token c + const float mv = (j < k_end && c < ne01 && mrow) ? static_cast(mrow[(c * nb31) / (int64_t) sizeof(sycl::half) + j]) : 0.0f; +#pragma unroll + for (int rr = 0; rr < RPT; ++rr) { + sSf[jj * RP + r0 + rr] = (j < k_end && c < ne01) ? s[rr] + mv : -INFINITY; + } + } + it.barrier(sycl::access::fence_space::local_space); + + if constexpr (!PIPE) { + load_rows(Vh, nb21, t0, pv); + } + store_rows(pv); // V tile t -> SLM (the K tile is done) + if constexpr (PIPE) { + if (t0 + TK < k_end) { + load_rows(Kh, nb11, t0 + TK, pk); // K tile t + 1, in flight during softmax + P.V + } + } + + // online softmax, one row per subgroup at a time + for (int r = w; r < R; r += DEC_WG / 16) { + float mt = -INFINITY; + for (int k = lane; k < TK; k += 16) { + mt = sycl::fmax(mt, sSf[k * RP + r]); + } + mt = sycl::reduce_over_group(sg, mt, sycl::maximum()); + const float m_old = sM[r]; + const float m_new = sycl::fmax(m_old, mt); // finite: m_old starts at DEC_MINF + float lt = 0.0f; + for (int k = lane; k < TK; k += 16) { + const float p = sycl::native::exp(sSf[k * RP + r] - m_new); // exp(-inf) = 0 + sSf[k * RP + r] = p; + lt += p; + } + lt = sycl::reduce_over_group(sg, lt, sycl::plus()); + if (lane == 0) { + const float a = sycl::native::exp(m_old - m_new); + sA[r] = a; + sL[r] = sL[r] * a + lt; + sM[r] = m_new; + } + } + it.barrier(sycl::access::fence_space::local_space); + + // P.V for head dim d = tid (padded rows R..RP-1 are never written out) + { + const int d = tid; + const int b = d / QK4_0; + const int wi = d % QK4_0; + const int nk = sycl::min(TK, k_end - t0); +#pragma unroll + for (int r = 0; r < RP; ++r) { + o[r] *= sA[r]; + } + const int ob = b * (int) sizeof(block_q4_0); // scale bytes ob, ob + 1 + const int oq = ob + 2 + wi % (QK4_0 / 2); // this dim's nibble byte + const int sh = wi < QK4_0 / 2 ? 0 : 4; +#pragma unroll 4 + for (int k = 0; k < nk; ++k) { + const int vr = k * KWP; + const uint32_t sw = sK[vr + ob / 4] >> (8 * (ob % 4)); // ob even: both scale bytes in one dword + const float dv = static_cast(sycl::bit_cast((uint16_t) (sw & 0xFFFFu))); + const int byte = (sK[vr + oq / 4] >> (8 * (oq % 4))) & 0xFF; + const float vv = (float) (((byte >> sh) & 0xF) - 8) * dv; +#pragma unroll + for (int r4 = 0; r4 < RP / 4; ++r4) { + const sycl::float4 p = sS[k * (RP / 4) + r4]; + o[4 * r4 + 0] += p[0] * vv; + o[4 * r4 + 1] += p[1] * vv; + o[4 * r4 + 2] += p[2] * vv; + o[4 * r4 + 3] += p[3] * vv; + } + } + } + it.barrier(sycl::access::fence_space::local_space); + } + +#pragma unroll + for (int r = 0; r < R; ++r) { + const int cr = r / G; + if (cr >= ne01) { + continue; + } + const int64_t jdu = ((int64_t) seq * ne01 + cr) * ne02 + kvh * G + r % G; + if (nsplit == 1) { + const float l = sL[r]; + dst[jdu * DEC_D + tid] = l > 0.0f ? o[r] / l : 0.0f; + } else { + parts[(jdu * nsplit + split) * DEC_D + tid] = o[r]; + if (tid == 0) { + meta[jdu * nsplit + split] = sycl::float2(sM[r], sL[r]); + } + } + } + }); + }); +} + +// ARC-LAB XMX variant for 3-4 query tokens (MTP verify): the scalar kernels are f32-ALU bound there (Q.K 163 + P.V +// 121 of 436 us per layer at 16K). Per tile of TK keys: K dequantized to f16 in SLM ([key][dim]); S^T = K Q^T on +// joint_matrix (A = 8 keys x 16 dims, B = Q^T 16 dims x 16 rows) stored straight into the transposed score layout; +// per-row online softmax writes P (f16, [row][key]); V dequantized into the same buffer; O += P V on joint_matrix +// (subgroup s owns head dims 16 s .. +15 for all row tiles), rows rescaled through the coordinate-aware apply. +template +static void fattn_dec_q4_0_xmx(const char * Q, const char * K, const char * V, const char * mask, float * dst, + float * parts, sycl::float2 * meta, float scale, int ne01, int ne02, int ne11, + int nkvh, int ne03, int64_t nb01, int64_t nb02, int64_t nb03, int64_t nb11, + int64_t nb12, int64_t nb13, int64_t nb21, int64_t nb22, int64_t nb23, int64_t nb31, + int64_t nb33, int ne33, int nsplit, int chunk, dpct::queue_ptr stream) { + namespace jm = sycl::ext::oneapi::experimental::matrix; + namespace ijm = sycl::ext::intel::experimental::matrix; + constexpr int R = G * NQ; // query rows per KV head (24) + constexpr int RP = (R + 15) / 16 * 16; // rows padded to whole 16-wide B tiles (32) + constexpr int MT = R / 8; // 8-row tiles of real rows (3) + constexpr int NB = DEC_D / QK4_0; + constexpr int DS = DEC_D + 16; // f16 row stride of the K / V tile ([key][dim]) + constexpr int PS = TK + 16; // f16 row stride of P ([row][key]) + constexpr int NSG = DEC_WG / 16; // 16 subgroups + constexpr int KW = DEC_D / QK4_0 * sizeof(block_q4_0) / 4; // 36 dwords per raw q4_0 row + constexpr int KWP = KW + 1; + static_assert(R % 8 == 0 && TK % 16 == 0 && (TK / 8) * (RP / 16) <= NSG && DEC_D / 16 == NSG, "shape"); + + stream->submit([&](sycl::handler & cgh) { + sycl::local_accessor sQT(sycl::range<1>(DEC_D * RP), cgh); // Q^T: [dim][row], scaled + sycl::local_accessor sKV(sycl::range<1>(TK * DS), cgh); // K tile, then V tile + sycl::local_accessor sRaw(sycl::range<1>(TK * KWP), cgh); // raw q4_0 rows of the tile + sycl::local_accessor sS(sycl::range<1>(TK * RP), cgh); // S^T: [key][row] + sycl::local_accessor sP(sycl::range<1>(RP * PS), cgh); // P: [row][key] + sycl::local_accessor sM(sycl::range<1>(R), cgh); + sycl::local_accessor sL(sycl::range<1>(R), cgh); + sycl::local_accessor sA(sycl::range<1>(R), cgh); + cgh.parallel_for( + sycl::nd_range<3>(sycl::range<3>(ne03, nkvh, (size_t) nsplit * DEC_WG), sycl::range<3>(1, 1, DEC_WG)), + [=](sycl::nd_item<3> it) [[sycl::reqd_sub_group_size(16)]] { + const int tid = it.get_local_id(2); + const int split = it.get_group(2); + const int kvh = it.get_group(1); + const int seq = it.get_group(0); + auto sg = it.get_sub_group(); + const int w = tid / 16; + const int lane = tid % 16; + auto pQT = sQT.template get_multi_ptr(); + auto pKV = sKV.template get_multi_ptr(); + auto pS = sS.template get_multi_ptr(); + auto pP = sP.template get_multi_ptr(); + + // Q^T (padded rows are zero) + for (int e = tid; e < DEC_D * RP; e += DEC_WG) { + const int d = e / RP, r = e % RP, c = r / G, g = r % G; + float qv = 0.0f; + if (r < R && c < ne01) { + qv = *(const float *) (Q + seq * nb03 + c * nb01 + (int64_t) (kvh * G + g) * nb02 + d * sizeof(float)) * scale; + } + sQT[e] = sycl::half(qv); + } + if (tid < R) { + sM[tid] = DEC_MINF; + sL[tid] = 0.0f; + sA[tid] = 1.0f; + } + + const int k_begin = split * chunk; + const int k_end = sycl::min(k_begin + chunk, ne11); + const char * Kh = K + seq * nb13 + kvh * nb12; + const char * Vh = V + seq * nb23 + kvh * nb22; + const sycl::half * mrow = mask ? (const sycl::half *) (mask + (seq % ne33) * nb33) : nullptr; + + // O accumulators: subgroup w owns head dims 16 w .. 16 w + 15, all MT row tiles + jm::joint_matrix O[MT]; +#pragma unroll + for (int mt = 0; mt < MT; ++mt) { + jm::joint_matrix_fill(sg, O[mt], 0.0f); + } + + // q4_0 rows -> f16 tile in two steps: coalesced dword copy of the raw rows, then an SLM -> SLM + // conversion where each work item writes 8 dims of one key as one 16-byte vector + auto dequant_tile = [&](const char * base, int64_t nb, int t0) { + for (int e = tid; e < TK * KW; e += DEC_WG) { + const int kk = e / KW, wd = e % KW, j = t0 + kk; + sRaw[kk * KWP + wd] = j < k_end ? ((const uint32_t *) (base + j * nb))[wd] : 0u; + } + it.barrier(sycl::access::fence_space::local_space); + for (int e = tid; e < TK * (DEC_D / 8); e += DEC_WG) { + const int kk = e / (DEC_D / 8), g = e % (DEC_D / 8), b = g / 4, part = g % 4; + const int row = kk * KWP; + auto byte_at = [&](int o) -> uint32_t { return (sRaw[row + o / 4] >> (8 * (o % 4))) & 0xFFu; }; + const int ob = b * (int) sizeof(block_q4_0); + const uint32_t sw = sRaw[row + ob / 4] >> (8 * (ob % 4)); // ob even: both scale bytes in one dword + const float dd = static_cast(sycl::bit_cast((uint16_t) (sw & 0xFFFFu))); + const int o0 = ob + 2 + (part % 2) * 8; + const int sh = part < 2 ? 0 : 4; + sycl::vec v; +#pragma unroll + for (int u = 0; u < 8; ++u) { + v[u] = sycl::half((float) ((int) ((byte_at(o0 + u) >> sh) & 0xFu) - 8) * dd); + } + v.store(0, pKV + (kk * DS + g * 8)); + } + }; + + it.barrier(sycl::access::fence_space::local_space); + + for (int t0 = k_begin; t0 < k_end; t0 += TK) { + dequant_tile(Kh, nb11, t0); + it.barrier(sycl::access::fence_space::local_space); + + // S^T tile (keys 8 kt .. +7, rows 16 nt .. +15) per subgroup + if (w < (TK / 8) * (RP / 16)) { + const int kt = w % (TK / 8), nt = w / (TK / 8); + jm::joint_matrix St; + jm::joint_matrix_fill(sg, St, 0.0f); +#pragma unroll 4 + for (int dk = 0; dk < DEC_D; dk += 16) { + jm::joint_matrix Ak; + jm::joint_matrix Bq; + jm::joint_matrix_load(sg, Ak, pKV + (kt * 8) * DS + dk, DS); + jm::joint_matrix_load(sg, Bq, pQT + dk * RP + nt * 16, RP); + jm::joint_matrix_mad(sg, St, Ak, Bq, St); + } + jm::joint_matrix_store(sg, St, pS + (kt * 8) * RP + nt * 16, RP, jm::layout::row_major); + } + it.barrier(sycl::access::fence_space::local_space); + + // online softmax per row (mask, -inf for keys past the slice) -> P [row][key] f16 + for (int r = w; r < R; r += NSG) { + const int c = r / G; + float sv[TK / 16]; + float mt = -INFINITY; +#pragma unroll + for (int i = 0; i < TK / 16; ++i) { + const int k = lane + 16 * i, j = t0 + k; + float v = -INFINITY; + if (j < k_end && c < ne01) { + v = sS[k * RP + r] + (mrow ? static_cast(mrow[(c * nb31) / (int64_t) sizeof(sycl::half) + j]) : 0.0f); + } + sv[i] = v; + mt = sycl::fmax(mt, v); + } + mt = sycl::reduce_over_group(sg, mt, sycl::maximum()); + const float m_old = sM[r]; + const float m_new = sycl::fmax(m_old, mt); + float lt = 0.0f; +#pragma unroll + for (int i = 0; i < TK / 16; ++i) { + const float pv = sycl::native::exp(sv[i] - m_new); + sP[r * PS + lane + 16 * i] = sycl::half(pv); + lt += pv; + } + lt = sycl::reduce_over_group(sg, lt, sycl::plus()); + if (lane == 0) { + const float a = sycl::native::exp(m_old - m_new); + sA[r] = a; + sL[r] = sL[r] * a + lt; + sM[r] = m_new; + } + } + it.barrier(sycl::access::fence_space::local_space); // S reads done, K tile free + + dequant_tile(Vh, nb21, t0); + it.barrier(sycl::access::fence_space::local_space); + + // O = diag(alpha) O + P V for head dims 16 w .. +15 +#pragma unroll + for (int mt = 0; mt < MT; ++mt) { + ijm::joint_matrix_apply(sg, O[mt], [=](float & x, size_t row, size_t) { x *= sA[mt * 8 + row]; }); + } +#pragma unroll + for (int kk = 0; kk < TK; kk += 16) { + jm::joint_matrix Bv; + jm::joint_matrix_load(sg, Bv, pKV + kk * DS + w * 16, DS); +#pragma unroll + for (int mt = 0; mt < MT; ++mt) { + jm::joint_matrix Ap; + jm::joint_matrix_load(sg, Ap, pP + (mt * 8) * PS + kk, PS); + jm::joint_matrix_mad(sg, O[mt], Ap, Bv, O[mt]); + } + } + it.barrier(sycl::access::fence_space::local_space); + } + + // write: rows r = mt * 8 + row -> (token c, head kvh * G + g); dims 16 w + col +#pragma unroll + for (int mt = 0; mt < MT; ++mt) { + ijm::joint_matrix_apply(sg, O[mt], [=](float & x, size_t row, size_t col) { + const int r = mt * 8 + (int) row, c = r / G; + if (c < ne01) { + const int64_t jdu = ((int64_t) seq * ne01 + c) * ne02 + kvh * G + r % G; + const int d = w * 16 + (int) col; + if (nsplit == 1) { + const float l = sL[r]; + dst[jdu * DEC_D + d] = l > 0.0f ? x / l : 0.0f; + } else { + parts[(jdu * nsplit + split) * DEC_D + d] = x; + } + } + }); + } + if (nsplit > 1 && tid < R && tid / G < ne01) { + const int r = tid; + const int64_t jdu = ((int64_t) seq * ne01 + r / G) * ne02 + kvh * G + r % G; + meta[jdu * nsplit + split] = sycl::float2(sM[r], sL[r]); + } + }); + }); +} + + +// ARC-LAB DPAS kernel (GGML_SYCL_FA_DEC_DPAS=1): the q4_0 cache feeds the XMX units without any dequantization pass. +// Q.K: s8 x s4 DPAS (inline vISA, xe2-dpas.hpp; was the IGC builtin intel_sub_group_i8_i4_matrix_mad_k32). +// A = Q rows as int8 (one RNE scale per row per 32-dim block, computed once per call), B = the raw 16 bytes of a key's +// q4_0 block XOR 0x88888888 (u4 n+8 -> s4 n), one key per lane. q4_0 byte i holds dims i and i+16, so A's k = 2i+h +// is dim i + 16h: lane l's A short = (Q[l], Q[l + 16]). S += int result * dq[row] * dk[key]. +// P.V: f16 DPAS (k16, inline vISA) with P from SLM (A layout) and V dequantized in registers by the subgroup that owns 16 head dims. +// Query tokens beyond NQ are handled by extra work-groups (grid dim 0 = sequence x token chunk), each reading the KV again. +namespace { +typedef short dp_short8 __attribute__((ext_vector_type(8))); +typedef int dp_int4 __attribute__((ext_vector_type(4))); +typedef int dp_int8 __attribute__((ext_vector_type(8))); +typedef float dp_float8 __attribute__((ext_vector_type(8))); +typedef unsigned dp_uint4 __attribute__((ext_vector_type(4))); +} + +// ST = 1 (GGML_SYCL_FA_DEC_STAGE, D 256 only): each key tile of K, then of V, is first copied into SLM with coalesced loads +// (consecutive work-items read consecutive dwords of a key row) and the lane = key reads come from there (row stride 37 +// dwords: conflict-free). Without it every lane reads its own key straight from memory: 16 cache lines per load +// instruction, ~90-110 GB/s per phase at 48K. The V copy is issued before the softmax and shares its barrier. +// ARC-LAB: launches a kernel lambda with the 256-register file (half the threads per EU; the 3+ token one-wave launch +// already runs only 2 work-groups x 16 subgroups = 32 threads per Xe core, which is what 256-register mode allows) +template struct fattn_dec_grf256 { + F f; + void operator()(sycl::nd_item<3> it) const { f(it); } + auto get(sycl::ext::oneapi::experimental::properties_tag) const { + return sycl::ext::oneapi::experimental::properties{ sycl::ext::oneapi::experimental::sub_group_size<16>, + sycl::ext::intel::experimental::grf_size<256> }; + } +}; + +// BL bit 0 (GGML_SYCL_FA_DEC_B2D, ST = 0 only): the lane = key reads use transposed 2D block loads (xe2dp::ld2d_t32_8x16: +// 16 keys x 8 dwords per message) instead of per-lane dword gathers. Same bytes, same math. Needs 64-byte aligned surfaces. +// B580, 128K, us: on its own SLOWER (spills; 5 tok 1069 -> 1220, 16 tok 2945 -> 3610). Off by default. +// BL bit 1: launch with the 256-register file (GGML_SYCL_FA_DEC_GRF, default 256 = on for 3+ query tokens; 0 = off). Not +// for 1-2 tokens: their one-wave launch puts 4 work-groups on a core, twice what 256-register mode can hold. +// B580, us per layer, 128K 5 / 8 / 16 tokens: 1070 / 1419 / 2840 -> 988-1006 / 1250 / 2450-2507; whole model (arctune x3, +// MTP + n-gram, sd3 head) 48K 105.2 -> 107.7 t/s, 115K 86.6 -> 90.5, outputs identical. +template +static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, const char * mask, float * dst, + float * parts, sycl::float2 * meta, float scale, int ne01, int ne02, int ne11, + int nkvh, int ne03, int64_t nb01, int64_t nb02, int64_t nb03, int64_t nb11, + int64_t nb12, int64_t nb13, int64_t nb21, int64_t nb22, int64_t nb23, int64_t nb31, + int64_t nb33, int ne33, int nsplit, int chunk, dpct::queue_ptr stream) { + constexpr int R = G * NQ; // query rows per KV head and token chunk + constexpr int RT = (R + 7) / 8; // 8-row DPAS tiles + constexpr int RP = RT * 8; + constexpr int TK = 16 * KG; // keys per tile (KG groups of 16, one key per lane) + constexpr int NB = D / QK4_0; // q4_0 blocks per row (8 at D 256, 16 at D 512) + constexpr int NSG = DEC_WG / 16; // 16 subgroups + constexpr int NCH = (RT + TC - 1) / TC; // row-tile chunks per key group in the Q.K phase + constexpr int DPS = D / NSG; // head dims per subgroup in P.V (16 at D 256, 32 at D 512): all in one block + constexpr int XT = DPS / 8; // 8-dim output tiles per subgroup + constexpr int R16 = (R + 15) / 16 * 16; // P.V computes O^T = V^T P^T: rows are the 16-wide N side + constexpr int RT16 = R16 / 16; + constexpr int PS = TK + 16; // f16 row stride of P ([row][key]); 32-byte aligned rows, fewer bank conflicts + static_assert(D % 256 == 0 && DPS % 16 == 0 && DPS <= QK4_0, "shape"); + constexpr int RW = NB * 18 / 4; // dwords per q4_0 key row (36 at D 256) + constexpr int SW = RW + 1; // SLM row stride (odd: no bank conflicts on lane = key reads) + static_assert(ST == 0 || (D == 256 && RW % 4 == 0), "staging is sized for D 256"); + static_assert((BL & 1) == 0 || (ST == 0 && D == 256), "block loads replace the per-lane reads of the D 256 unstaged path"); + const int nqc = (ne01 + NQ - 1) / NQ; // token chunks + const int probe_env = ggml_sycl_knob("GGML_SYCL_FA_DEC_DPAS_PROBE", 0); + const int probe = probe_env; // phase probe: bit0 skip Q.K, bit1 skip P.V, bit2 skip softmax, bit3 Q.K loads only (no DPAS / scale math) + // ARC-LAB: GGML_SYCL_FA_DEC_PF: prefetch the next phase's key rows while this phase computes (the probes showed Q.K and + // P.V serialised behind the loads): 1 = into L1 (default), 2 = into L3, 0 = off + const int pf = ggml_sycl_knob("GGML_SYCL_FA_DEC_PF", 1); + + stream->submit([&](sycl::handler & cgh) { + sycl::local_accessor sQ8(sycl::range<1>(RT * NB * 16), cgh); // [tile][block][lane] -> 8 rows + sycl::local_accessor sDQ(sycl::range<1>(RT * NB), cgh); // [tile][block] -> 8 rows + sycl::local_accessor sS(sycl::range<1>(RP * TK), cgh); // [row][key] + sycl::local_accessor sP(sycl::range<1>(R16 * PS / 16), cgh); // P f16 [row][key], rows of PS halves + sycl::local_accessor sM(sycl::range<1>(R16), cgh); + sycl::local_accessor sL(sycl::range<1>(R16), cgh); + sycl::local_accessor sA(sycl::range<1>(R16), cgh); + sycl::local_accessor sKV(sycl::range<1>(ST ? TK * SW : 1), cgh); // staged K or V tile + const sycl::nd_range<3> ndr(sycl::range<3>((size_t) ne03 * nqc, nkvh, (size_t) nsplit * DEC_WG), sycl::range<3>(1, 1, DEC_WG)); + auto kern = [=](sycl::nd_item<3> it) [[sycl::reqd_sub_group_size(16)]] { + const int tid = it.get_local_id(2); + const int split = it.get_group(2); + const int kvh = it.get_group(1); + const int seq = it.get_group(0) / nqc; + const int c0 = (it.get_group(0) % nqc) * NQ; // first query token of this chunk + auto sg = it.get_sub_group(); + const int w = tid / 16; + const int lane = tid % 16; + short * sQs = (short *) &sQ8[0]; + short * sPs = (short *) &sP[0]; + + // Q -> int8 per (row, block), RNE; padded rows / tokens past ne01 are zero + for (int e = tid; e < RP * NB; e += DEC_WG) { + const int r = e / NB, b = e % NB, c = c0 + r / G, g = r % G; + float x[QK4_0]; + float amax = 0.0f; + const bool real = r < R && c < ne01; + const float * qr = (const float *) (Q + seq * nb03 + (int64_t) c * nb01 + (int64_t) (kvh * G + g) * nb02) + b * QK4_0; +#pragma unroll + for (int i = 0; i < QK4_0; ++i) { + x[i] = real ? qr[i] * scale : 0.0f; + amax = sycl::fmax(amax, sycl::fabs(x[i])); + } + const float dq = amax / 127.0f; + const float iq = amax > 0.0f ? 127.0f / amax : 0.0f; + const int t = r / 8, m = r % 8; + ((float *) &sDQ[0])[(t * NB + b) * 8 + m] = dq; +#pragma unroll + for (int l = 0; l < 16; ++l) { + const int lo = (int) sycl::rint(x[l] * iq), hi = (int) sycl::rint(x[l + 16] * iq); + sQs[((t * NB + b) * 16 + l) * 8 + m] = (short) ((lo & 0xFF) | ((hi & 0xFF) << 8)); + } + } + for (int e = tid; e < R16 * PS; e += DEC_WG) { + sPs[e] = 0; // padded rows keep P = 0 + } + if (tid < R16) { + sM[tid] = DEC_MINF; + sL[tid] = 0.0f; + sA[tid] = 1.0f; + } + it.barrier(sycl::access::fence_space::local_space); + + const int k_begin = split * chunk; + const int k_end = sycl::min(k_begin + chunk, ne11); + const char * Kh = K + seq * nb13 + kvh * nb12; + const char * Vh = V + seq * nb23 + kvh * nb22; + const sycl::half * mrow = mask ? (const sycl::half *) (mask + (seq % ne33) * nb33) : nullptr; + // 2D surfaces (BL): rows = keys (pitch nb11 / nb21). Heads interleaved within a token row (the real cache + // layout, nb12 < nb11): surface = the sequence's rows, this head at dword x0; else the head's own rows. + const bool kil = nb12 < nb11, vil = nb22 < nb21; + const long long kbs = (long long) (K + seq * nb13 + (kil ? 0 : kvh * nb12)); + const long long vbs = (long long) (V + seq * nb23 + (vil ? 0 : kvh * nb22)); + const int kx0 = kil ? (int) (kvh * nb12 / 4) : 0, vx0 = vil ? (int) (kvh * nb22 / 4) : 0; + + // O^T accumulators: subgroup w owns head dims DPS w .. +DPS-1 (XT 8-dim tiles), lane = query row + dp_float8 o[RT16][XT]; +#pragma unroll + for (int t = 0; t < RT16; ++t) { +#pragma unroll + for (int x = 0; x < XT; ++x) { + o[t][x] = 0.0f; + } + } + // those dims lie in V block (DPS w) / 32; dim d of the block = nibble (d / 16) of qs byte d % 16 + const int vb = (DPS * w) / QK4_0, vd0 = (DPS * w) % QK4_0; + + // coalesced copy of the tile's key rows (keys past k_end repeat key k_begin, as the direct path) + // 16-byte loads (a 144-byte row = 9, rows 16-byte aligned), all issued before any SLM store so they overlap + auto stage = [&](const char * base, int64_t nb, int t0) { + constexpr int RV = RW / 4, NV = TK * RV, PER = (NV + DEC_WG - 1) / DEC_WG; + sycl::uint4 v[PER]; +#pragma unroll + for (int i = 0; i < PER; ++i) { + const int e = tid + i * DEC_WG; + if (e < NV) { + const int key = e / RV, q = e - key * RV; + const int j = t0 + key < k_end ? t0 + key : k_begin; + v[i] = ((const sycl::uint4 *) (base + (int64_t) j * nb))[q]; + } + } +#pragma unroll + for (int i = 0; i < PER; ++i) { + const int e = tid + i * DEC_WG; + if (e < NV) { + const int key = e / RV, q = e - key * RV, o = key * SW + 4 * q; + sKV[o] = v[i][0]; sKV[o + 1] = v[i][1]; sKV[o + 2] = v[i][2]; sKV[o + 3] = v[i][3]; + } + } + }; + + // a 144-byte q4_0 row (D 256) spans at most 3 cache lines from its start + 0 / 64 / 128 + auto prefetch_rows = [&](const char * base, int64_t nb, int t0n) { + if (pf == 0 || t0n >= k_end) { + return; + } + for (int e = tid; e < TK * 3; e += DEC_WG) { + const int j = t0n + e / 3; + if (j < k_end) { + void * pa = (void *) (base + (int64_t) j * nb + (e % 3) * 64); + if (pf == 2) { + sycl::ext::oneapi::experimental::prefetch(pa, sycl::ext::oneapi::experimental::properties{ sycl::ext::oneapi::experimental::prefetch_hint_L3 }); + } else { + sycl::ext::oneapi::experimental::prefetch(pa, sycl::ext::oneapi::experimental::properties{ sycl::ext::oneapi::experimental::prefetch_hint_L1 }); + } + } + } + }; + + for (int t0 = k_begin; t0 < k_end; t0 += TK) { + if constexpr (ST) { + stage(Kh, nb11, t0); + it.barrier(sycl::access::fence_space::local_space); + } + if constexpr (D == 256) { + prefetch_rows(Vh, nb21, t0); // V of this tile is read right after Q.K + } + // ---- S = Q K^T: job = (key group, chunk of TC row tiles) + for (int job = w; job < ((probe & 1) ? 0 : KG * NCH); job += NSG) { + const int kg = job % KG, ch = job / KG; + const int j = t0 + kg * 16 + lane; + const bool jv = j < k_end; + const uint32_t * kr = (const uint32_t *) (Kh + (int64_t) (jv ? j : k_begin) * nb11); + // BL: the 16 keys' 36-dword rows (+4 unused) in 5 loads of 8 dwords, lane = key; issued two blocks + // ahead of their first use (all 5 up front = 40 more registers: spills) + xe2dp::u32x8 kb[(BL & 1) ? 5 : 1]; + auto kload = [&](int i) { + kb[i] = xe2dp::ld2d_t32_8x16(kbs, (int) nb11 - 1, ne11 - 1, (int) nb11 - 1, kx0 + 8 * i, t0 + kg * 16); + }; + if constexpr (BL & 1) { + kload(0); + kload(1); + } + dp_float8 s[TC]; +#pragma unroll + for (int tt = 0; tt < TC; ++tt) { + s[tt] = 0.0f; + } +#pragma unroll + for (int b = 0; b < NB; ++b) { + // block b = bytes 18 b .. 18 b + 17: fp16 scale, then 16 bytes of nibbles + const int o0 = 18 * b; // even b: dword aligned; odd b: 2 bytes in + const int wd = o0 / 4; // the block's 18 bytes lie in these 5 dwords + uint32_t k0, k1, k2, k3, k4; + if constexpr (BL & 1) { + if (b == 1 || b == 3 || b == 5) { // block b + 2 starts in dword chunk (b + 1) / 2 + 1 + kload((b + 1) / 2 + 1); + } + } + if constexpr (ST) { + const int sb = (kg * 16 + lane) * SW + wd; + k0 = sKV[sb]; k1 = sKV[sb + 1]; k2 = sKV[sb + 2]; k3 = sKV[sb + 3]; k4 = sKV[sb + 4]; + } else if constexpr (BL & 1) { + k0 = kb[wd / 8][wd % 8]; k1 = kb[(wd + 1) / 8][(wd + 1) % 8]; k2 = kb[(wd + 2) / 8][(wd + 2) % 8]; + k3 = kb[(wd + 3) / 8][(wd + 3) % 8]; k4 = kb[(wd + 4) / 8][(wd + 4) % 8]; + } else { + k0 = kr[wd]; k1 = kr[wd + 1]; k2 = kr[wd + 2]; k3 = kr[wd + 3]; k4 = kr[wd + 4]; + } + uint32_t sc, q0, q1, q2, q3; + if ((o0 & 3) == 0) { + sc = k0 & 0xFFFFu; + q0 = (k0 >> 16) | (k1 << 16); + q1 = (k1 >> 16) | (k2 << 16); + q2 = (k2 >> 16) | (k3 << 16); + q3 = (k3 >> 16) | (k4 << 16); + } else { + sc = k0 >> 16; // scale in the high half of k0 + q0 = k1; q1 = k2; q2 = k3; q3 = k4; + } + const float dk = static_cast(sycl::bit_cast((uint16_t) sc)); + const dp_int4 bv = { (int) (q0 ^ 0x88888888u), (int) (q1 ^ 0x88888888u), + (int) (q2 ^ 0x88888888u), (int) (q3 ^ 0x88888888u) }; + if (probe & 8) { // keep the loads live, skip the math + s[0][0] += (float) (int) ((q0 ^ q1 ^ q2 ^ q3 ^ sc) & 1u); + continue; + } +#pragma unroll + for (int tt = 0; tt < TC; ++tt) { + const int t = ch * TC + tt; + if (t < RT) { + const dp_int8 ia = xe2dp::dpas_s4s8_r8(sQ8[(t * NB + b) * 16 + lane], bv); + const dp_float8 sc8 = sDQ[t * NB + b] * dk; // one broadcast vector load per tile and block + s[tt] += __builtin_convertvector(ia, dp_float8) * sc8; + } + } + } +#pragma unroll + for (int tt = 0; tt < TC; ++tt) { + const int t = ch * TC + tt; + if (t < RT) { +#pragma unroll + for (int m = 0; m < 8; ++m) { + const int r = t * 8 + m, c = c0 + r / G; + if (r < R) { + const bool ok = jv && c < ne01; + const float mv = (ok && mrow) ? static_cast(mrow[((int64_t) c * nb31) / (int64_t) sizeof(sycl::half) + j]) : 0.0f; + sS[r * TK + kg * 16 + lane] = ok ? s[tt][m] + mv : -INFINITY; + } + } + } + } + } + it.barrier(sycl::access::fence_space::local_space); + if constexpr (ST) { + stage(Vh, nb21, t0); // K is done; the softmax below does not touch sKV + } + + // ---- online softmax per row -> P (f16, DPAS A layout) + for (int r = w; r < ((probe & 4) ? 0 : R); r += NSG) { + float sv[KG]; + float mt = -INFINITY; +#pragma unroll + for (int i = 0; i < KG; ++i) { + sv[i] = sS[r * TK + i * 16 + lane]; + mt = sycl::fmax(mt, sv[i]); + } + mt = sycl::reduce_over_group(sg, mt, sycl::maximum()); + const float m_old = sM[r]; + const float m_new = sycl::fmax(m_old, mt); + float lt = 0.0f; +#pragma unroll + for (int i = 0; i < KG; ++i) { + const float pv = sycl::native::exp(sv[i] - m_new); + sPs[r * PS + i * 16 + lane] = (short) sycl::bit_cast(sycl::half(pv)); + lt += pv; + } + lt = sycl::reduce_over_group(sg, lt, sycl::plus()); + if (lane == 0) { + const float a = sycl::native::exp(m_old - m_new); + sA[r] = a; + sL[r] = sL[r] * a + lt; + sM[r] = m_new; + } + } + it.barrier(sycl::access::fence_space::local_space); + + if constexpr (D == 256) { + prefetch_rows(Kh, nb11, t0 + TK); // next tile's K, read after this P.V + } + // ---- O^T = diag(alpha) O^T + V^T P^T: A = V^T (8 dims x 16 keys, lane = key), B = P^T (16 keys x 16 rows) +#pragma unroll + for (int t = 0; t < RT16; ++t) { + const float a = sA[t * 16 + lane]; +#pragma unroll + for (int x = 0; x < XT; ++x) { + o[t][x] *= a; + } + } + // BL: this subgroup's V block of the next 16 keys is loaded one key group ahead + xe2dp::u32x8 vnext; + if constexpr (BL & 1) { + if (!(probe & 2)) { + vnext = xe2dp::ld2d_t32_8x16(vbs, (int) nb21 - 1, ne11 - 1, (int) nb21 - 1, vx0 + 18 * vb / 4, t0); + } + } +#pragma unroll 1 + for (int kg = 0; kg < ((probe & 2) ? 0 : KG); ++kg) { + const int j = t0 + kg * 16 + lane; + const bool jv = j < k_end; + const uint32_t * vr = (const uint32_t *) (Vh + (int64_t) (jv ? j : k_begin) * nb21); + const int o0 = 18 * vb, wd = o0 / 4; // block start; even block: dword aligned, odd: 2 bytes in + uint32_t w0, w1, w2, w3, w4; + if constexpr (BL & 1) { + const xe2dp::u32x8 vc = vnext; + if (kg + 1 < KG) { + vnext = xe2dp::ld2d_t32_8x16(vbs, (int) nb21 - 1, ne11 - 1, (int) nb21 - 1, vx0 + wd, t0 + (kg + 1) * 16); + } + w0 = vc[0]; w1 = vc[1]; w2 = vc[2]; w3 = vc[3]; w4 = vc[4]; + } else if constexpr (ST) { + const int sb = (kg * 16 + lane) * SW + wd; + w0 = sKV[sb]; w1 = sKV[sb + 1]; w2 = sKV[sb + 2]; w3 = sKV[sb + 3]; w4 = sKV[sb + 4]; + } else { + w0 = vr[wd]; w1 = vr[wd + 1]; w2 = vr[wd + 2]; w3 = vr[wd + 3]; w4 = vr[wd + 4]; + } + uint32_t sc, q[4]; + if ((o0 & 3) == 0) { + sc = w0 & 0xFFFFu; + q[0] = (w0 >> 16) | (w1 << 16); q[1] = (w1 >> 16) | (w2 << 16); + q[2] = (w2 >> 16) | (w3 << 16); q[3] = (w3 >> 16) | (w4 << 16); + } else { + sc = w0 >> 16; + q[0] = w1; q[1] = w2; q[2] = w3; q[3] = w4; + } + const float dv = jv ? static_cast(sycl::bit_cast((uint16_t) sc)) : 0.0f; + dp_short8 va[XT]; +#pragma unroll + for (int x = 0; x < XT; ++x) { +#pragma unroll + for (int m = 0; m < 8; ++m) { + const int d = vd0 + x * 8 + m; // dim within block vb + const int byte = d % 16, sh = (d / 16) * 4; + const int nib = (q[byte / 4] >> (8 * (byte % 4) + sh)) & 0xF; + va[x][m] = (short) sycl::bit_cast(sycl::half((float) (nib - 8) * dv)); + } + } +#pragma unroll + for (int t = 0; t < RT16; ++t) { + const dp_int8 pb = sP[((t * 16 + lane) * PS + kg * 16) / 16]; +#pragma unroll + for (int x = 0; x < XT; ++x) { + o[t][x] = xe2dp::dpas_hf_r8(va[x], pb, o[t][x]); + } + } + } + it.barrier(sycl::access::fence_space::local_space); + } + + // write: lane = row r = 16 t + lane -> (token c, head kvh * G + g); dims 16 w + 8 x + m +#pragma unroll + for (int t = 0; t < RT16; ++t) { + const int r = t * 16 + lane, c = c0 + r / G; + if (r < R && c < ne01) { + const int64_t jdu = ((int64_t) seq * ne01 + c) * ne02 + kvh * G + r % G; + const float l = sL[r]; +#pragma unroll + for (int x = 0; x < XT; ++x) { +#pragma unroll + for (int m = 0; m < 8; ++m) { + const int d = w * DPS + x * 8 + m; + if (nsplit == 1) { + dst[jdu * D + d] = l > 0.0f ? o[t][x][m] / l : 0.0f; + } else { + parts[(jdu * nsplit + split) * D + d] = o[t][x][m]; + } + } + } + } + } + if (nsplit > 1 && tid < R && c0 + tid / G < ne01) { + const int64_t jdu = ((int64_t) seq * ne01 + c0 + tid / G) * ne02 + kvh * G + tid % G; + meta[jdu * nsplit + split] = sycl::float2(sM[tid], sL[tid]); + } + }; + if constexpr ((BL & 2) != 0) { + cgh.parallel_for(ndr, fattn_dec_grf256{ kern }); + } else { + cgh.parallel_for(ndr, kern); + } + }); +} + +static int fattn_dec_dpas_maxq() { // 0 = DPAS kernel off; else it serves 1 .. maxq query tokens + static const bool ok = [] { + // the kernel needs Xe2's 16-lane int8 x int4 DPAS; elsewhere fall back instead of failing at the first launch + if (getenv("GGML_SYCL_FA_DEC_DPAS") && atoi(getenv("GGML_SYCL_FA_DEC_DPAS")) && !getenv("GGML_SYCL_FA_DEC_DPAS_ANYGPU") && + !ggml_sycl_device_is_xe2()) { + GGML_LOG_WARN("%s: GGML_SYCL_FA_DEC_DPAS needs an Xe2 or newer GPU (Arc B-series, Lunar Lake, Panther Lake); " + "%s is not one, using the regular decode attention\n", __func__, + ggml_sycl_info().devices[ggml_sycl_get_device()].hw_info.name.c_str()); + return false; + } + return getenv("GGML_SYCL_FA_DEC_DPAS_ANYGPU") != nullptr || ggml_sycl_device_is_xe2(); + }(); + // on by default on Xe3: Arc B390 q4_0 KV decode at 16K +4-6%, KLD vs the regular kernel within run-to-run error + const int v = + ok ? ggml_sycl_knob("GGML_SYCL_FA_DEC_DPAS", ggml_sycl_device_is_xe3(ggml_sycl_get_device()) ? 1 : 0) : 0; + // =1 -> default cap 256: long n-gram verify batches stay on this kernel instead of oneDNN's f16 copy of the whole KV + // cache (B580, per layer: 48K 64 tok 28.8 -> 4.9 ms, 115K 256 tok 253 -> 44 ms; KLD 0.01270 vs 0.01280, same within + // error; whole model MTP 48K 73.2 -> 76.1 t/s, 115K 60.3 -> 65.1). =32 restores the old cap. + return v == 1 ? 256 : v; +} + +bool ggml_sycl_flash_attn_ext_dec_supported(const ggml_tensor * dst) { + static const bool off = getenv("GGML_SYCL_FA_DEC_OFF") != nullptr; + if (off) { + return false; + } + const int maxq = fattn_dec_dpas_maxq(); + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; + const ggml_tensor * sinks = dst->src[4]; + float max_bias = 0.0f, softcap = 0.0f; + memcpy(&max_bias, (const float *) dst->op_params + 1, sizeof(float)); + memcpy(&softcap, (const float *) dst->op_params + 2, sizeof(float)); + return K->type == GGML_TYPE_Q4_0 && V->type == GGML_TYPE_Q4_0 && Q->type == GGML_TYPE_F32 && + // Bonsai 2 27B (head 256, 6 query heads per KV head); with the DPAS kernel also Gemma 4's global layers (512, 16) + ((K->ne[0] == DEC_D && Q->ne[2] == 6 * K->ne[2]) || (maxq && K->ne[0] == 512 && Q->ne[2] == 16 * K->ne[2]) || + (maxq && K->ne[0] == DEC_D && Q->ne[2] == 2 * K->ne[2])) && // Gemma 4 sliding layers (256, 2) + V->ne[0] == K->ne[0] && Q->ne[0] == K->ne[0] && V->ne[2] == K->ne[2] && + Q->ne[1] >= 1 && Q->ne[1] <= (maxq ? maxq : 4) && Q->ne[3] == K->ne[3] && V->ne[3] == K->ne[3] && // 5-8: TILE is faster (v3<6,8> 1172-1240 vs 1049 us @16K) + !sinks && max_bias == 0.0f && softcap == 0.0f && + (!mask || (mask->type == GGML_TYPE_F16 && mask->ne[2] == 1)) && + Q->nb[0] == sizeof(float) && K->nb[0] == ggml_type_size(K->type) && V->nb[0] == ggml_type_size(V->type) && + K->nb[1] % 4 == 0 && K->nb[2] % 4 == 0 && K->nb[3] % 4 == 0 && ((uintptr_t) K->data) % 4 == 0 && + V->nb[1] % 4 == 0 && V->nb[2] % 4 == 0 && V->nb[3] % 4 == 0 && ((uintptr_t) V->data) % 4 == 0 && + Q->nb[1] % 16 == 0 && Q->nb[2] % 16 == 0 && Q->nb[3] % 16 == 0 && ((uintptr_t) Q->data) % 16 == 0 && + (!maxq || (K->nb[1] % 16 == 0 && K->nb[2] % 16 == 0 && K->nb[3] % 16 == 0 && ((uintptr_t) K->data) % 16 == 0)); +} + +void ggml_sycl_flash_attn_ext_dec(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; + + float scale = 1.0f; + memcpy(&scale, (const float *) dst->op_params + 0, sizeof(float)); + + const int ne01 = Q->ne[1]; + const bool dpas = fattn_dec_dpas_maxq() > 0; + const bool d512 = K->ne[0] == 512; // Gemma 4 global layers: DPAS kernel only, token chunks of 2 (16 heads x 2 rows) + const bool g2 = !d512 && Q->ne[2] == 2 * K->ne[2]; // Gemma 4 sliding layers: DPAS kernel only, token chunks of 8 + const int nq = d512 ? (ne01 <= 1 ? 1 : 2) : g2 ? (ne01 <= 1 ? 1 : 8) : ne01 <= 1 ? 1 : ne01 <= 2 ? 2 : ne01 <= 4 ? 4 : 8; + // keys per tile (must match the instantiations below); DPAS: 16 x KG + // SLM-staged K/V tiles (128 keys) for single-token decode: 48K 432 -> 247 us with the split count below. For 2+ tokens + // the extra SLM halves the work-groups per core and it loses (4 tok 48K 458 unstaged vs ~593 staged). GGML_SYCL_FA_DEC_STAGE=0 off. + const bool stage_env = ggml_sycl_knob("GGML_SYCL_FA_DEC_STAGE", 1) != 0; + // ARC-LAB: 5-token query chunks (default, GGML_SYCL_FA_DEC_NQ5=1): MTP 4 drafts + 1 = 5-token verifies otherwise run the + // 8-token kernel with 3 empty tokens (48 rows of Q.K / softmax for 30). B580 per layer: 16K 215 -> 175 us, 48K 637 -> + // 502 us. =0 off, =2 SLM-staged (slower: 48K 579), =3 256-key tiles (48K 513). + const int nq5_env = ggml_sycl_knob("GGML_SYCL_FA_DEC_NQ5", 1); + // ARC-LAB: GGML_SYCL_FA_DEC_B2D: 1 = transposed 2D block loads for the unstaged DPAS paths (2+ query tokens), 2 = also + // for 1 token instead of the SLM staging; 0 = per-lane reads. Needs 64-byte aligned K / V surfaces, else per-lane reads. + const int b2d_env = ggml_sycl_knob("GGML_SYCL_FA_DEC_B2D", 0); + auto b2d_fits = [](const ggml_tensor * t) { + const bool il = t->nb[2] < t->nb[1]; // heads interleaved in a token row + return ((uintptr_t) t->data) % 64 == 0 && (t->ne[3] == 1 || t->nb[3] % 64 == 0) && t->nb[1] % 16 == 0 && + t->nb[1] >= 64 && t->nb[1] < (1 << 24) && t->ne[1] < (1 << 24) && + (il ? (t->nb[2] % 4 == 0 && t->nb[2] * (t->ne[2] - 1) + (int64_t) (t->ne[0] / QK4_0) * 18 <= (int64_t) t->nb[1]) + : t->nb[2] % 64 == 0); + }; + const bool b2d_shape = dpas && b2d_env && K->ne[0] == DEC_D && Q->ne[2] == 6 * K->ne[2]; + const bool b2d = b2d_shape && b2d_fits(K) && b2d_fits(V); + if (b2d_shape) { + static bool logged_on = false, logged_off = false; + bool & logged = b2d ? logged_on : logged_off; + if (!logged) { + logged = true; + GGML_LOG_INFO("%s: GGML_SYCL_FA_DEC_B2D=%d %s (K data %% 64 = %d, nb1 %zu nb2 %zu)\n", __func__, b2d_env, + b2d ? "engaged" : "NOT engaged: surface not 64-byte aligned, per-lane reads", + (int) (((uintptr_t) K->data) % 64), K->nb[1], K->nb[2]); + } + } + const int bl = (b2d ? 1 : 0) | ((ggml_sycl_knob("GGML_SYCL_FA_DEC_GRF", 256) == 256 && nq >= 4) ? 2 : 0); + const int nq5 = (dpas && !d512 && !g2 && ne01 == 5) ? nq5_env : 0; + const bool stage = dpas && stage_env && !d512 && !g2 && (nq == 1 || nq5 == 2) && !(b2d && b2d_env >= 2 && nq == 1); + const int tk = (d512 || g2) ? (nq == 1 ? 256 : 128) + : nq5 == 3 ? 256 + : (stage || nq5) ? 128 + : dpas ? (nq == 1 ? 256 : nq == 2 ? 128 : nq == 4 ? 256 : 128) : nq == 8 ? 32 : nq == 4 ? 64 : 128; + const int ne11 = K->ne[1]; + + // slices: enough work-groups to fill the GPU (~64 slices per KV head, ~256 in all), at least one tile each + const int target_env = ggml_sycl_knob("GGML_SYCL_FA_DEC_SPLITS", 0); + // DPAS (D 256, 6 rows): one wave of work-groups - Xe cores x the work-groups that fit per core (SLM-bound: 4 for 1-2 + // query tokens, 2 for 4-8) - over the KV heads. A fixed 64 per head ran 3-6 waves with a mostly idle last one: B580 + // 48K 1 tok 432 -> 302 us (247 staged), 4 tok 648 -> 458 us. Xe2: 8 EUs per core, nsm = EUs / 16. + const int xe_cores = std::max(1, ggml_sycl_info().devices[ctx.device].nsm * 2); + const int wave = xe_cores * (nq <= 2 ? 4 : 2); + const int target = target_env ? target_env + : (dpas && !d512 && !g2) ? std::max(1, wave / std::max(1, (int) K->ne[2])) + : std::max(64, 256 / std::max(1, (int) K->ne[2])); + const int ntiles = (ne11 + tk - 1) / tk; + int nsplit = std::max(1, std::min(target, ntiles)); + const int chunk = ((ntiles + nsplit - 1) / nsplit) * tk; + nsplit = (ne11 + chunk - 1) / chunk; + + ggml_sycl_pool_alloc parts(ctx.pool()); + ggml_sycl_pool_alloc meta(ctx.pool()); + if (nsplit > 1) { + parts.alloc((size_t) nsplit * ggml_nelements(dst)); + meta.alloc((size_t) nsplit * ggml_nrows(dst)); + } + + dpct::queue_ptr stream = ctx.stream(); + const char * mdata = mask ? (const char *) mask->data : nullptr; + const int64_t nb31 = mask ? mask->nb[1] : 0; + const int64_t nb33 = mask ? mask->nb[3] : 0; + const int ne33 = mask ? (int) mask->ne[3] : 1; + +#define FATTN_DEC_DPAS_BL(NQ_, KG_, TC_, BL_) fattn_dec_q4_0_dpas((const char *) Q->data, \ + (const char *) K->data, (const char *) V->data, mdata, (float *) dst->data, parts.get(), meta.get(), scale, ne01, \ + (int) Q->ne[2], ne11, (int) K->ne[2], (int) Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], K->nb[1], K->nb[2], K->nb[3], \ + V->nb[1], V->nb[2], V->nb[3], nb31, nb33, ne33, nsplit, chunk, stream) +#define FATTN_DEC_DPAS(NQ_, KG_, TC_) do { switch (bl) { \ + case 3: FATTN_DEC_DPAS_BL(NQ_, KG_, TC_, 3); break; case 2: FATTN_DEC_DPAS_BL(NQ_, KG_, TC_, 2); break; \ + case 1: FATTN_DEC_DPAS_BL(NQ_, KG_, TC_, 1); break; default: FATTN_DEC_DPAS_BL(NQ_, KG_, TC_, 0); break; } } while (0) + if (d512) { +#define FATTN_DEC_DPAS512(NQ_, KG_, TC_) fattn_dec_q4_0_dpas<512, 16, NQ_, KG_, TC_>((const char *) Q->data, (const char *) K->data, \ + (const char *) V->data, mdata, (float *) dst->data, parts.get(), meta.get(), scale, ne01, (int) Q->ne[2], ne11, \ + (int) K->ne[2], (int) Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], K->nb[1], K->nb[2], K->nb[3], V->nb[1], V->nb[2], \ + V->nb[3], nb31, nb33, ne33, nsplit, chunk, stream) + if (nq == 1) { + FATTN_DEC_DPAS512(1, 16, 2); // 16 rows = 2 tiles, one chunk of 2 per key group + } else { + FATTN_DEC_DPAS512(2, 8, 2); // 32 rows = 4 tiles, 2 chunks x 8 key groups + } +#undef FATTN_DEC_DPAS512 + } else if (g2) { +#define FATTN_DEC_DPASG2(NQ_, KG_, TC_) fattn_dec_q4_0_dpas((const char *) Q->data, (const char *) K->data, \ + (const char *) V->data, mdata, (float *) dst->data, parts.get(), meta.get(), scale, ne01, (int) Q->ne[2], ne11, \ + (int) K->ne[2], (int) Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], K->nb[1], K->nb[2], K->nb[3], V->nb[1], V->nb[2], \ + V->nb[3], nb31, nb33, ne33, nsplit, chunk, stream) + if (nq == 1) { + FATTN_DEC_DPASG2(1, 16, 1); // 2 rows (1 tile) + } else { + FATTN_DEC_DPASG2(8, 8, 1); // 16 rows = 2 tiles, 8 key groups x 2 + } +#undef FATTN_DEC_DPASG2 + } else if (nq5 == 1) { + FATTN_DEC_DPAS(5, 8, 2); + } else if (nq5 == 3) { + FATTN_DEC_DPAS(5, 16, 2); // 256-key tiles, as the 4-token kernel + } else if (stage) { +#define FATTN_DEC_DPAS_ST(NQ_, KG_, TC_) fattn_dec_q4_0_dpas((const char *) Q->data, (const char *) K->data, \ + (const char *) V->data, mdata, (float *) dst->data, parts.get(), meta.get(), scale, ne01, (int) Q->ne[2], ne11, \ + (int) K->ne[2], (int) Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], K->nb[1], K->nb[2], K->nb[3], V->nb[1], V->nb[2], \ + V->nb[3], nb31, nb33, ne33, nsplit, chunk, stream) + switch (nq) { + case 1: FATTN_DEC_DPAS_ST(1, 8, 1); break; + case 8: FATTN_DEC_DPAS_ST(5, 8, 2); break; // only with nq5 == 2 (stage is otherwise 1-token only) + case 2: FATTN_DEC_DPAS_ST(2, 8, 1); break; + case 4: FATTN_DEC_DPAS_ST(4, 8, 3); break; + default: FATTN_DEC_DPAS_ST(8, 8, 3); break; + } +#undef FATTN_DEC_DPAS_ST + } else if (dpas) { + switch (nq) { + case 1: FATTN_DEC_DPAS(1, 16, 1); break; + case 2: FATTN_DEC_DPAS(2, 8, 1); break; + case 4: FATTN_DEC_DPAS(4, 16, 3); break; + default: FATTN_DEC_DPAS(8, 8, 3); break; // 8-token chunks, one work-group row per chunk + } + } +#undef FATTN_DEC_DPAS +#undef FATTN_DEC_DPAS_BL +#define FATTN_DEC_CALL(NQ_) fattn_dec_q4_0<6, NQ_, (NQ_ == 4 ? 64 : 128), (NQ_ != 4)>( /* PIPE: see v4 notes */(const char *) Q->data, (const char *) K->data, (const char *) V->data, \ + mdata, (float *) dst->data, parts.get(), meta.get(), scale, ne01, (int) Q->ne[2], ne11, (int) K->ne[2], \ + (int) Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], K->nb[1], K->nb[2], K->nb[3], V->nb[1], V->nb[2], V->nb[3], nb31, \ + nb33, ne33, nsplit, chunk, stream) + if (!dpas && !d512 && !g2) switch (nq) { + case 1: FATTN_DEC_CALL(1); break; + case 2: FATTN_DEC_CALL(2); break; + case 8: // 5-8 tokens (MTP depth 4+, short n-gram drafts): v3 kernel, 48 rows, 32-key tiles, ~59 KB SLM + fattn_dec_q4_0_v3<6, 8>((const char *) Q->data, (const char *) K->data, (const char *) V->data, mdata, + (float *) dst->data, parts.get(), meta.get(), scale, ne01, (int) Q->ne[2], ne11, (int) K->ne[2], + (int) Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], K->nb[1], K->nb[2], K->nb[3], V->nb[1], V->nb[2], V->nb[3], + nb31, nb33, ne33, nsplit, chunk, stream); + break; + default: + if (static const int xmx = getenv("GGML_SYCL_FA_DEC_XMX") ? atoi(getenv("GGML_SYCL_FA_DEC_XMX")) : 0; xmx) { // ARC-LAB opt-in +#define FATTN_DEC_XMX(TK_) fattn_dec_q4_0_xmx<6, 4, TK_>((const char *) Q->data, (const char *) K->data, (const char *) V->data, mdata, \ + (float *) dst->data, parts.get(), meta.get(), scale, ne01, (int) Q->ne[2], ne11, (int) K->ne[2], \ + (int) Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], K->nb[1], K->nb[2], K->nb[3], V->nb[1], V->nb[2], V->nb[3], \ + nb31, nb33, ne33, nsplit, chunk, stream) + if (xmx == 32) { + FATTN_DEC_XMX(32); + } else { + FATTN_DEC_XMX(64); + } +#undef FATTN_DEC_XMX + break; + } + fattn_dec_q4_0_v3<6, 4>((const char *) Q->data, (const char *) K->data, (const char *) V->data, mdata, + (float *) dst->data, parts.get(), meta.get(), scale, ne01, (int) Q->ne[2], ne11, (int) K->ne[2], + (int) Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], K->nb[1], K->nb[2], K->nb[3], V->nb[1], V->nb[2], V->nb[3], + nb31, nb33, ne33, nsplit, chunk, stream); + break; + } +#undef FATTN_DEC_CALL + + if (nsplit > 1) { + auto combine = [&](auto dk) { + constexpr int D = decltype(dk)::value; + const sycl::range<3> grid(Q->ne[3], Q->ne[2], (size_t) ne01 * D); + const size_t nbytes_shared = nsplit * sizeof(sycl::float2); + float * parts_p = parts.get(); + sycl::float2 * meta_p = meta.get(); + float * dst_p = (float *) dst->data; + stream->submit([&](sycl::handler & cgh) { + sycl::local_accessor lm(sycl::range<1>(nbytes_shared), cgh); + cgh.parallel_for(sycl::nd_range<3>(grid, sycl::range<3>(1, 1, D)), + [=](sycl::nd_item<3>) [[sycl::reqd_sub_group_size(16)]] { + flash_attn_combine_results(parts_p, meta_p, dst_p, nsplit, + lm.get_multi_ptr().get()); + }); + }); + }; + if (d512) { + combine(std::integral_constant{}); + } else { + combine(std::integral_constant{}); + } + } +} diff --git a/ggml/src/ggml-sycl/fattn-dec.hpp b/ggml/src/ggml-sycl/fattn-dec.hpp new file mode 100644 index 000000000000..32760be8b001 --- /dev/null +++ b/ggml/src/ggml-sycl/fattn-dec.hpp @@ -0,0 +1,7 @@ +#pragma once + +#include "common.hpp" + +// ARC-LAB decode attention straight from a q4_0 KV cache (see fattn-dec.cpp). +bool ggml_sycl_flash_attn_ext_dec_supported(const ggml_tensor * dst); +void ggml_sycl_flash_attn_ext_dec(ggml_backend_sycl_context & ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-sycl/fattn-mkl.cpp b/ggml/src/ggml-sycl/fattn-mkl.cpp index 2d164a0840fd..d06ed4530c0e 100644 --- a/ggml/src/ggml-sycl/fattn-mkl.cpp +++ b/ggml/src/ggml-sycl/fattn-mkl.cpp @@ -126,79 +126,65 @@ static void mkl_fa_online_softmax_chunk( int64_t mask_row_stride, int mask_n_heads, float logit_softcap, int64_t wg_size) { - const int64_t wg = ((q_rows + wg_size - 1) / wg_size) * wg_size; - + // ARC-LAB: one work-group per query row (was one work-item per row walking the whole chunk: neighbouring + // work-items read addresses chunk_size floats apart, 72% of generation time at 115K context). Same math. + GGML_UNUSED(wg_size); + constexpr int SWG = 256; stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for(sycl::nd_range<1>(wg, wg_size), + cgh.parallel_for(sycl::nd_range<1>((size_t) q_rows * SWG, SWG), [=](sycl::nd_item<1> item) { - int jc_rel = item.get_global_id(0); - if (jc_rel >= q_rows) return; - int jc_abs = q0 + jc_rel; + const int jc_rel = item.get_group(0); + const int tid = item.get_local_id(0); + const int jc_abs = q0 + jc_rel; + auto grp = item.get_group(); const int gqa_group = jc_abs / n_queries; const int q_row = jc_abs % n_queries; - // Score buffers are tile-local (relative index). - const float * __restrict KQ_row = KQ_f32 - + jc_rel * (int64_t)chunk_size; - // Persistent accumulator is full-sized (absolute index). - float * __restrict vkq = VKQ_accum - + jc_abs * (int64_t)DV; + const float * __restrict KQ_row = KQ_f32 + jc_rel * (int64_t)chunk_size; + float * __restrict vkq = VKQ_accum + jc_abs * (int64_t)DV; const sycl::half * mask_h = nullptr; - int64_t m_stride = 0; if (mask_data) { - int m_head = (mask_n_heads > 1) - ? (kvh_head + gqa_group) : 0; - mask_h = mask_data + (int64_t)m_head * mask_head_stride; - m_stride = mask_row_stride; + const int m_head = (mask_n_heads > 1) ? (kvh_head + gqa_group) : 0; + mask_h = mask_data + (int64_t)m_head * mask_head_stride + (int64_t)q_row * mask_row_stride + chunk_start; } - - // Row-wise local maximum (softcap before mask) - float local_max = -1e30f; - for (int i = 0; i < chunk_size; i++) { + auto score = [&](int i) { float s = KQ_row[i]; if (logit_softcap != 0.0f) { s = logit_softcap * sycl::tanh(s); } if (mask_h) { - s += (float)mask_h[q_row * m_stride - + (chunk_start + i)]; + s += (float)mask_h[i]; } - if (s > local_max) local_max = s; - } + return s; + }; - // Rescale previous accumulator by exp(old_max - new_max) - float old_max = KQ_max[jc_abs]; - float new_max = (old_max > local_max) ? old_max : local_max; - float rescale = (old_max < -1e29f) ? 1.0f - : sycl::native::exp(old_max - new_max); + const float old_max = KQ_max[jc_abs]; // read by every item before item 0 updates it below + float local_max = -1e30f; + for (int i = tid; i < chunk_size; i += SWG) { + local_max = sycl::fmax(local_max, score(i)); + } + local_max = sycl::reduce_over_group(grp, local_max, sycl::maximum()); - for (int v = 0; v < DV; v++) { + const float new_max = (old_max > local_max) ? old_max : local_max; + const float rescale = (old_max < -1e29f) ? 1.0f : sycl::native::exp(old_max - new_max); + for (int v = tid; v < DV; v += SWG) { vkq[v] *= rescale; } - // Softmax and write S_f16 (tile-local index) + sycl::half * __restrict S_row = S_f16 + jc_rel * (int64_t)chunk_size; float local_sum = 0.0f; - sycl::half * __restrict S_row = S_f16 - + jc_rel * (int64_t)chunk_size; - - for (int i = 0; i < chunk_size; i++) { - float s = KQ_row[i]; - if (logit_softcap != 0.0f) { - s = logit_softcap * sycl::tanh(s); - } - if (mask_h) { - s += (float)mask_h[q_row * m_stride - + (chunk_start + i)]; - } - float val = sycl::native::exp(s - new_max); + for (int i = tid; i < chunk_size; i += SWG) { + const float val = sycl::native::exp(score(i) - new_max); S_row[i] = sycl::half(val); local_sum += val; } - - KQ_sum[jc_abs] = KQ_sum[jc_abs] * rescale + local_sum; - KQ_max[jc_abs] = new_max; + local_sum = sycl::reduce_over_group(grp, local_sum, sycl::plus()); + if (tid == 0) { + KQ_sum[jc_abs] = KQ_sum[jc_abs] * rescale + local_sum; + KQ_max[jc_abs] = new_max; + } }); }); } diff --git a/ggml/src/ggml-sycl/fattn-onednn.cpp b/ggml/src/ggml-sycl/fattn-onednn.cpp index a501295192fb..caea33f23636 100644 --- a/ggml/src/ggml-sycl/fattn-onednn.cpp +++ b/ggml/src/ggml-sycl/fattn-onednn.cpp @@ -27,6 +27,15 @@ bool ggml_sycl_flash_attn_ext_onednn_supported(const ggml_tensor * dst) { const ggml_tensor * mask = dst->src[3]; const ggml_tensor * sinks = dst->src[4]; + // ARC-LAB: heads interleaved per token (llama.cpp's KV-cache view, nb[2] < nb[1]) must be packed: the SDPA / dequant + // paths derive the token stride from the head count. A gap between tokens (test-backend-ops kv_view + permute) read + // the wrong rows (ERR ~1.2); the real cache is always packed. See ggml-org#27769. + for (const ggml_tensor * t : {K, V}) { + if (t->nb[2] < t->nb[1] && t->nb[1] != t->nb[2] * t->ne[2]) { + return false; + } + } + // F16 KV: native SDPA at any KV length. // Non-F16: dequant to F16 then SDPA at prefill lengths. Only the // standard quantized KV cache types (Q4_0-Q8_0) and F32 are accepted diff --git a/ggml/src/ggml-sycl/fattn.cpp b/ggml/src/ggml-sycl/fattn.cpp index a85eb721f6af..b52e8a505975 100644 --- a/ggml/src/ggml-sycl/fattn.cpp +++ b/ggml/src/ggml-sycl/fattn.cpp @@ -19,6 +19,7 @@ #include "fattn-vec.hpp" #include "fattn.hpp" #include "fattn-onednn.hpp" +#include "fattn-dec.hpp" #define FATTN_VEC_CASE(D, type_K, type_V) \ @@ -98,6 +99,7 @@ enum best_fattn_kernel { BEST_FATTN_KERNEL_NONE = 0, BEST_FATTN_KERNEL_VEC = 100, BEST_FATTN_KERNEL_ONEDNN = 150, // oneDNN SDPA: native F16 (PR #25222) + BEST_FATTN_KERNEL_DEC = 160, // ARC-LAB: q4_0 KV decode, 1-4 query tokens (fattn-dec.cpp) BEST_FATTN_KERNEL_TILE = 200, BEST_FATTN_KERNEL_MKL = 300, }; @@ -112,6 +114,11 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const if(!g_ggml_sycl_enable_flash_attention) return BEST_FATTN_KERNEL_NONE; + // ARC-LAB: generation / MTP-verify batches on a q4_0 cache read it directly (TILE dequantizes the whole cache per call) + if (ggml_sycl_flash_attn_ext_dec_supported(dst)) { + return BEST_FATTN_KERNEL_DEC; + } + const ggml_tensor * KQV = dst; const ggml_tensor * Q = dst->src[0]; const ggml_tensor * K = dst->src[1]; @@ -165,6 +172,11 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const // (nb1 == H * ne0*2). Only pathological test strides like nb1=32 or // nb1=75 for ne0=40 fall through to TILE. bool kv_strides_ok = true; + for (const ggml_tensor * t : {K, V}) { // ARC-LAB: interleaved heads must be packed (see fattn-onednn.cpp) + if (t->nb[2] < t->nb[1] && t->nb[1] != t->nb[2] * t->ne[2]) { + kv_strides_ok = false; + } + } for (const ggml_tensor * t : {K, V}) { if (t->type == GGML_TYPE_F16 && t->nb[1] % (t->ne[0] * 2) != 0) { kv_strides_ok = false; @@ -262,7 +274,10 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const } } } else { - if (Q->ne[1] <= 2) { + // like the CUDA backend: with GQA packing available, the tile kernel reads each quantized K/V head once + // for all its query heads; the vec kernel re-reads it per query head (gqa_ratio x the KV traffic) + static const bool gqa_tile = ggml_sycl_get_env("GGML_SYCL_FA_QUANT_GQA_TILE", 1) != 0; + if (Q->ne[1] <= 2 && !(gqa_tile && gqa_opt_applies)) { return BEST_FATTN_KERNEL_VEC; } } @@ -324,6 +339,9 @@ void ggml_sycl_flash_attn_ext(ggml_backend_sycl_context & ctx, ggml_tensor * dst case BEST_FATTN_KERNEL_MKL: ggml_sycl_flash_attn_ext_mkl(ctx, dst); break; + case BEST_FATTN_KERNEL_DEC: + ggml_sycl_flash_attn_ext_dec(ctx, dst); + break; } // --- Output fingerprint (GGML_SYCL_MKL_FA_DIAG=1) --- diff --git a/ggml/src/ggml-sycl/fusion.cpp b/ggml/src/ggml-sycl/fusion.cpp index 709bc8ca2a22..c8da4117d8c1 100644 --- a/ggml/src/ggml-sycl/fusion.cpp +++ b/ggml/src/ggml-sycl/fusion.cpp @@ -1,4 +1,5 @@ #include "fusion.hpp" +#include "mmvq.hpp" #include @@ -29,8 +30,20 @@ static bool ggml_sycl_should_fuse_mul_mat_glu(const ggml_tensor * gate, const gg return false; } - // only q4_K has a fused reorder GEMV so far, and it walks whole super-blocks - if (wu->type != GGML_TYPE_Q4_K || wu->ne[0] % QK_K != 0) { + // types with a fused reorder GEMV; each walks whole blocks + // ARC-LAB: Q6_K (the MTP draft head's FFN) behind GGML_SYCL_Q6K_GLU=1: the generic reorder GLU kernel, whose per-row + // dot products are the unfused reorder mat-vec's and whose epilogue is the SWIGLU kernel's silu(gate) * up + const bool q6k_ok = wu->type == GGML_TYPE_Q6_K && wu->ne[0] % QK_K == 0 && ggml_sycl_knob("GGML_SYCL_Q6K_GLU", 0); + const bool type_ok = (wu->type == GGML_TYPE_Q4_K && wu->ne[0] % QK_K == 0) || + (wu->type == GGML_TYPE_PQ2_0 && wu->ne[0] % QK_PQ2_0 == 0) || + (wu->type == GGML_TYPE_PTQ1_0 && wu->ne[0] % QK_PTQ1_0 == 0) || q6k_ok; + if (!type_ok) { + return false; + } + // single-token PQ2_0 decode is faster unfused (its own 2-row GEMV); fuse for 2..8 columns. ARC-LAB: PTQ1_0 has a + // single-column fused kernel (pairs core, GGML_SYCL_PTQ1_GLU1=0 turns it off) + if (wu->type != GGML_TYPE_Q4_K && wu->type != GGML_TYPE_Q6_K && act->ne[1] < 2 && + !(wu->type == GGML_TYPE_PTQ1_0 && ggml_sycl_ptq1_0_glu_n1_enabled())) { return false; } diff --git a/ggml/src/ggml-sycl/fwht.cpp b/ggml/src/ggml-sycl/fwht.cpp index fb48d7fec70d..19b5d2e9f978 100644 --- a/ggml/src/ggml-sycl/fwht.cpp +++ b/ggml/src/ggml-sycl/fwht.cpp @@ -402,3 +402,201 @@ bool ggml_sycl_op_fwht(ggml_backend_sycl_context & ctx, const ggml_tensor * src, return false; } } + +// ARC-LAB (Torchit1/llama.cpp arc-b580): FWHT with the Hadamard sign flip done during the load +// (dst = H * (src * signs)), F32 or F16 input. Kept beside the general FWHT above, which carries the +// wide and Kronecker sizes from upstream; only the fused sign-flip entry point below uses it. +namespace fwht_signed_impl { + +template +static void fwht_kernel(const T * __restrict__ src, float * __restrict__ dst, const int64_t n_rows, + const float scale, const float * __restrict__ signs, const int n_blk, + const sycl::nd_item<2> & item) { + const sycl::sub_group sg = item.get_sub_group(); + + const int64_t r = item.get_global_id(0); + if (r >= n_rows) { + return; + } + + src += r * N; + dst += r * N; + + constexpr int el_w = N / WARP_SIZE; + static_assert(el_w >= 1 && N % WARP_SIZE == 0, "row must be a whole number of sub-group widths"); + + float reg[el_w]; + const int lane = sg.get_local_linear_id(); + + const float * signs_row = has_signs ? signs + (r % n_blk) * N : nullptr; +#pragma unroll + for (int i = 0; i < el_w; ++i) { + reg[i] = (float) src[i * WARP_SIZE + lane] * scale; + if constexpr (has_signs) { + reg[i] *= signs_row[i * WARP_SIZE + lane]; + } + } + + // Butterflies inside the sub-group. The partner of a lane with bit h clear is the + // lower index of the pair, so it takes the sum and the upper takes lower - upper. +#pragma unroll + for (int h = 1; h < WARP_SIZE; h *= 2) { +#pragma unroll + for (int j = 0; j < el_w; ++j) { + const float val = reg[j]; + const float val2 = dpct::permute_sub_group_by_xor(sg, val, h, WARP_SIZE); + + reg[j] = (lane & h) == 0 ? val + val2 : val2 - val; + } + } + + // Butterflies across registers: h is a multiple of WARP_SIZE, so the partner of + // element i*WARP_SIZE + lane lives in reg[i + h/WARP_SIZE] on the same lane. +#pragma unroll + for (int h = WARP_SIZE; h < N; h *= 2) { + const int step = h / WARP_SIZE; +#pragma unroll + for (int j = 0; j < el_w; j += 2 * step) { +#pragma unroll + for (int k = 0; k < step; ++k) { + const float x = reg[j + k]; + const float y = reg[j + k + step]; + + reg[j + k] = x + y; + reg[j + k + step] = x - y; + } + } + } + +#pragma unroll + for (int i = 0; i < el_w; ++i) { + dst[i * WARP_SIZE + lane] = reg[i]; + } +} + +// Wide rows: one work-group per row, butterflies through local memory. +template +static void fwht_kernel_slm(const T * __restrict__ src, float * __restrict__ dst, const float scale, + const float * __restrict__ signs, const int n_blk, float * s, const sycl::nd_item<1> & item) { + const int64_t r = item.get_group(0); + const int tid = item.get_local_id(0); + const int nt = item.get_local_range(0); + + src += r * N; + dst += r * N; + const float * signs_row = has_signs ? signs + (r % n_blk) * N : nullptr; + + for (int i = tid; i < N; i += nt) { + float v = (float) src[i] * scale; + if constexpr (has_signs) { + v *= signs_row[i]; + } + s[i] = v; + } + item.barrier(sycl::access::fence_space::local_space); + + for (int h = 1; h < N; h *= 2) { + for (int idx = tid; idx < N / 2; idx += nt) { + const int j = (idx / h) * 2 * h + idx % h; + const float x = s[j]; + const float y = s[j + h]; + s[j] = x + y; + s[j + h] = x - y; + } + item.barrier(sycl::access::fence_space::local_space); + } + + for (int i = tid; i < N; i += nt) { + dst[i] = s[i]; + } +} + +template +static void launch_fwht(const T * src, float * dst, const int64_t n_rows, const float scale, const float * signs, + const int n_blk, dpct::queue_ptr stream) { + if constexpr (N <= 1024) { + constexpr int rows_per_block = 4; + + const int64_t num_blocks = (n_rows + rows_per_block - 1) / rows_per_block; + + // dim 1 is the fastest-varying, so a sub-group is exactly one row's WARP_SIZE lanes. + const sycl::range<2> global(num_blocks * rows_per_block, WARP_SIZE); + const sycl::range<2> local(rows_per_block, WARP_SIZE); + + stream->parallel_for(sycl::nd_range<2>(global, local), + [=](sycl::nd_item<2> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + fwht_kernel(src, dst, n_rows, scale, signs, n_blk, item); + }); + } else { + constexpr int nt = 256; + stream->submit([&](sycl::handler & cgh) { + sycl::local_accessor s(sycl::range<1>(N), cgh); + cgh.parallel_for(sycl::nd_range<1>(n_rows * nt, nt), [=](sycl::nd_item<1> item) { + fwht_kernel_slm(src, dst, scale, signs, n_blk, + s.get_multi_ptr().get(), item); + }); + }); + } +} + +template +static bool fwht_dispatch(const T * src_d, float * dst_d, const int n, const int64_t rows, const float scale, + const float * signs, const int n_blk, dpct::queue_ptr stream) { + switch (n) { +#define FWHT_CASE(NN) \ + case NN: \ + if (signs) { \ + launch_fwht(src_d, dst_d, rows, scale, signs, n_blk, stream); \ + } else { \ + launch_fwht(src_d, dst_d, rows, scale, nullptr, 1, stream); \ + } \ + return true; + FWHT_CASE(64) + FWHT_CASE(128) + FWHT_CASE(256) + FWHT_CASE(512) + FWHT_CASE(1024) + FWHT_CASE(2048) + FWHT_CASE(4096) + FWHT_CASE(8192) +#undef FWHT_CASE + default: + return false; + } +} + +static bool fwht_run(ggml_backend_sycl_context & ctx, const ggml_tensor * src, ggml_tensor * dst, + const ggml_tensor * signs_t) { + if ((src->type != GGML_TYPE_F32 && src->type != GGML_TYPE_F16) || dst->type != GGML_TYPE_F32) { + return false; + } + if (ggml_nelements(src) != ggml_nelements(dst) || !ggml_is_contiguous(src) || !ggml_is_contiguous(dst)) { + return false; + } + + const int n = (int) dst->ne[0]; + const int64_t rows = ggml_nelements(dst) / n; + + const float * signs = nullptr; + int n_blk = 1; + if (signs_t) { + if (signs_t->type != GGML_TYPE_F32 || !ggml_is_contiguous(signs_t) || signs_t->ne[0] % n != 0) { + return false; + } + signs = (const float *) signs_t->data; + n_blk = (int) (signs_t->ne[0] / n); + } + + const float scale = 1.0f / std::sqrt((float) n); + if (src->type == GGML_TYPE_F32) { + return fwht_dispatch((const float *) src->data, (float *) dst->data, n, rows, scale, signs, n_blk, ctx.stream()); + } + return fwht_dispatch((const sycl::half *) src->data, (float *) dst->data, n, rows, scale, signs, n_blk, ctx.stream()); +} + +} // namespace fwht_signed_impl + +bool ggml_sycl_op_fwht_signed(ggml_backend_sycl_context & ctx, const ggml_tensor * src, const ggml_tensor * signs, + ggml_tensor * dst) { + return fwht_signed_impl::fwht_run(ctx, src, dst, signs); +} diff --git a/ggml/src/ggml-sycl/fwht.hpp b/ggml/src/ggml-sycl/fwht.hpp index cd238cfaf37b..7dee0ffe38ef 100644 --- a/ggml/src/ggml-sycl/fwht.hpp +++ b/ggml/src/ggml-sycl/fwht.hpp @@ -9,4 +9,9 @@ // ordinary mat-mul dispatch. bool ggml_sycl_op_fwht(ggml_backend_sycl_context & ctx, const ggml_tensor * src, ggml_tensor * dst); +// Same, with the Hadamard sign flip (dst = H * (src * signs)) done during the load. +// signs: F32, ne[0] a multiple of the block width, repeating per block row. +bool ggml_sycl_op_fwht_signed(ggml_backend_sycl_context & ctx, const ggml_tensor * src, const ggml_tensor * signs, + ggml_tensor * dst); + #endif // GGML_SYCL_FWHT_HPP diff --git a/ggml/src/ggml-sycl/gated_delta_net.cpp b/ggml/src/ggml-sycl/gated_delta_net.cpp index 8468bbf5bd07..740a3a1efded 100644 --- a/ggml/src/ggml-sycl/gated_delta_net.cpp +++ b/ggml/src/ggml-sycl/gated_delta_net.cpp @@ -1,3 +1,4 @@ +#include #include #include "dpct/helper.hpp" #include "common.hpp" @@ -30,7 +31,9 @@ void gated_delta_net_sycl(const float * q, const sycl::uint3 rq3_magic, float scale, int64_t state_slot_stride, - int K) { + int K, + const int32_t * s_ids, + int64_t s_row_stride) { auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>(); const uint32_t h_idx = item_ct1.get_group(2); const uint32_t sequence = item_ct1.get_group(1); @@ -45,7 +48,9 @@ void gated_delta_net_sycl(const float * q, // input state holds s0 only [S_v, S_v, H, n_seqs] — seq stride is D = H * S_v * S_v. // output state layout (per-slot D * n_seqs) — same per-(seq,head) offset as before. - const int64_t state_in_offset = sequence * H * S_v * S_v + h_idx * S_v * S_v; + // fused gather: read this sequence's live state straight from its cache row s_ids[sequence] + const int64_t state_in_offset = (s_ids ? (int64_t) s_ids[sequence] * s_row_stride : (int64_t) sequence * H * S_v * S_v) + + h_idx * S_v * S_v; const int64_t state_out_offset = (sequence * H + h_idx) * S_v * S_v; state += state_out_offset; curr_state += state_in_offset + col * S_v; @@ -162,6 +167,195 @@ void gated_delta_net_sycl(const float * q, } } +// ARC-LAB token-blocked sequential kernel (non-KDA, S_v = 128). Same per-token recurrence and per-column +// arithmetic/reduction order as gated_delta_net_sycl, but the sequential kernel stalls on global-memory latency +// every token (~0.4 us/step on the B580 even with the GPU mostly idle) and needs ~5 rounds of threads at 48 heads. +// Here each subgroup owns NC columns and shares its k/q registers across them (CUDA's cols_per_warp), and the +// work-group stages T tokens of k, q, v, exp(g), beta in SLM; the next block's global loads are issued into +// registers before the current block is computed, so their latency overlaps T steps of compute. +template +static void gdn_blocked_sycl(const float * q, const float * k, const float * v, const float * g, const float * beta, + const float * curr_state, float * dst, float * state, int64_t H, int64_t n_tokens, + int64_t n_seqs, int64_t sq1, int64_t sq2, int64_t sq3, int64_t sv1, int64_t sv2, + int64_t sv3, int64_t sb1, int64_t sb2, int64_t sb3, int64_t neqk1, int64_t rq3, + float scale, int64_t state_slot_stride, int K, const int32_t * s_ids, + int64_t s_row_stride, dpct::queue_ptr stream) { + constexpr int S = 128, L = 16, RPL = S / L; + constexpr int NCW = NC * NW, WGS = NW * L, ncb = S / NCW; + constexpr int PK = T * S / WGS, PV = T * NCW / WGS; // per-thread prefetch registers + static_assert((T * S) % WGS == 0 && (T * NCW) % WGS == 0 && T <= WGS && S % NCW == 0, "gdn_blocked shape"); + stream->submit([&](sycl::handler & cgh) { + sycl::local_accessor sk(sycl::range<1>(T * S), cgh); + sycl::local_accessor sq(sycl::range<1>(T * S), cgh); + sycl::local_accessor sv(sycl::range<1>(T * NCW), cgh); + sycl::local_accessor sgb(sycl::range<1>(2 * T), cgh); + cgh.parallel_for(sycl::nd_range<1>(sycl::range<1>((size_t) (n_seqs * H * ncb * WGS)), sycl::range<1>(WGS)), + [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(16)]] { + const int tid = it.get_local_id(0); + const int lane = tid % L; + const int w = tid / L; + const int64_t grp = it.get_group(0); + const int64_t seq = grp / (H * ncb); + const int64_t h = (grp / ncb) % H; + const int colb = (int) (grp % ncb) * NCW; // first column of the work-group + const int col0 = colb + w * NC; // first column of this subgroup + const int64_t iq1 = h % neqk1; + const int64_t iq3 = seq / rq3; + + const float * qb = q + iq3 * sq3 + iq1 * sq1; + const float * kb = k + iq3 * sq3 + iq1 * sq1; + const float * vb = v + seq * sv3 + h * sv1 + colb; + const float * gp = g + seq * sb3 + h * sb1; + const float * bp = beta + seq * sb3 + h * sb1; + float * attn = dst + (seq * n_tokens * H + h) * S; + float * st = state + (seq * H + h) * S * S; + const float * cs = curr_state + (s_ids ? (int64_t) s_ids[seq] * s_row_stride : seq * H * S * S) + h * S * S; + + float s[NC][RPL]; +#pragma unroll + for (int c = 0; c < NC; ++c) { +#pragma unroll + for (int r = 0; r < RPL; ++r) { + s[c][r] = cs[(col0 + c) * S + r * L + lane]; + } + } + + float pk[PK], pq[PK], pv[PV], pg = 0.0f, pb = 0.0f; + auto fetch = [&](int64_t t0) { +#pragma unroll + for (int j = 0; j < PK; ++j) { + const int e = tid + j * WGS, t = e / S, i = e % S; + const bool ok = t0 + t < n_tokens; + pk[j] = ok ? kb[(t0 + t) * sq2 + i] : 0.0f; + pq[j] = ok ? qb[(t0 + t) * sq2 + i] : 0.0f; + } +#pragma unroll + for (int j = 0; j < PV; ++j) { + const int e = tid + j * WGS, t = e / NCW, c = e % NCW; + pv[j] = t0 + t < n_tokens ? vb[(t0 + t) * sv2 + c] : 0.0f; + } + if (tid < T && t0 + tid < n_tokens) { + pg = gp[(t0 + tid) * sb2]; + pb = bp[(t0 + tid) * sb2]; + } + }; + fetch(0); + + for (int64_t t0 = 0; t0 < n_tokens; t0 += T) { + it.barrier(sycl::access::fence_space::local_space); // previous block's SLM reads are done +#pragma unroll + for (int j = 0; j < PK; ++j) { + sk[tid + j * WGS] = pk[j]; + sq[tid + j * WGS] = pq[j]; + } +#pragma unroll + for (int j = 0; j < PV; ++j) { + sv[tid + j * WGS] = pv[j]; + } + if (tid < T) { + sgb[tid] = sycl::native::exp(pg); + sgb[T + tid] = pb; + } + it.barrier(sycl::access::fence_space::local_space); + if (t0 + T < n_tokens) { + fetch(t0 + T); // in flight while this block is computed + } + + const int nt = (int) sycl::min((int64_t) T, n_tokens - t0); + for (int t = 0; t < nt; ++t) { + float kr[RPL], qr[RPL]; +#pragma unroll + for (int r = 0; r < RPL; ++r) { + kr[r] = sk[t * S + r * L + lane]; + qr[r] = sq[t * S + r * L + lane]; + } + const float g_val = sgb[t]; + const float beta_val = sgb[T + t]; + float out = 0.0f; +#pragma unroll + for (int c = 0; c < NC; ++c) { + float kv_shard = 0.0f; +#pragma unroll + for (int r = 0; r < RPL; ++r) { + kv_shard += s[c][r] * kr[r]; + } + const float kv_col = warp_reduce_sum(kv_shard); + const float delta_col = (sv[t * NCW + w * NC + c] - g_val * kv_col) * beta_val; + float attn_partial = 0.0f; +#pragma unroll + for (int r = 0; r < RPL; ++r) { + s[c][r] = g_val * s[c][r] + kr[r] * delta_col; + attn_partial += s[c][r] * qr[r]; + } + const float attn_col = warp_reduce_sum(attn_partial); // same sum on every lane + if (lane == c) { + out = attn_col; + } + } + if (lane < NC) { + attn[(t0 + t) * S * H + col0 + lane] = out * scale; + } + if constexpr (keep_rs_t) { + const int target_slot = (int) (n_tokens - 1 - (t0 + t)); + if (target_slot >= 0 && target_slot < K) { + float * cst = st + target_slot * state_slot_stride; +#pragma unroll + for (int c = 0; c < NC; ++c) { +#pragma unroll + for (int r = 0; r < RPL; ++r) { + cst[(col0 + c) * S + r * L + lane] = s[c][r]; + } + } + } + } + } + } + + if constexpr (!keep_rs_t) { +#pragma unroll + for (int c = 0; c < NC; ++c) { +#pragma unroll + for (int r = 0; r < RPL; ++r) { + st[(col0 + c) * S + r * L + lane] = s[c][r]; + } + } + } + }); + }); +} + +template +static void launch_gdn_blocked(int variant, const float * q, const float * k, const float * v, const float * g, + const float * beta, const float * s0, float * dst, float * state, int64_t H, + int64_t n_tokens, int64_t n_seqs, int64_t sq1, int64_t sq2, int64_t sq3, int64_t sv1, + int64_t sv2, int64_t sv3, int64_t sb1, int64_t sb2, int64_t sb3, int64_t neqk1, + int64_t rq3, float scale, int64_t state_slot_stride, int K, const int32_t * s_ids, + int64_t s_row_stride, dpct::queue_ptr stream) { +#define GDN_BLK(NC, NW, T) gdn_blocked_sycl(q, k, v, g, beta, s0, dst, state, H, n_tokens, n_seqs, \ + sq1, sq2, sq3, sv1, sv2, sv3, sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, s_ids, s_row_stride, stream) + if (variant < 0) { + // auto: widest NC that still leaves ~768 subgroups (~60% of the B580's 1280 thread slots) to hide latency. + // B580, 1024 tokens (old kernel -> this): 48 heads 3383 -> 1688 us (NC 8, T 8); 32 heads 2762 -> 1024 (NC 4); + // 4 heads 403 -> 312 (NC 1). NC 8 at 32 heads = 1767, NC 4 at 48 heads = 1934 (more than one round of threads). + const int64_t cols = H * n_seqs * 128; + variant = cols >= 768 * 8 ? 9 : cols >= 768 * 4 ? 3 : cols >= 768 * 2 ? 4 : 5; + } + switch (variant) { // lab sweep (GGML_SYCL_GDN_BLK_VARIANT); NC columns/subgroup, NW subgroups, T tokens/block + case 1: GDN_BLK(8, 16, 16); break; + case 2: GDN_BLK(8, 8, 16); break; + case 3: GDN_BLK(4, 16, 16); break; + case 4: GDN_BLK(2, 16, 16); break; + case 5: GDN_BLK(1, 16, 16); break; + case 6: GDN_BLK(4, 32, 16); break; + case 7: GDN_BLK(4, 8, 32); break; + case 8: GDN_BLK(8, 8, 8); break; + case 9: GDN_BLK(8, 16, 8); break; + case 10: GDN_BLK(4, 16, 8); break; + default: GDN_BLK(4, 8, 16); break; + } +#undef GDN_BLK +} + template static void launch_gated_delta_net(const float * q_d, const float * k_d, @@ -189,8 +383,21 @@ static void launch_gated_delta_net(const float * q_d, float scale, int64_t state_slot_stride, int K, + const int32_t * s_ids, + int64_t s_row_stride, dpct::queue_ptr stream) { //TODO: Add chunked kernel for even faster pre-fill + if constexpr (!KDA) { + static const bool blk_off = getenv("GGML_SYCL_GDN_BLOCKED_OFF") != nullptr; + static const int blk_variant = getenv("GGML_SYCL_GDN_BLK_VARIANT") ? atoi(getenv("GGML_SYCL_GDN_BLK_VARIANT")) : -1; + static const int blk_min = getenv("GGML_SYCL_GDN_BLK_MIN") ? atoi(getenv("GGML_SYCL_GDN_BLK_MIN")) : 16; + if (!blk_off && S_v == 128 && n_tokens >= blk_min) { + launch_gdn_blocked(blk_variant, q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, + n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, sb1, sb2, sb3, neqk1, rq3, scale, + state_slot_stride, K, s_ids, s_row_stride, stream); + return; + } + } const int warp_size = ggml_sycl_info().devices[ggml_sycl_get_device()].warp_size; const int num_warps = 4; @@ -208,7 +415,7 @@ static void launch_gated_delta_net(const float * q_d, [=](sycl::nd_item<3> /*item_ct1*/) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { gated_delta_net_sycl(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, sq1, sq2, sq3, sv1, sv2, sv3, sb1, sb2, - sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K); + sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K, s_ids, s_row_stride); }); } break; @@ -219,7 +426,7 @@ static void launch_gated_delta_net(const float * q_d, [=](sycl::nd_item<3> /*item_ct1*/) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { gated_delta_net_sycl(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, sq1, sq2, sq3, sv1, sv2, sv3, sb1, sb2, - sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K); + sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K, s_ids, s_row_stride); }); } break; @@ -230,7 +437,7 @@ static void launch_gated_delta_net(const float * q_d, [=](sycl::nd_item<3> /*item_ct1*/) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { gated_delta_net_sycl( q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, sq1, sq2, - sq3, sv1, sv2, sv3, sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K); + sq3, sv1, sv2, sv3, sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K, s_ids, s_row_stride); }); } break; @@ -242,7 +449,7 @@ static void launch_gated_delta_net(const float * q_d, [=](sycl::nd_item<3> /*item_ct1*/) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { gated_delta_net_sycl( q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, sq1, sq2, - sq3, sv1, sv2, sv3, sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K); + sq3, sv1, sv2, sv3, sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K, s_ids, s_row_stride); }); } break; @@ -253,6 +460,203 @@ static void launch_gated_delta_net(const float * q_d, } } +// ARC-LAB: chunked prefill for the scalar-gate gated delta rule (S_k = S_v = 128, one sequence). +// Per token: delta = beta * (v - gamma * S^T k), S <- gamma * S + k delta^T, o = scale * S^T q, gamma = exp(g). +// Per chunk of C tokens (UT / WY form, as build_delta_net_chunking), with G = cumsum(g) inside the chunk: +// A[t][s] = beta_t exp(G_t - G_s) k_t.k_s (s < t) +// R = beta * (V - exp(G) * (K S0)) -> Delta = (I + A)^-1 R (forward substitution) +// O = scale * (exp(G) * (Q S0) + (exp(G_t - G_s) q_t.k_s)_{s<=t} Delta) +// S_C = exp(G_last) S0 + K^T (exp(G_last - G) * Delta) +// One work-group per (head, block of W value columns): the value columns of S evolve independently, the C x C +// terms are recomputed per block. q/k of a chunk are shared in SLM by all W columns; every product is fused, so +// the serial chain is one step per chunk instead of one per token. fp32 throughout. +namespace { +constexpr int GDN_CH_C = 16; // tokens per chunk +constexpr int GDN_CH_W = 32; // value columns per work-group +constexpr int GDN_CH_S = 128; // head size +constexpr int GDN_CH_KP = GDN_CH_S + 1; // padded SLM row stride of the K / Q tiles +constexpr int GDN_CH_WG = 256; +} + +template +static void gdn_chunked_sycl_t(const float * q, const float * k, const float * v, const float * g, const float * beta, + const float * s_in, float * dst, float * s_out, int64_t H, int64_t n_proc, + int64_t sq1, int64_t sq2, int64_t sv1, int64_t sv2, int64_t sb1, int64_t sb2, + int64_t neqk1, float scale, const int32_t * s_ids, int64_t s_row_stride, + dpct::queue_ptr stream) { + constexpr int S = GDN_CH_S, KP = GDN_CH_KP, WG = GDN_CH_WG; + constexpr int ncb = S / W; + static_assert((C * W) % WG == 0 || C * W < WG, "C*W vs WG"); + stream->submit([&](sycl::handler & cgh) { + sycl::local_accessor sS(sycl::range<1>(S * W), cgh); + sycl::local_accessor sK(sycl::range<1>(C * KP), cgh); + sycl::local_accessor sQ(sycl::range<1>(C * KP), cgh); + sycl::local_accessor sV(sycl::range<1>(C * W), cgh); + sycl::local_accessor sD(sycl::range<1>(C * W), cgh); + sycl::local_accessor sA(sycl::range<1>(C * C), cgh); + sycl::local_accessor sQK(sycl::range<1>(C * C), cgh); + sycl::local_accessor sG(sycl::range<1>(C), cgh); + sycl::local_accessor sB(sycl::range<1>(C), cgh); + cgh.parallel_for(sycl::nd_range<1>(sycl::range<1>((size_t) H * ncb * WG), sycl::range<1>(WG)), + [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(16)]] { + const int tid = it.get_local_id(0); + const int grp = it.get_group(0); + const int h = grp / ncb; + const int col0 = (grp % ncb) * W; + const int64_t hk = h % neqk1; + + const float * sin = s_in + (s_ids ? (int64_t) s_ids[0] * s_row_stride : (int64_t) 0) + (int64_t) h * S * S; + for (int e = tid; e < S * W; e += WG) { + const int c = e / S, i = e % S; + sS[i * W + c] = sin[(int64_t) (col0 + c) * S + i]; + } + it.barrier(sycl::access::fence_space::local_space); + + for (int64_t t0 = 0; t0 < n_proc; t0 += C) { + const int nc = (int) sycl::min((int64_t) C, n_proc - t0); + for (int e = tid; e < C * S; e += WG) { + const int t = e / S, i = e % S; + float kv = 0.0f, qv = 0.0f; + if (t < nc) { + const int64_t off = (t0 + t) * sq2 + hk * sq1 + i; + kv = k[off]; + qv = q[off]; + } + sK[t * KP + i] = kv; + sQ[t * KP + i] = qv; + } + for (int e = tid; e < C * W; e += WG) { + const int t = e / W, c = e % W; + sV[e] = t < nc ? v[(t0 + t) * sv2 + h * sv1 + col0 + c] : 0.0f; + } + if (tid < C) { + // padded tokens: g = 0 keeps G at the last real value, beta = 0 zeroes their delta + sG[tid] = tid < nc ? g[(t0 + tid) * sb2 + h * sb1] : 0.0f; + sB[tid] = tid < nc ? beta[(t0 + tid) * sb2 + h * sb1] : 0.0f; + } + it.barrier(sycl::access::fence_space::local_space); + if (tid == 0) { + float acc = 0.0f; + for (int t = 0; t < C; ++t) { + acc += sG[t]; + sG[t] = acc; + } + } + it.barrier(sycl::access::fence_space::local_space); + + // C x C terms: A (strictly lower, beta and decay folded in) and the decayed q.k (lower incl. diag) + for (int e = tid; e < 2 * C * C; e += WG) { + const int which = e / (C * C), ts = e % (C * C), t = ts / C, s2 = ts % C; + const int ra = which ? t * KP : t * KP; + float d = 0.0f; + if (which == 0) { + for (int i = 0; i < S; ++i) d += sK[ra + i] * sK[s2 * KP + i]; + sA[ts] = s2 < t ? sB[t] * sycl::exp(sG[t] - sG[s2]) * d : 0.0f; + } else { + for (int i = 0; i < S; ++i) d += sQ[ra + i] * sK[s2 * KP + i]; + sQK[ts] = s2 <= t ? sycl::exp(sG[t] - sG[s2]) * d : 0.0f; + } + } + // K S0 and Q S0 for this column block (C x W each), sharing the S0 reads + float qs_reg[(C * W + WG - 1) / WG]; +#pragma unroll + for (int r = 0; r < (C * W + WG - 1) / WG; ++r) { + const int e = tid + r * WG, t = e / W, c = e % W; + if (e >= C * W) break; + float ks = 0.0f, qs = 0.0f; + for (int i = 0; i < S; ++i) { + const float sv = sS[i * W + c]; + ks += sK[t * KP + i] * sv; + qs += sQ[t * KP + i] * sv; + } + const float eg = sycl::exp(sG[t]); + sD[e] = sB[t] * (sV[e] - eg * ks); // R + qs_reg[r] = eg * qs; + } + it.barrier(sycl::access::fence_space::local_space); + + // Delta = (I + A)^-1 R: forward substitution, one column per work-item + if (tid < W) { + const int c = tid; + for (int t = 1; t < C; ++t) { + float r = sD[t * W + c]; + for (int s2 = 0; s2 < t; ++s2) r -= sA[t * C + s2] * sD[s2 * W + c]; + sD[t * W + c] = r; + } + } + it.barrier(sycl::access::fence_space::local_space); + +#pragma unroll + for (int r = 0; r < (C * W + WG - 1) / WG; ++r) { + const int e = tid + r * WG, t = e / W, c = e % W; + if (e < C * W && t < nc) { + float o = qs_reg[r]; + for (int s2 = 0; s2 <= t; ++s2) o += sQK[t * C + s2] * sD[s2 * W + c]; + dst[((t0 + t) * H + h) * S + col0 + c] = o * scale; + } + } + + const float glast = sG[C - 1]; + const float egl = sycl::exp(glast); + float dec[C]; +#pragma unroll + for (int t = 0; t < C; ++t) dec[t] = sycl::exp(glast - sG[t]); + for (int e = tid; e < S * W; e += WG) { + const int i = e / W, c = e % W; + float acc = egl * sS[e]; +#pragma unroll + for (int t = 0; t < C; ++t) acc += sK[t * KP + i] * dec[t] * sD[t * W + c]; + sS[e] = acc; + } + it.barrier(sycl::access::fence_space::local_space); + } + + float * so = s_out + (int64_t) h * S * S; + for (int e = tid; e < S * W; e += WG) { + const int c = e / S, i = e % S; + so[(int64_t) (col0 + c) * S + i] = sS[i * W + c]; + } + }); + }); +} + +static void gdn_chunked_sycl(const float * q, const float * k, const float * v, const float * g, const float * beta, + const float * s_in, float * dst, float * s_out, int64_t H, int64_t n_proc, + int64_t sq1, int64_t sq2, int64_t sv1, int64_t sv2, int64_t sb1, int64_t sb2, + int64_t neqk1, float scale, const int32_t * s_ids, int64_t s_row_stride, + dpct::queue_ptr stream) { + static const int variant = getenv("GGML_SYCL_GDN_CH_VARIANT") ? atoi(getenv("GGML_SYCL_GDN_CH_VARIANT")) : 0; // lab sweep +#define GDN_CH_CALL(CC, WW) gdn_chunked_sycl_t(q, k, v, g, beta, s_in, dst, s_out, H, n_proc, sq1, sq2, sv1, sv2, sb1, sb2, neqk1, scale, s_ids, s_row_stride, stream) + switch (variant) { + case 1: GDN_CH_CALL(16, 64); break; + case 2: GDN_CH_CALL(32, 32); break; + case 3: GDN_CH_CALL(32, 16); break; + case 4: GDN_CH_CALL(8, 32); break; + case 5: GDN_CH_CALL(16, 16); break; + case 6: GDN_CH_CALL(8, 64); break; + default: GDN_CH_CALL(16, 32); break; + } +#undef GDN_CH_CALL +} + +// GET_ROWS (recurrent-state gather) -> GATED_DELTA_NET src[5] registrations, per context, reset per graph: +// the gather is skipped and the kernel indexes the cache rows directly (port of the CUDA #220 fusion). +struct ggml_sycl_gdn_gather { + const float * base; + const int32_t * ids; + int64_t row_stride; +}; +static std::unordered_map> g_gdn_gathers; + +void ggml_sycl_gdn_gather_reset(const ggml_backend_sycl_context & ctx) { + g_gdn_gathers[&ctx].clear(); +} + +void ggml_sycl_gdn_gather_set(const ggml_backend_sycl_context & ctx, const ggml_tensor * gdn, const float * base, + const int32_t * ids, int64_t row_stride) { + g_gdn_gathers[&ctx][gdn] = { base, ids, row_stride }; +} + static void ggml_sycl_op_gated_delta_net_impl(ggml_backend_sycl_context & ctx, ggml_tensor * dst, const ggml_sycl_gated_delta_net_fused_cache * cache) { ggml_tensor * src_q = dst->src[0]; @@ -288,8 +692,19 @@ static void ggml_sycl_op_gated_delta_net_impl(ggml_backend_sycl_context & ctx, g const float * g_d = (const float *) src_g->data; const float * b_d = (const float *) src_beta->data; - const float * s_d = (const float *) src_state->data; - float * dst_d = (float *) dst->data; + const float * s_d = (const float *) src_state->data; + float * dst_d = (float *) dst->data; + const int32_t * s_ids = nullptr; + int64_t s_row_stride = 0; + { + auto & m = g_gdn_gathers[&ctx]; + auto it = m.find(dst); + if (it != m.end()) { + s_d = it->second.base; + s_ids = it->second.ids; + s_row_stride = it->second.row_stride; + } + } GGML_ASSERT(ggml_is_contiguous_rows(src_q)); GGML_ASSERT(ggml_is_contiguous_rows(src_k)); @@ -331,21 +746,45 @@ static void ggml_sycl_op_gated_delta_net_impl(ggml_backend_sycl_context & ctx, g if (keep_rs) { launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, stream); + sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, s_ids, s_row_stride, stream); } else { launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, stream); + sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, s_ids, s_row_stride, stream); } } else { + // ARC-LAB chunked prefill (OPT-IN, GGML_SYCL_GDN_CHUNKED=1): all but the last K tokens (the snapshot window) + // go through gdn_chunked_sycl; the sequential kernel then finishes the tail from that state. Correct, but on + // the B580 every shape measured slower than the sequential kernel (scalar SLM math, no XMX) - see README. + static const bool chunk_off = getenv("GGML_SYCL_GDN_CHUNKED") == nullptr; + const int64_t n_tail = keep_rs ? (int64_t) K : 0; + const int64_t n_proc = n_tokens - n_tail; + if (!chunk_off && S_v == GDN_CH_S && n_seqs == 1 && neq3 == 1 && n_proc >= 64) { + if (n_tail == 0) { + float * so = state_d; // final state straight to its destination (dst tail or cache slot 0) + gdn_chunked_sycl(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, so, H, n_proc, sq1, sq2, sv1, sv2, sb1, sb2, + neqk1, scale, s_ids, s_row_stride, stream); + return; + } + ggml_sycl_pool_alloc mid(ctx.pool(), (size_t) H * S_v * S_v); + gdn_chunked_sycl(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, mid.get(), H, n_proc, sq1, sq2, sv1, sv2, sb1, sb2, + neqk1, scale, s_ids, s_row_stride, stream); + // tail: the sequential kernel on tokens [n_proc, n_tokens) from the mid state; n_seqs == 1 so shifting + // the base pointers by n_proc tokens keeps every layout (dst row = t * H + h) + launch_gated_delta_net(q_d + n_proc * sq2, k_d + n_proc * sq2, v_d + n_proc * sv2, + g_d + n_proc * sb2, b_d + n_proc * sb2, mid.get(), dst_d + n_proc * S_v * H, state_d, + S_v, H, n_tail, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, + sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, nullptr, 0, stream); + return; + } if (keep_rs) { launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, stream); + sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, s_ids, s_row_stride, stream); } else { launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, stream); + sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, s_ids, s_row_stride, stream); } } } diff --git a/ggml/src/ggml-sycl/gated_delta_net.hpp b/ggml/src/ggml-sycl/gated_delta_net.hpp index 7903b8e06df2..da5dda3f4969 100644 --- a/ggml/src/ggml-sycl/gated_delta_net.hpp +++ b/ggml/src/ggml-sycl/gated_delta_net.hpp @@ -17,3 +17,8 @@ void ggml_sycl_gated_delta_net(ggml_backend_sycl_context & ctx, ggml_tensor * ds // same op, but writes the snapshot(s) into the cache instead of dst (see ggml_sycl_try_gdn_cache_fusion) void ggml_sycl_op_gated_delta_net_fused_cache(ggml_backend_sycl_context & ctx, ggml_tensor * dst, ggml_sycl_gated_delta_net_fused_cache cache); + +// fused recurrent-state gather (see ggml_sycl_try_gdn_gather_skip): reset per graph, set per GDN node +void ggml_sycl_gdn_gather_reset(const ggml_backend_sycl_context & ctx); +void ggml_sycl_gdn_gather_set(const ggml_backend_sycl_context & ctx, const ggml_tensor * gdn, const float * base, + const int32_t * ids, int64_t row_stride); diff --git a/ggml/src/ggml-sycl/gemm.hpp b/ggml/src/ggml-sycl/gemm.hpp index c202da110beb..94bc58efe026 100644 --- a/ggml/src/ggml-sycl/gemm.hpp +++ b/ggml/src/ggml-sycl/gemm.hpp @@ -81,6 +81,54 @@ class DnnlGemmWrapper { matmul_prim.execute(stream, matmul_args); } + // dst[t * m + r] = sum_k act[t * k + c] * w[r][c] * scale[c / 128][r]; w is s4, row r contiguous along k. + static void gemm_s4_group(ggml_backend_sycl_context & ctx, int m, int n, int k, const sycl::half * act, + const uint8_t * w4, const float * scales, float * dst, const queue_ptr & q) { + auto stream = ctx.stream_dnnl(q); + auto eng = ctx.engine_dnnl(q); + + const auto act_md = dnnl::memory::desc({ n, k }, dt::f16, tag::ab); + const auto w_md = dnnl::memory::desc({ k, m }, dt::s4, tag::ba); + const auto dst_md = dnnl::memory::desc({ n, m }, dt::f32, tag::ab); + const auto sc_md = dnnl::memory::desc({ k / 128, m }, dt::f32, tag::ab); // oneDNN reads scales as dense ab, whatever the tag + + dnnl::primitive_attr attr; + attr.set_scratchpad_mode(dnnl::scratchpad_mode::user); + attr.set_scales(DNNL_ARG_WEIGHTS, (1 << 0) | (1 << 1), { 128, 1 }, dt::f32); + attr.set_fpmath_mode(dnnl::fpmath_mode::f16, true); + + auto pd = dnnl::matmul::primitive_desc(eng, act_md, w_md, dst_md, attr); + auto prim = dnnl::matmul(pd); + + std::unordered_map args; + args.insert({ DNNL_ARG_SRC, dnnl::memory(act_md, eng, const_cast(act)) }); + args.insert({ DNNL_ARG_WEIGHTS, dnnl::memory(w_md, eng, const_cast(w4)) }); + args.insert({ DNNL_ARG_DST, dnnl::memory(pd.dst_desc(), eng, dst) }); + args.insert({ DNNL_ARG_ATTR_SCALES | DNNL_ARG_WEIGHTS, dnnl::memory(sc_md, eng, const_cast(scales)) }); + args.insert({ DNNL_ARG_SCRATCHPAD, ctx.get_scratchpad_mem(pd.scratchpad_desc(), eng, q) }); + prim.execute(stream, args); + } + + // dst[t * m + r] = sum_k act[t * k + c] * w[r][c], all int8, int32 accumulate, f32 out (no scales) + static void gemm_s8(ggml_backend_sycl_context & ctx, int m, int n, int k, const int8_t * act, const int8_t * w, + float * dst, const queue_ptr & q) { + auto stream = ctx.stream_dnnl(q); + auto eng = ctx.engine_dnnl(q); + const auto act_md = dnnl::memory::desc({ n, k }, dt::s8, tag::ab); + const auto w_md = dnnl::memory::desc({ k, m }, dt::s8, tag::ba); + const auto dst_md = dnnl::memory::desc({ n, m }, dt::f32, tag::ab); + dnnl::primitive_attr attr; + attr.set_scratchpad_mode(dnnl::scratchpad_mode::user); + auto pd = dnnl::matmul::primitive_desc(eng, act_md, w_md, dst_md, attr); + auto prim = dnnl::matmul(pd); + std::unordered_map args; + args.insert({ DNNL_ARG_SRC, dnnl::memory(act_md, eng, const_cast(act)) }); + args.insert({ DNNL_ARG_WEIGHTS, dnnl::memory(w_md, eng, const_cast(w)) }); + args.insert({ DNNL_ARG_DST, dnnl::memory(pd.dst_desc(), eng, dst) }); + args.insert({ DNNL_ARG_SCRATCHPAD, ctx.get_scratchpad_mem(pd.scratchpad_desc(), eng, q) }); + prim.execute(stream, args); + } + static void row_gemm(ggml_backend_sycl_context & ctx, int m, int n, int k, const void * a, dt at, const void * b, dt bt, void * c, dt ct, const queue_ptr & q) { diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 97b5f35659f7..5dcf49c877ed 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -10,7 +10,13 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // +#include +#include +#include +#include #include +#include +#include #include #include #include @@ -51,6 +57,7 @@ #include "ggml.h" #include "ggml-sycl.h" +#include "ptq1-t2.hpp" #include "ggml-impl.h" #include "ggml-backend-impl.h" @@ -60,6 +67,7 @@ #include "ggml-sycl/common.hpp" #include "ggml-sycl/element_wise.hpp" #include "ggml-sycl/fwht.hpp" +#include "ggml-sycl/w8a8.hpp" #include "ggml-sycl/gemm.hpp" #include "ggml-sycl/getrows.hpp" #include "ggml-sycl/norm.hpp" @@ -91,6 +99,9 @@ static bool g_sycl_loaded = false; int g_ggml_sycl_debug = 0; int g_ggml_sycl_enable_optimize = 1; +int g_ggml_sycl_quant_f16_gemm = 1; +int g_ggml_sycl_quant_s4_gemm = 1; +int g_ggml_sycl_quant_w8a8 = 1; int g_ggml_sycl_enable_graph = 0; int g_ggml_sycl_enable_dnn = 1; int g_ggml_sycl_fa_onednn = 1; @@ -320,12 +331,32 @@ static const char* dev2dev_int2str(int dev2dev) { } } +// ARC-LAB: use the legacy Level Zero adapter unless the user chose one. On the B580 (oneAPI 2025.3, where V2 is the +// default) plain decode measured +1-1.5% in llama-bench and MTP +0.5%, outputs identical. Set at library load, before +// any SYCL platform query; SYCL_UR_USE_LEVEL_ZERO_V2=1 in the environment keeps V2. +namespace { +struct ggml_sycl_l0_adapter_default { + ggml_sycl_l0_adapter_default() { + if (getenv("SYCL_UR_USE_LEVEL_ZERO_V2") == nullptr) { +#ifdef _WIN32 + _putenv_s("SYCL_UR_USE_LEVEL_ZERO_V2", "0"); +#else + setenv("SYCL_UR_USE_LEVEL_ZERO_V2", "0", 0); +#endif + } + } +} ggml_sycl_l0_adapter_default_init; +} + static void ggml_check_sycl() try { static bool initialized = false; if (!initialized) { g_ggml_sycl_debug = ggml_sycl_get_env("GGML_SYCL_DEBUG", 0); g_ggml_sycl_enable_optimize = ggml_sycl_get_env("GGML_SYCL_ENABLE_OPT", 1); + g_ggml_sycl_quant_f16_gemm = ggml_sycl_get_env("GGML_SYCL_QUANT_F16_GEMM", 1); + g_ggml_sycl_quant_s4_gemm = ggml_sycl_get_env("GGML_SYCL_QUANT_S4_GEMM", 1); + g_ggml_sycl_quant_w8a8 = ggml_sycl_get_env("GGML_SYCL_QUANT_W8A8", 1); g_ggml_sycl_enable_graph = ggml_sycl_get_env("GGML_SYCL_ENABLE_GRAPH", 0); g_ggml_sycl_enable_dnn = ggml_sycl_get_env("GGML_SYCL_ENABLE_DNN", 1); g_ggml_sycl_fa_onednn = ggml_sycl_get_env("GGML_SYCL_FA_ONEDNN", 1); @@ -420,6 +451,8 @@ static void ggml_check_sycl() try { #endif GGML_LOG_INFO(" GGML_SYCL_ENABLE_OPT: %d\n", g_ggml_sycl_enable_optimize); + GGML_LOG_INFO(" GGML_SYCL_QUANT_F16_GEMM: %d\n", g_ggml_sycl_quant_f16_gemm); + GGML_LOG_INFO(" GGML_SYCL_QUANT_S4_GEMM: %d\n", g_ggml_sycl_quant_s4_gemm); #if defined(GGML_SYCL_SUPPORT_VMM) GGML_LOG_INFO(" GGML_SYCL_ENABLE_VMM: %d\n", g_ggml_sycl_enable_vmm); @@ -644,6 +677,10 @@ catch (sycl::exception const &exc) { std::exit(1); } +// ARC-LAB: per-token accounting of synchronous buffer_set_tensor calls, printed with the GGML_SYCL_HOST_TIMING report +static double g_st_ms = 0; static long g_st_calls = 0; static size_t g_st_bytes = 0; +static void ggml_sycl_note_set_tensor(size_t bytes, double ms) { g_st_ms += ms; g_st_calls++; g_st_bytes += bytes; } + static void ggml_backend_sycl_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor *tensor, const void *data, size_t offset, @@ -654,6 +691,29 @@ static void ggml_backend_sycl_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_backend_sycl_buffer_context * ctx = ( ggml_backend_sycl_buffer_context *)buffer->context; ggml_sycl_set_device(ctx->device); auto stream = &(dpct::dev_mgr::instance().get_device(ctx->device).default_queue()); + // ARC-LAB: GGML_SYCL_HOST_TIMING counts these (llama.cpp writes every per-token input through here, synchronously) + struct st_timer { + size_t bytes; std::chrono::steady_clock::time_point t0 = std::chrono::steady_clock::now(); + ~st_timer() { ggml_sycl_note_set_tensor(bytes, std::chrono::duration(std::chrono::steady_clock::now() - t0).count()); } + } st_guard{ size }; + // ARC-LAB: GGML_SYCL_FAST_SET_TENSOR=1: small writes (<= 4 MB, i.e. per-token inputs, not weights) skip the device-wide + // wait and the malloc'd bounce buffer (the mmap workaround is for weights loaded from disk): one pinned staging copy and + // one wait on this queue. Still synchronous for the caller. + const int fast_st = ggml_sycl_knob("GGML_SYCL_FAST_SET_TENSOR", 0); // 2 = keep the device-wide wait + if (size <= (4u << 20) && fast_st) { + if (fast_st == 2) { + SYCL_CHECK(CHECK_TRY_ERROR(dpct::dev_mgr::instance().get_device(ctx->device).queues_wait_and_throw())); + } + static char * pinned = nullptr; + if (!pinned) { + pinned = (char *) sycl::malloc_host(4u << 20, *stream); + } + if (pinned) { + memcpy(pinned, data, size); + SYCL_CHECK(CHECK_TRY_ERROR((*stream).memcpy((char *) tensor->data + offset, pinned, size).wait())); + return; + } + } SYCL_CHECK(CHECK_TRY_ERROR(dpct::dev_mgr::instance().get_device(ctx->device).queues_wait_and_throw())); #ifndef _WIN32 // Note: Use host buffer to save the data from mmap(), then copy to device. It's workaround for mmap() issue on PVC GPU. @@ -977,16 +1037,18 @@ static size_t ggml_backend_sycl_buffer_type_get_max_size(ggml_backend_buffer_typ static bool ggml_sycl_device_has_dpas16(int device); -// GGML_SYCL_DISABLE_XMX=1 keeps PQ2_0/PTQ1_0 off the XMX path (and its layout) -static bool ggml_sycl_xmx_disabled() { - static const bool disabled = ggml_sycl_get_env("GGML_SYCL_DISABLE_XMX", 0); - return disabled; +// GGML_SYCL_DISABLE_XMX=1 keeps PQ2_0/PTQ1_0 off the XMX path (and its layout), =0 keeps them on it. Unset, Xe3 +// leaves them to the MMVQ reorder: on an Arc B390 (PTQ1_0 weights) that is +16% decode and +6% prompt with ~0.9 GB less +// memory, at a higher but still small KLD (6.7e-4 vs 2.4e-4) +static bool ggml_sycl_xmx_disabled(int device) { + static const char * env = getenv("GGML_SYCL_DISABLE_XMX"); + return env ? atoi(env) != 0 : ggml_sycl_device_is_xe3(device); } // PTQ1_0 weights are expanded into the 34-byte PQ2_0 XMX blocks on first use (pq2_xmx.hpp), so on devices // that run that path their allocation reserves room for the expanded form static bool ggml_sycl_ptq1_xmx_expands(int device, const ggml_tensor * tensor) { - return tensor->type == GGML_TYPE_PTQ1_0 && g_ggml_sycl_enable_optimize && !ggml_sycl_xmx_disabled() && + return tensor->type == GGML_TYPE_PTQ1_0 && g_ggml_sycl_enable_optimize && !ggml_sycl_xmx_disabled(device) && tensor->ne[2] == 1 && tensor->ne[3] == 1 && ggml_sycl_pq2_xmx_supports_ne0(tensor->ne[0]) && ggml_sycl_device_has_dpas16(device); } @@ -1004,6 +1066,11 @@ static size_t ggml_backend_sycl_buffer_type_get_alloc_size(ggml_backend_buffer_t size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); } } + // ARC-LAB: room for the in-place TernSYCL 2-bit repack (GGML_SYCL_PTQ1_T2) + if ((tensor->type == GGML_TYPE_PTQ1_0 || tensor->type == GGML_TYPE_PQ2_0) && tensor->ne[2] == 1 && tensor->ne[3] == 1 && + ggml_sycl_t2_wants(tensor->name, tensor->ne[0], tensor->ne[1])) { + size = std::max(size, ggml_sycl_t2_bytes(tensor->ne[0], tensor->ne[1])); + } const auto * buft_ctx = (const ggml_backend_sycl_buffer_type_context *) buft->context; if (ggml_sycl_ptq1_xmx_expands(buft_ctx->device, tensor)) { @@ -2726,8 +2793,36 @@ inline void ggml_sycl_op_mul_mat_sycl( #ifdef GGML_SYCL_F16 bool use_fp16 = true; // TODO(Yu) SYCL capability check #else - // ternary weights are exact in FP16 and the FP16 GEMM is much faster than FP32 - bool use_fp16 = src0->type == GGML_TYPE_PQ2_0 || src0->type == GGML_TYPE_PTQ1_0; + // ternary weights are exact in FP16 and the FP16 GEMM is much faster than FP32; other quantized weights + // take it for large batches too (ARC-LAB: fp16 GEMM runs on XMX, fp32 GEMM is ALU bound) + bool use_fp16 = src0->type == GGML_TYPE_PQ2_0 || src0->type == GGML_TYPE_PTQ1_0 || + (g_ggml_sycl_quant_f16_gemm && ggml_is_quantized(src0->type) && src1_ncols > 32); +#endif + +#if GGML_SYCL_DNNL + // PQ2_0 / PTQ1_0 batches: repack to s4 + group scales and let oneDNN decompress inside the GEMM (XMX), + // instead of writing and reading back a full fp16 copy of the weights. + if (g_ggml_sycl_quant_w8a8 && g_ggml_sycl_enable_dnn && (src0->type == GGML_TYPE_PQ2_0 || src0->type == GGML_TYPE_PTQ1_0) && + src1_ncols > 32 && ggml_is_contiguous(src0) && row_diff == src0->ne[1] && ldc == row_diff && + dst->op_params[0] == GGML_PREC_DEFAULT) { + const bool reordered = src0->extra && ((ggml_tensor_extra_gpu *) src0->extra)->optimized_feature.reorder; + if (ggml_sycl_w8a8_mul_mat(ctx, src0->type, reordered, src0_dd_i, src1_ddf_i, dst_dd_i, row_diff, src1_ncols, + ne10, stream)) { + return; + } + } + if (g_ggml_sycl_quant_s4_gemm && g_ggml_sycl_enable_dnn && (src0->type == GGML_TYPE_PQ2_0 || src0->type == GGML_TYPE_PTQ1_0) && + src1_ncols > 32 && ggml_is_contiguous(src0) && row_diff == src0->ne[1] && ldc == row_diff && + dst->op_params[0] == GGML_PREC_DEFAULT) { + const bool reordered = src0->extra && ((ggml_tensor_extra_gpu *) src0->extra)->optimized_feature.reorder; + ggml_sycl_pool_alloc w4(ctx.pool(), row_diff * ne00 / 2); + ggml_sycl_pool_alloc scales(ctx.pool(), row_diff * ne00 / 128); + ggml_sycl_quant_to_s4(src0->type, reordered, src0_dd_i, w4.get(), scales.get(), row_diff, ne00, stream); + ggml_sycl_pool_alloc src1_as_f16(ctx.pool(), src1_ncols * ne10); + ggml_get_to_fp16_sycl(src1->type, dst)(src1_ddf_i, src1_as_f16.get(), src1_ncols * ne10, stream); + DnnlGemmWrapper::gemm_s4_group(ctx, row_diff, src1_ncols, ne10, src1_as_f16.get(), w4.get(), scales.get(), dst_dd_i, stream); + return; + } #endif #if GGML_SYCL_DNNL && defined(GGML_SYCL_HAS_BF16) @@ -2925,6 +3020,138 @@ inline void ggml_sycl_op_argsort(ggml_backend_sycl_context & ctx, ggml_tensor * main_stream, ctx.device, ctx.pool()); } +// ARC-LAB: two-pass top-k for long rows (the MTP draft picks its tokens from the 248K-entry vocabulary: the one-group +// kernel above took ~550 us per call on the B580, one work-group on one Xe core plus a serial merge by one thread). +// Pass 1: up to 256 groups per row, each thread keeps a sorted top-k of a strided slice in registers, then the group +// merges its 256 lists pairwise in local memory (log2 steps of an O(k) merge). Pass 2: one group per row merges the +// per-group winners the same way. Ties keep the lower index. +namespace topk2 { +constexpr int WG = 256; + +template +static inline void insert(float (&v)[K], int (&ix)[K], const int k, const float val, const int idx) { + if (!(val > v[k - 1] || (val == v[k - 1] && idx < ix[k - 1] && ix[k - 1] >= 0))) { + return; + } + int pos = k - 1; + while (pos > 0 && (val > v[pos - 1] || (val == v[pos - 1] && idx < ix[pos - 1]))) { + v[pos] = v[pos - 1]; + ix[pos] = ix[pos - 1]; + pos--; + } + v[pos] = val; + ix[pos] = idx; +} + +// merge the sorted lists of threads [0, WG) in sv/si (stride k) down to thread 0; every thread calls it +template +static inline void tree_merge(const sycl::nd_item<2> & it, float * sv, int * si, const int k, const int tid) { + for (int s = WG / 2; s > 0; s >>= 1) { + float mv[K]; + int mi[K]; + if (tid < s) { + const float * av = sv + tid * k; + const int * ai = si + tid * k; + const float * bv = sv + (tid + s) * k; + const int * bi = si + (tid + s) * k; + int a = 0, b = 0; + for (int o = 0; o < k; ++o) { + const bool take_a = av[a] > bv[b] || (av[a] == bv[b] && (bi[b] < 0 || (ai[a] >= 0 && ai[a] < bi[b]))); + if (take_a) { + mv[o] = av[a]; + mi[o] = ai[a]; + ++a; + } else { + mv[o] = bv[b]; + mi[o] = bi[b]; + ++b; + } + } + } + it.barrier(sycl::access::fence_space::local_space); + if (tid < s) { + for (int o = 0; o < k; ++o) { + sv[tid * k + o] = mv[o]; + si[tid * k + o] = mi[o]; + } + } + it.barrier(sycl::access::fence_space::local_space); + } +} + +template +static void run(const float * src, int32_t * dst, const int64_t ncols, const int64_t nrows, const int k, + ggml_backend_sycl_context & ctx, dpct::queue_ptr q) { + const int nblk = (int) std::min(256, (ncols + 2047) / 2048); + ggml_sycl_pool_alloc cv(ctx.pool(), (size_t) nrows * nblk * k); + ggml_sycl_pool_alloc ci(ctx.pool(), (size_t) nrows * nblk * k); + float * cvp = cv.get(); + int * cip = ci.get(); + q->submit([&](sycl::handler & cgh) { + sycl::local_accessor sv(sycl::range<1>(WG * k), cgh); + sycl::local_accessor si(sycl::range<1>(WG * k), cgh); + cgh.parallel_for(sycl::nd_range<2>({ (size_t) nrows, (size_t) nblk * WG }, { 1, WG }), [=](sycl::nd_item<2> it) { + const int64_t row = it.get_group(0); + const int blk = it.get_group(1); + const int tid = it.get_local_id(1); + const float * r = src + row * ncols; + float v[K]; + int ix[K]; + for (int i = 0; i < k; ++i) { + v[i] = -FLT_MAX; + ix[i] = -1; + } + for (int64_t c = (int64_t) blk * WG + tid; c < ncols; c += (int64_t) nblk * WG) { + insert(v, ix, k, r[c], (int) c); + } + float * svp = sv.template get_multi_ptr().get(); + int * sip = si.template get_multi_ptr().get(); + for (int i = 0; i < k; ++i) { + svp[tid * k + i] = v[i]; + sip[tid * k + i] = ix[i]; + } + it.barrier(sycl::access::fence_space::local_space); + tree_merge(it, svp, sip, k, tid); + if (tid < k) { + cvp[(row * nblk + blk) * k + tid] = svp[tid]; + cip[(row * nblk + blk) * k + tid] = sip[tid]; + } + }); + }); + q->submit([&](sycl::handler & cgh) { + sycl::local_accessor sv(sycl::range<1>(WG * k), cgh); + sycl::local_accessor si(sycl::range<1>(WG * k), cgh); + cgh.parallel_for(sycl::nd_range<2>({ (size_t) nrows, (size_t) WG }, { 1, WG }), [=](sycl::nd_item<2> it) { + const int64_t row = it.get_group(0); + const int tid = it.get_local_id(1); + float v[K]; + int ix[K]; + for (int i = 0; i < k; ++i) { + v[i] = -FLT_MAX; + ix[i] = -1; + } + for (int c = tid; c < nblk * k; c += WG) { + const int id = cip[row * nblk * k + c]; + if (id >= 0) { + insert(v, ix, k, cvp[row * nblk * k + c], id); + } + } + float * svp = sv.template get_multi_ptr().get(); + int * sip = si.template get_multi_ptr().get(); + for (int i = 0; i < k; ++i) { + svp[tid * k + i] = v[i]; + sip[tid * k + i] = ix[i]; + } + it.barrier(sycl::access::fence_space::local_space); + tree_merge(it, svp, sip, k, tid); + if (tid < k) { + dst[row * k + tid] = sip[tid]; + } + }); + }); +} +} // namespace topk2 + static void ggml_sycl_op_top_k(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; @@ -2946,6 +3173,17 @@ static void ggml_sycl_op_top_k(ggml_backend_sycl_context & ctx, ggml_tensor * ds GGML_ASSERT(k > 0 && k <= 32); GGML_ASSERT(k <= ncols); + static const bool old_topk = getenv("GGML_SYCL_TOPK_OLD") != nullptr; // ARC-LAB A/B switch + if (!old_topk && ncols >= 8192) { + if (k <= 8) { + topk2::run<8>(src0_dd, dst_dd, ncols, nrows, k, ctx, main_stream); + } else if (k <= 16) { + topk2::run<16>(src0_dd, dst_dd, ncols, nrows, k, ctx, main_stream); + } else { + topk2::run<32>(src0_dd, dst_dd, ncols, nrows, k, ctx, main_stream); + } + return; + } top_k_f32_sycl(src0_dd, dst_dd, ncols, nrows, k, main_stream); } @@ -3852,6 +4090,8 @@ inline bool ggml_sycl_supports_reorder_mmvq(enum ggml_type type) { switch (type) { case GGML_TYPE_Q1_0: case GGML_TYPE_Q4_0: + case GGML_TYPE_PQ2_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q8_0: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: @@ -4044,6 +4284,80 @@ static bool reorder_qw_q4_0(uint8_t * data_device, const int ncols, const int nr return true; } +static bool reorder_qw_pq2_0(uint8_t * data_device, const int ncols, const int nrows, size_t size, size_t offset, + dpct::queue_ptr stream) { + sycl_reorder_temp_buffer tmp(stream, size); + if (!tmp) { + GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, size); + return false; + } + uint8_t * tmp_buf = static_cast(tmp.ptr); + + sycl::event copy_event; + SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, size))); + if (!g_ggml_sycl_use_async_mem_op) { + copy_event.wait(); + } + + GGML_ASSERT((size % sizeof(block_pq2_0) == 0)); + GGML_ASSERT((offset % sizeof(block_pq2_0) == 0)); + int offset_blks = offset / sizeof(block_pq2_0); + auto qs_ptr = data_device + offset_blks * QK_PQ2_0 / 4; + auto d_ptr = (sycl::half *) (qs_ptr + ncols * nrows / 4) + offset_blks; + + auto reorder_event = stream->parallel_for( + size / sizeof(block_pq2_0), + [=](auto i) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + const block_pq2_0 * x = (const block_pq2_0 *) tmp_buf; + const int ib = i; + + for (int j = 0; j < QK_PQ2_0 / 4; j++) { + *(qs_ptr + ib * QK_PQ2_0 / 4 + j) = x[ib].qs[j]; + } + *(d_ptr + ib) = x[ib].d; + }); + if (!g_ggml_sycl_use_async_mem_op) { + reorder_event.wait_and_throw(); + } + return true; +} + +static bool reorder_qw_ptq1_0(uint8_t * data_device, size_t size, dpct::queue_ptr stream) { + sycl_reorder_temp_buffer tmp(stream, size); + if (!tmp) { + GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, size); + return false; + } + uint8_t * tmp_buf = static_cast(tmp.ptr); + + sycl::event copy_event; + SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, size))); + if (!g_ggml_sycl_use_async_mem_op) { + copy_event.wait(); + } + + GGML_ASSERT((size % sizeof(block_ptq1_0) == 0)); + const int64_t nb = size / sizeof(block_ptq1_0); + auto qs_ptr = data_device; + auto qh_ptr = data_device + nb * 24; + auto d_ptr = (sycl::half *) (data_device + nb * 26); + + auto reorder_event = stream->parallel_for(nb, [=](auto i) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + const block_ptq1_0 * x = (const block_ptq1_0 *) tmp_buf; + const int64_t ib = i; + for (int j = 0; j < 24; j++) { + qs_ptr[ib * 24 + j] = x[ib].qs[j]; + } + qh_ptr[ib * 2 + 0] = x[ib].qh[0]; + qh_ptr[ib * 2 + 1] = x[ib].qh[1]; + d_ptr[ib] = x[ib].d; + }); + if (!g_ggml_sycl_use_async_mem_op) { + reorder_event.wait_and_throw(); + } + return true; +} + static bool reorder_qw_q8_0(uint8_t * data_device, const int ncols, const int nrows, size_t size, size_t offset, dpct::queue_ptr stream) { sycl_reorder_temp_buffer tmp(stream, size); @@ -4488,6 +4802,10 @@ static bool reorder_qw(const ggml_tensor * src0, dpct::queue_ptr stream) { return reorder_qw_q4_0(data_device, ncols, nrows, size, 0, stream); case GGML_TYPE_Q8_0: return reorder_qw_q8_0(data_device, ncols, nrows, size, 0, stream); + case GGML_TYPE_PQ2_0: + return reorder_qw_pq2_0(data_device, ncols, nrows, size, 0, stream); + case GGML_TYPE_PTQ1_0: + return reorder_qw_ptq1_0(data_device, size, stream); case GGML_TYPE_Q2_K: return reorder_qw_q2_k(data_device, size, 0, stream); case GGML_TYPE_Q3_K: @@ -4523,6 +4841,15 @@ static void opt_for_reorder(ggml_backend_sycl_context * ctx, const ggml_tensor * return; // Skip permutations and already reordered tensors } + // PQ2_0/PTQ1_0 on 16-wide DPAS devices belong to the XMX layout (ggml_sycl_pq2_xmx_use) unless + // ggml_sycl_xmx_disabled; it rewrites the same bytes in place, so reordering them as well would leave one path + // reading the other's layout + if ((src0->type == GGML_TYPE_PQ2_0 || src0->type == GGML_TYPE_PTQ1_0) && + (extra->optimized_feature.xmx_pq2 || + (!ggml_sycl_xmx_disabled(ctx->device) && ggml_sycl_device_has_dpas16(ctx->device)))) { + return; + } + switch (mm_algorithm) { case mul_mat_algo::DMMV: if (!ggml_sycl_supports_reorder_dmmv(src0->type)) { @@ -4596,7 +4923,12 @@ static bool ggml_sycl_pq2_xmx_use(ggml_backend_sycl_context & ctx, const ggml_te if (extra && extra->optimized_feature.xmx_pq2) { return true; } - if (!g_ggml_sycl_enable_optimize || ggml_sycl_xmx_disabled() || !ggml_sycl_device_has_dpas16(ctx.device)) { + // already in another in-place layout (MMVQ reorder or the TernSYCL 2-bit repack): rewriting it would corrupt it + if (extra && (extra->optimized_feature.reorder || extra->optimized_feature.t2)) { + return false; + } + if (!g_ggml_sycl_enable_optimize || ggml_sycl_xmx_disabled(ctx.device) || + !ggml_sycl_device_has_dpas16(ctx.device)) { return false; } // op offload refills COMPUTE buffers from host memory every time, so an in-place layout there goes stale @@ -4624,9 +4956,123 @@ static bool ggml_sycl_pq2_xmx_use(ggml_backend_sycl_context & ctx, const ggml_te return true; } +// ARC-LAB: small-output bf16 mat-mul for 2..8 activation columns (speculative-decoding verify batches through Bonsai's +// delta-net gate projections, 48 x 5120). The generic path converted the whole bf16 matrix and ran a GEMM per call. +// One 256-wide work-group per output row reads the row once (8 bf16 per load) and dots it with every column. +static bool ggml_sycl_bf16_small_ok(const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst) { + static const bool off = getenv("GGML_SYCL_BF16_ROWSPLIT_OFF") != nullptr; + return !off && src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 && + src0->ne[1] <= 512 && src1->ne[1] >= 2 && src1->ne[1] <= 8 && src0->ne[0] % 8 == 0 && + src1->ne[2] * src1->ne[3] == 1 && src0->ne[2] * src0->ne[3] == 1 && + ggml_is_contiguous(src0) && ggml_is_contiguous_rows(src1) && ggml_is_contiguous(dst) && + ((uintptr_t) src0->data % 16) == 0 && !ggml_backend_buffer_is_sycl_split(src0->buffer); +} + +static void ggml_sycl_mul_mat_bf16_small(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + constexpr int WG = 256; + const int ncols_x = src0->ne[0]; + const int nrows = src0->ne[1]; + const int ncols_y = src1->ne[1]; + const int64_t sy = src1->nb[1] / sizeof(float); + const int64_t sd = dst->nb[1] / sizeof(float); + const uint16_t * x = (const uint16_t *) src0->data; + const float * y = (const float *) src1->data; + float * d = (float *) dst->data; + ctx.stream()->parallel_for( + sycl::nd_range<1>(sycl::range<1>((size_t) nrows * WG), sycl::range<1>(WG)), + [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + const int row = it.get_group(0); + const int tid = it.get_local_id(0); + const uint16_t * xr = x + (size_t) row * ncols_x; + float sum[8] = { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f }; + for (int i = tid * 8; i < ncols_x; i += WG * 8) { + const sycl::vec w = *(const sycl::vec *) (xr + i); + float wf[8]; +#pragma unroll + for (int j = 0; j < 8; ++j) { + wf[j] = sycl::bit_cast((uint32_t) w[j] << 16); + } +#pragma unroll + for (int c = 0; c < 8; ++c) { + if (c < ncols_y) { + const float * yc = y + c * sy + i; +#pragma unroll + for (int j = 0; j < 8; ++j) { + sum[c] += wf[j] * yc[j]; + } + } + } + } +#pragma unroll + for (int c = 0; c < 8; ++c) { + if (c < ncols_y) { + const float v = sycl::reduce_over_group(it.get_group(), sum[c], sycl::plus()); + if (tid == 0) { + d[c * sd + row] = v; + } + } + } + }); +} + +// ARC-LAB: PTQ1_0 weights chosen by GGML_SYCL_PTQ1_T2 are repacked in place (their buffer was sized for it) on first use +// and from then on only this path may read them. +static bool ggml_sycl_t2_tensor(const ggml_tensor * w) { + if ((w->type != GGML_TYPE_PTQ1_0 && w->type != GGML_TYPE_PQ2_0) || w->ne[2] != 1 || w->ne[3] != 1 || !w->extra) { + return false; + } + const auto * extra = static_cast(w->extra); + return extra->optimized_feature.t2 || ggml_sycl_t2_wants(w->name, w->ne[0], w->ne[1]); +} + +static bool ggml_sycl_t2_mul_mat_try(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, + ggml_tensor * dst) { + if (!ggml_sycl_t2_tensor(src0)) { + return false; + } + auto * extra = static_cast(src0->extra); + GGML_ASSERT(!extra->optimized_feature.reorder && "PTQ1_0 T2 tensor was reordered"); + const int64_t K = src0->ne[0], N = src0->ne[1]; + GGML_ASSERT(src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 && src1->ne[0] == K && src1->nb[0] == sizeof(float)); + GGML_ASSERT(ggml_is_contiguous(dst) && dst->ne[0] == N); + GGML_ASSERT(src1->ne[3] == 1 && (src1->ne[2] == 1 || src1->nb[2] == src1->ne[1] * src1->nb[1])); + dpct::queue_ptr stream = ctx.stream(); + if (!extra->optimized_feature.t2) { + if (!ggml_sycl_t2_repack(*stream, src0->data, K, N, src0->type == GGML_TYPE_PQ2_0)) { + GGML_ABORT("PQ2_0 tensor %s uses code 3 (+2), which the 2-bit XMX layout cannot hold: unset GGML_SYCL_PTQ1_T2", + src0->name); + } + extra->optimized_feature.t2 = true; + } + // tokens: ne[1] x ne[2] rows of K floats (row stride nb[1]; planes back to back) + const int64_t M = src1->ne[1] * src1->ne[2]; + // ARC-LAB: large prompt batches through oneDNN's int8 GEMM (faster than GemmMT there), GGML_SYCL_T2_W8A8_MIN (0 = off) + static const int64_t w8a8_env = getenv("GGML_SYCL_T2_W8A8_MIN") ? atoll(getenv("GGML_SYCL_T2_W8A8_MIN")) : 0; + const int64_t w8a8_min = ggml_sycl_knob("GGML_SYCL_T2_W8A8_MIN", (int) w8a8_env); + if (w8a8_min > 0 && M >= w8a8_min && g_ggml_sycl_enable_dnn && src1->nb[1] == (size_t) K * sizeof(float) && + ggml_sycl_w8a8_mul_mat_t2(ctx, src0->data, (const float *) src1->data, K, (float *) dst->data, N, M, K, stream)) { + return true; + } + ggml_sycl_t2_mul_mat(*stream, src0->data, (const float *) src1->data, src1->nb[1] / sizeof(float), (float *) dst->data, + M, N, K, src1->view_src && src1->view_offs == 0 ? src1->view_src : src1); + return true; +} + +// ARC-LAB: largest batch run as MMVQ column chunks instead of dequantize-all + GEMM (GGML_SYCL_MMVQ_CHUNK_MAX, default 32). +// A chunk re-reads the quantized weights; the GEMM path writes and reads the whole matrix as f16 per call, so chunks win +// for speculative-verify sized batches (Gemma 4 12B q4_0: n-gram drafts of 33+ tokens were ~94% dequantization). +static int64_t mmvq_chunk_max() { + static const int64_t v = getenv("GGML_SYCL_MMVQ_CHUNK_MAX") ? atoll(getenv("GGML_SYCL_MMVQ_CHUNK_MAX")) : 4 * MMVQ_MAX_BATCH_SIZE; + return v; +} + static void ggml_sycl_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); + if (dst->op == GGML_OP_MUL_MAT && ggml_sycl_t2_mul_mat_try(ctx, src0, src1, dst)) { + return; + } + // Handle HADAMARAD hint given from further up the pipeline and pass it to the correct // kernel. // @@ -4664,6 +5110,45 @@ static void ggml_sycl_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tensor min_compute_capability = ggml_sycl_info().devices[ctx.device].cc; } + // ARC-LAB: reordered q4_0 with 5..64 columns on XMX (weights streamed once for all columns), GGML_SYCL_Q4_0_DPAS=1 + { + static const bool q4d_on = getenv("GGML_SYCL_Q4_0_DPAS") && atoi(getenv("GGML_SYCL_Q4_0_DPAS")) != 0 && + ggml_sycl_device_is_xe2(); + static const int q4d_min = getenv("GGML_SYCL_Q4_0_DPAS_MIN") ? atoi(getenv("GGML_SYCL_Q4_0_DPAS_MIN")) : 5; + static const int q4d_max = getenv("GGML_SYCL_Q4_0_DPAS_MAX") ? atoi(getenv("GGML_SYCL_Q4_0_DPAS_MAX")) : 256; + const auto * ex = static_cast(src0->extra); + if (q4d_on && !split && src0->type == GGML_TYPE_Q4_0 && ex && ex->optimized_feature.reorder && + ggml_is_contiguous(src0) && src0->ne[2] == 1 && src0->ne[3] == 1 && src1->type == GGML_TYPE_F32 && + dst->type == GGML_TYPE_F32 && src1->ne[2] == 1 && src1->ne[3] == 1 && src1->nb[0] == sizeof(float) && + src1->ne[1] >= q4d_min && src1->ne[1] <= q4d_max && ggml_is_contiguous(dst) && + ggml_sycl_q4_0_dpas_gemm(ctx, src0->data, (const float *) src1->data, src1->nb[1] / (int64_t) sizeof(float), + (float *) dst->data, (int) src0->ne[1], (int) src0->ne[0], (int) src1->ne[1], ctx.stream())) { + return; + } + } + + // Small batches (e.g. speculative verify) on quantized weights: run MMVQ on chunks of columns + // instead of dequantizing all of src0 to fp32 (the output matrix is ~5 GB as fp32). + if (!split && ggml_is_quantized(src0->type) && ggml_is_contiguous(src0) && src0->ne[2] == 1 && src0->ne[3] == 1 && + src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 && src1->ne[2] == 1 && src1->ne[3] == 1 && + src1->ne[1] > MMVQ_MAX_BATCH_SIZE && src1->nb[0] == sizeof(float) && + (src1->ne[1] <= mmvq_chunk_max() || (size_t) src0->ne[0] * src0->ne[1] * sizeof(float) > (1ull << 30))) { + for (int64_t c0 = 0; c0 < src1->ne[1]; c0 += MMVQ_MAX_BATCH_SIZE) { + const int64_t nc = std::min(MMVQ_MAX_BATCH_SIZE, src1->ne[1] - c0); + ggml_tensor src1_c = *src1; + ggml_tensor dst_c = *dst; + src1_c.ne[1] = nc; + src1_c.data = (char *) src1->data + c0 * src1->nb[1]; + src1_c.nb[2] = src1_c.nb[3] = nc * src1->nb[1]; + dst_c.ne[1] = nc; + dst_c.data = (char *) dst->data + c0 * dst->nb[1]; + dst_c.nb[2] = dst_c.nb[3] = nc * dst->nb[1]; + dst_c.src[1] = &src1_c; + ggml_sycl_mul_mat(ctx, src0, &src1_c, &dst_c); + } + return; + } + // check data types and tensor shapes for custom matrix multiplication kernels: bool use_dequantize_mul_mat_vec = can_use_dequantize_mul_mat_vec(src0, src1, dst); @@ -4714,6 +5199,8 @@ static void ggml_sycl_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tensor } else if (!split && src0->type == GGML_TYPE_F16 && !ggml_is_transposed(src0) && !ggml_is_transposed(src1) && src1->ne[2] * src1->ne[3] > 1) { // KQ + KQV multi-batch ggml_sycl_mul_mat_batched_sycl(ctx, src0, src1, dst); + } else if (!split && ggml_sycl_bf16_small_ok(src0, src1, dst)) { + ggml_sycl_mul_mat_bf16_small(ctx, src0, src1, dst); } else if (use_dequantize_mul_mat_vec) { opt_for_reorder(&ctx, src0, src1, dst, mul_mat_algo::DMMV); ggml_sycl_op_mul_mat(ctx, src0, src1, dst, ggml_sycl_op_dequantize_mul_mat_vec); @@ -4721,7 +5208,12 @@ static void ggml_sycl_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tensor opt_for_reorder(&ctx, src0, src1, dst, mul_mat_algo::MMVQ); ggml_tensor_extra_gpu * extra = static_cast(src0->extra); if (extra && extra->optimized_feature.reorder) { - ggml_sycl_op_mul_mat(ctx, src0, src1, dst, ggml_sycl_op_mul_mat_vec_q); + if (src0->type == GGML_TYPE_PTQ1_0 && ggml_sycl_ptq1_0_ncols_dec_enabled() && src1->ne[1] >= 2 && src1->ne[1] <= 8) { + // ARC-LAB: decode-once PTQ1_0 multi-column kernel reads the chunk-interleaved activation layout + ggml_sycl_op_mul_mat(ctx, src0, src1, dst, ggml_sycl_op_mul_mat_vec_q); + } else { + ggml_sycl_op_mul_mat(ctx, src0, src1, dst, ggml_sycl_op_mul_mat_vec_q); + } } else { ggml_sycl_op_mul_mat(ctx, src0, src1, dst, ggml_sycl_op_mul_mat_vec_q); } @@ -4735,9 +5227,13 @@ static void ggml_sycl_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tensor // Fused dense-FFN mat-vec for the {mul_mat(gate), mul_mat(up), GLU} subgraph at node_idx. // Returns false if it declined, in which case the caller runs the three nodes normally. static bool ggml_sycl_mul_mat_glu_mmvq_fused(ggml_backend_sycl_context & ctx, ggml_cgraph * cgraph, int node_idx) { - if (!ggml_sycl_can_fuse(cgraph, node_idx, { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT, GGML_OP_GLU }, {})) { + static const bool fuse_off = getenv("GGML_SYCL_GLU_FUSE_OFF") != nullptr; // ARC-LAB A/B switch + if (fuse_off || !ggml_sycl_can_fuse(cgraph, node_idx, { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT, GGML_OP_GLU }, {})) { return false; } + if (ggml_sycl_t2_tensor(cgraph->nodes[node_idx]->src[0]) || ggml_sycl_t2_tensor(cgraph->nodes[node_idx + 1]->src[0])) { + return false; // ARC-LAB: 2-bit repacked weights (ptq1-t2.cpp) run unfused + } ggml_tensor * glu = cgraph->nodes[node_idx + 2]; ggml_tensor * gate = glu->src[0]; @@ -4783,8 +5279,15 @@ static bool ggml_sycl_mul_mat_glu_mmvq_fused(ggml_backend_sycl_context & ctx, gg (size_t) ne11 * src1_padded_cols * sizeof(block_q8_1) / QK8_1); char * src1_ddq = src1_q8_alloc.get(); - quantize_row_q8_1_sycl((const float *) act->data, src1_ddq, (int) ne00, (int) ne11, - src1_padded_cols, stream); + if (wu->type == GGML_TYPE_PTQ1_0 && ggml_sycl_ptq1_0_ncols_dec_enabled() && ne11 <= 8 && + !(ne11 == 1 && ggml_sycl_ptq1_0_glu_n1_enabled())) { + // ARC-LAB: the decode-once PTQ1_0 fused kernel reads the chunk-interleaved activation layout + quantize_row_q8_1_sycl((const float *) act->data, src1_ddq, (int) ne00, + (int) ne11, src1_padded_cols, stream); + } else { + quantize_row_q8_1_sycl((const float *) act->data, src1_ddq, (int) ne00, (int) ne11, + src1_padded_cols, stream); + } return ggml_sycl_mul_mat_vec_q_glu_reorder(wu->type, ggml_get_glu_op(glu), wu->data, wg->data, src1_ddq, (float *) glu->data, (int) ne00, (int) wu->ne[1], (int) ne11, @@ -5616,6 +6119,19 @@ static void ggml_backend_sycl_free(ggml_backend_t backend) { delete backend; } +// ARC-LAB: pinned upload ring for ggml_backend_sycl_set_tensor_async (GGML_SYCL_PINNED_UPLOAD) +static char * g_upload_ring = nullptr; +static const size_t g_upload_ring_bytes = 32u << 20; +static size_t g_upload_off = 0; +// GGML_SYCL_HOST_TIMING=1 also prints the uploads of the first decode-sized graphs (name, bytes) +static void ggml_sycl_log_upload(const ggml_tensor * t, size_t size) { + static const bool on = getenv("GGML_SYCL_HOST_TIMING") && atoi(getenv("GGML_SYCL_HOST_TIMING")); + static int n = 0; + if (on && n < 60) { + ++n; + fprintf(stderr, "upload: %-28s %zu bytes\n", t->name, size); + } +} static void ggml_backend_sycl_set_tensor_async(ggml_backend_t backend, ggml_tensor *tensor, const void *data, size_t offset, @@ -5628,6 +6144,27 @@ static void ggml_backend_sycl_set_tensor_async(ggml_backend_t backend, GGML_ASSERT(buf->buft == ggml_backend_sycl_buffer_type(sycl_ctx->device) && "unsupported buffer type"); const queue_ptr stream = sycl_ctx->stream(sycl_ctx->device, 0); + // ARC-LAB: GGML_SYCL_PINNED_UPLOAD=1 stages small per-token inputs (token embedding, positions, masks, state ids) in a + // pinned host ring first: a copy straight from pageable memory goes through the driver's staging path. The ring + // restarts at each synchronize (the in-order queue has consumed every staged copy by then). + if (ggml_sycl_knob("GGML_SYCL_PINNED_UPLOAD", 0) && size <= g_upload_ring_bytes / 4) { + if (!g_upload_ring) { + g_upload_ring = (char *) sycl::malloc_host(g_upload_ring_bytes, *stream); + } + if (g_upload_ring) { + if (g_upload_off + size > g_upload_ring_bytes) { + stream->wait(); + g_upload_off = 0; + } + char * staged = g_upload_ring + g_upload_off; + memcpy(staged, data, size); + g_upload_off = (g_upload_off + size + 255) & ~(size_t) 255; + SYCL_CHECK(CHECK_TRY_ERROR((stream)->memcpy((char *) tensor->data + offset, staged, size))); + ggml_sycl_log_upload(tensor, size); + return; + } + } + ggml_sycl_log_upload(tensor, size); SYCL_CHECK(CHECK_TRY_ERROR( (stream)->memcpy((char *)tensor->data + offset, data, size))); } @@ -5688,11 +6225,58 @@ catch (sycl::exception const &exc) { std::exit(1); } +// ARC-LAB: GGML_SYCL_HOST_TIMING=1 -> host-side token timing without a tracer (tracers slow the host 2x and invent gaps). +// Per graph: between = host time since the previous synchronize returned (sampling, graph build, logits copy), +// enqueue = time to submit every kernel, wait = synchronize after the last submit. Plus host enqueue time per op type. +// Printed every 64 graphs with more than 500 nodes (decode graphs), then reset. +struct sycl_host_timing { + bool on = getenv("GGML_SYCL_HOST_TIMING") && atoi(getenv("GGML_SYCL_HOST_TIMING")); + using clk = std::chrono::steady_clock; + clk::time_point last_sync_end{}, t_in{}, t_enq{}; + bool in_big = false; + int graphs = 0; + double between = 0, enqueue = 0, wait = 0, waits_mid = 0; + long nodes = 0, syncs = 0; + std::map> per_op; + void report() { + if (graphs == 0) return; + fprintf(stderr, "host-timing: %d graphs, per graph: between %.3f ms, enqueue %.3f ms (%ld nodes), wait %.3f ms, other syncs %.1f\n", + graphs, between / graphs, enqueue / graphs, nodes / graphs, wait / graphs, (double) syncs / graphs); + fprintf(stderr, "host-timing: set_tensor (sync uploads) per graph: %.3f ms, %.1f calls, %.0f bytes\n", g_st_ms / graphs, + (double) g_st_calls / graphs, (double) g_st_bytes / graphs); + g_st_ms = 0; g_st_calls = 0; g_st_bytes = 0; + std::vector> v; + for (auto & kv : per_op) v.push_back({ kv.second.first, kv.first }); + std::sort(v.rbegin(), v.rend()); + for (size_t i = 0; i < v.size() && i < 12; ++i) { + fprintf(stderr, "host-timing: %-22s %.3f ms/graph (%ld calls/graph)\n", v[i].second.c_str(), v[i].first / graphs, + per_op[v[i].second].second / graphs); + } + graphs = 0; between = enqueue = wait = 0; nodes = 0; syncs = 0; per_op.clear(); + } +}; +static sycl_host_timing g_ht; + static void ggml_backend_sycl_synchronize(ggml_backend_t backend) try { GGML_SYCL_DEBUG("[SYCL] call %s\n", __func__); ggml_backend_sycl_context * sycl_ctx = (ggml_backend_sycl_context *)backend->context; const queue_ptr stream = sycl_ctx->stream(sycl_ctx->device, 0); + const auto ht_t0 = sycl_host_timing::clk::now(); SYCL_CHECK(CHECK_TRY_ERROR((stream)->wait())); + g_upload_off = 0; + if (g_ht.on) { + const auto t1 = sycl_host_timing::clk::now(); + if (g_ht.in_big) { // first synchronize after a decode graph + g_ht.wait += std::chrono::duration(t1 - ht_t0).count(); + g_ht.in_big = false; + if (++g_ht.graphs == 64) { + g_ht.report(); + } + } else { + g_ht.syncs++; + } + g_ht.last_sync_end = t1; + } GGML_UNUSED(backend); } @@ -5780,11 +6364,834 @@ static int ggml_sycl_try_gdn_cache_fusion(const ggml_cgraph * cgraph, int node_i return skip; } +// GET_ROWS that gathers a recurrent state row feeding only GATED_DELTA_NET src[5] (possibly through a RESHAPE): +// skip it and let the kernel read the cache row directly. Single sequence only. GGML_SYCL_GDN_GATHER=0 disables. +static bool ggml_sycl_try_gdn_gather_skip(ggml_backend_sycl_context & ctx, const ggml_cgraph * cgraph, int node_idx) { + static const bool enabled = ggml_sycl_get_env("GGML_SYCL_GDN_GATHER", 1) != 0; + if (!enabled) { + return false; + } + const ggml_tensor * gr = cgraph->nodes[node_idx]; + if (gr->op != GGML_OP_GET_ROWS || gr->type != GGML_TYPE_F32 || (gr->flags & GGML_TENSOR_FLAG_OUTPUT) || + !ggml_is_contiguous(gr)) { + return false; + } + const ggml_tensor * cache = gr->src[0]; + const ggml_tensor * ids = gr->src[1]; + if (cache->type != GGML_TYPE_F32 || ids->type != GGML_TYPE_I32 || !cache->data || !ids->data || + (ids->buffer && ggml_backend_buffer_is_host(ids->buffer)) || cache->nb[0] != sizeof(float) || + cache->nb[1] % sizeof(float) != 0 || !ggml_is_contiguous(ids) || ggml_nelements(ids) != 1 || + gr->ne[1] != 1 || gr->ne[2] != 1 || gr->ne[3] != 1 || gr->ne[0] != cache->ne[0]) { + return false; + } + if (ggml_node_get_use_count(cgraph, node_idx) != 1) { + return false; + } + const ggml_tensor * cur = gr; + for (int j = node_idx + 1; j < cgraph->n_nodes; ++j) { + const ggml_tensor * n = cgraph->nodes[j]; + if (n->op == GGML_OP_GATED_DELTA_NET && n->src[5] == cur) { + const ggml_tensor * v = n->src[2]; + const int64_t D = v->ne[0] * v->ne[0] * v->ne[1]; + if (gr->ne[0] != D || v->ne[3] != 1 || ggml_nelements(cur) != D) { + return false; + } + ggml_sycl_gdn_gather_set(ctx, n, (const float *) cache->data, (const int32_t *) ids->data, + (int64_t) (cache->nb[1] / sizeof(float))); + return true; + } + if (n->op == GGML_OP_RESHAPE && n->src[0] == cur) { + if (ggml_node_get_use_count(cgraph, j) != 1) { + return false; + } + cur = n; + continue; + } + for (int s = 0; s < GGML_MAX_SRC; ++s) { + if (n->src[s] == cur || (n->view_src != nullptr && n->view_src == gr)) { + return false; + } + } + } + return false; +} + +// ARC-LAB: single-column PTQ1_0 mat-vecs that read the same activation (GDN qkv + z, attention q + k + v) in ONE launch +// with ONE quantization (in-model each launch costs ~10 us beyond its weight bytes). The graph interleaves them with +// other nodes, so later members are pulled forward to node i: only when their output overlaps no tensor that a node in +// between reads or writes (the allocator reuses dead memory). GGML_SYCL_PTQ1_MULTI=0 turns it off. +static bool ggml_sycl_ptq1_multi_eligible(ggml_backend_sycl_context & ctx, const ggml_tensor * mm) { + if (mm->op != GGML_OP_MUL_MAT || (mm->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) { + return false; + } + const ggml_tensor * w = mm->src[0]; + const ggml_tensor * act = mm->src[1]; + if (w->type != GGML_TYPE_PTQ1_0 || act->type != GGML_TYPE_F32 || mm->type != GGML_TYPE_F32) { + return false; + } + // 1 column: pairs kernel (rows % 4); 2..8 (MTP verify): decode-once kernel (rows % 2) + if (act->ne[1] < 1 || act->ne[1] > 8 || act->ne[2] != 1 || act->ne[3] != 1 || w->ne[2] != 1 || w->ne[3] != 1 || + w->ne[0] % QK_PTQ1_0 != 0 || w->ne[1] % 4 != 0 || mm->ne[0] != w->ne[1]) { + return false; + } + if (act->ne[1] > 1 && !ggml_sycl_ptq1_0_ncols_dec_enabled()) { + return false; + } + static const bool ncols_on = !getenv("GGML_SYCL_PTQ1_MULTI_NCOLS") || atoi(getenv("GGML_SYCL_PTQ1_MULTI_NCOLS")) != 0; + if (act->ne[1] > 1 && !ncols_on) { + return false; + } + if (!ggml_is_contiguous(w) || !ggml_is_contiguous(act) || !ggml_is_contiguous(mm)) { + return false; + } + if (ggml_backend_buffer_is_sycl_split(w->buffer) || g_ggml_sycl_prioritize_dmmv || !should_reorder_tensor(ctx, mm)) { + return false; + } + const auto * extra = static_cast(w->extra); + return extra && extra->optimized_feature.reorder; // reordered on its first unfused run +} + +static bool ggml_sycl_tensors_overlap(const ggml_tensor * a, const ggml_tensor * b) { + if (!a || !b || !a->data || !b->data) { + return false; + } + const char * a0 = (const char *) a->data; + const char * b0 = (const char *) b->data; + return a0 < b0 + ggml_nbytes(b) && b0 < a0 + ggml_nbytes(a); +} + +// A member whose output memory is still in use by nodes in between (GDN z: the allocator hands its buffer to the conv / +// delta-net intermediates) is computed into a private scratch slot instead and copied to its tensor at its own node. +static constexpr int PTQ1_MULTI_SLOTS = 4; +static constexpr size_t PTQ1_MULTI_SLOT_BYTES = 256 * 1024; + +static float * ggml_sycl_ptq1_multi_slot(queue_ptr stream) { + static std::map scratch; // per device + static int next = 0; + const int dev = ggml_sycl_get_device(); + float *& buf = scratch[dev]; + if (!buf) { + buf = (float *) sycl::malloc_device(PTQ1_MULTI_SLOTS * PTQ1_MULTI_SLOT_BYTES, *stream); + if (!buf) { + return nullptr; + } + } + next = (next + 1) % PTQ1_MULTI_SLOTS; + return buf + next * (PTQ1_MULTI_SLOT_BYTES / sizeof(float)); +} + +static bool ggml_sycl_try_ptq1_multi(ggml_backend_sycl_context & ctx, ggml_cgraph * cgraph, int i, + std::vector & done, std::vector & pending) { + static const bool on = !getenv("GGML_SYCL_PTQ1_MULTI") || atoi(getenv("GGML_SYCL_PTQ1_MULTI")) != 0; + static int log = getenv("GGML_SYCL_PTQ1_MULTI_LOG") ? atoi(getenv("GGML_SYCL_PTQ1_MULTI_LOG")) : 0; + ggml_tensor * mm0 = cgraph->nodes[i]; + if (!on || !ggml_sycl_ptq1_multi_eligible(ctx, mm0)) { + return false; + } + const ggml_tensor * act = mm0->src[1]; + int members[3] = { i, -1, -1 }; + bool staged[3] = { false, false, false }; + int nm = 1; + const int jend = std::min(cgraph->n_nodes, i + 512); + for (int j = i + 1; j < jend && nm < 3; ++j) { + ggml_tensor * mj = cgraph->nodes[j]; + if (done[j] || mj->op != GGML_OP_MUL_MAT || mj->src[1] != act || mj->src[0]->ne[0] != mm0->src[0]->ne[0] || + !ggml_sycl_ptq1_multi_eligible(ctx, mj)) { + continue; + } + bool safe = true; + for (int k = i + 1; k < j && safe; ++k) { + const ggml_tensor * nk = cgraph->nodes[k]; + bool member = false; + for (int m = 1; m < nm; ++m) { + member = member || members[m] == k; + } + if (member) { + continue; + } + safe = !ggml_sycl_tensors_overlap(mj, nk); + for (int s = 0; s < GGML_MAX_SRC && safe; ++s) { + safe = !ggml_sycl_tensors_overlap(mj, nk->src[s]); + } + } + for (int m = 0; m < nm && safe; ++m) { + safe = !ggml_sycl_tensors_overlap(mj, cgraph->nodes[members[m]]) && !ggml_sycl_tensors_overlap(mj, act); + } + if (safe || ggml_nbytes(mj) <= PTQ1_MULTI_SLOT_BYTES) { + staged[nm] = !safe; + members[nm++] = j; + } + } + if (nm < 2) { + return false; + } + + const int64_t ne00 = mm0->src[0]->ne[0]; + const int ne11 = (int) act->ne[1]; + const int src1_padded_cols = GGML_PAD((int) ne00, MATRIX_ROW_PADDING); + const int stride_y_bytes = src1_padded_cols * (int) sizeof(block_q8_1) / QK8_1; + const queue_ptr stream = ctx.stream(); + ggml_sycl_pool_alloc q8(ctx.pool(), (size_t) ne11 * stride_y_bytes); + if (ne11 == 1) { + quantize_row_q8_1_sycl((const float *) act->data, q8.get(), (int) ne00, 1, + src1_padded_cols, stream); + } else { + quantize_row_q8_1_sycl((const float *) act->data, q8.get(), (int) ne00, ne11, + src1_padded_cols, stream); + } + const void * vx[3]; + float * dst[3]; + int nrows[3]; + for (int m = 0; m < nm; ++m) { + ggml_tensor * t = cgraph->nodes[members[m]]; + vx[m] = t->src[0]->data; + dst[m] = (float *) t->data; + nrows[m] = (int) t->src[0]->ne[1]; + if (staged[m]) { + dst[m] = ggml_sycl_ptq1_multi_slot(stream); + if (!dst[m]) { + return false; + } + } + } + const bool launched = ne11 == 1 ? + ggml_sycl_ptq1_0_mul_mat_vec_multi(vx, dst, nrows, nm, q8.get(), (int) ne00, stream) : + ggml_sycl_ptq1_0_mul_mat_vec_multi_ncols(vx, dst, nrows, nm, q8.get(), (int) ne00, ne11, stride_y_bytes, stream); + if (!launched) { + return false; + } + if (log > 0) { + GGML_LOG_INFO("[ptq1-multi] %s", mm0->name); + for (int m = 1; m < nm; ++m) { + GGML_LOG_INFO(" + %s (+%d%s)", cgraph->nodes[members[m]]->name, members[m] - i, staged[m] ? ", staged" : ""); + } + GGML_LOG_INFO("\n"); + --log; + } + for (int m = 1; m < nm; ++m) { + done[members[m]] = 1; + if (staged[m]) { + pending[members[m]] = dst[m]; // copied into the tensor when the loop reaches it + } + } + return true; +} + +// ARC-LAB: same-input T2 mat-vecs (q+k+v, GDN qkv+z, FFN gate+up; <= 8 tokens) in one launch (GGML_SYCL_T2_MULTI, +// default on). Grouping and safety as ggml_sycl_try_ptq1_multi: a later member computed early must not overlap any node +// in between, else its result is staged in a slot and copied when the loop reaches it. +static bool ggml_sycl_t2_multi_eligible(ggml_backend_sycl_context & ctx, const ggml_tensor * mm) { + if (mm->op != GGML_OP_MUL_MAT || (mm->flags & GGML_TENSOR_FLAG_COMPUTE) == 0 || !ggml_sycl_t2_tensor(mm->src[0])) { + return false; + } + auto * extra = static_cast(mm->src[0]->extra); + if (extra && !extra->optimized_feature.t2 && mm->src[0]->ne[2] == 1) { // first use: repack here, as the unmerged path + const ggml_tensor * w = mm->src[0]; + if (!ggml_sycl_t2_repack(*ctx.stream(), w->data, w->ne[0], w->ne[1], w->type == GGML_TYPE_PQ2_0)) { + return false; + } + extra->optimized_feature.t2 = true; + } + const ggml_tensor * act = mm->src[1]; + const int64_t M = act->ne[1] * act->ne[2]; + return extra && extra->optimized_feature.t2 && act->type == GGML_TYPE_F32 && mm->type == GGML_TYPE_F32 && + act->nb[0] == sizeof(float) && act->ne[3] == 1 && (act->ne[2] == 1 || act->nb[2] == act->ne[1] * act->nb[1]) && + M >= 1 && M <= 8 && ggml_is_contiguous(mm) && mm->ne[0] == mm->src[0]->ne[1]; +} + +static bool ggml_sycl_try_t2_multi(ggml_backend_sycl_context & ctx, ggml_cgraph * cgraph, int i, std::vector & done, + std::vector & pending) { + static int log = getenv("GGML_SYCL_T2_MULTI_LOG") ? atoi(getenv("GGML_SYCL_T2_MULTI_LOG")) : 0; + ggml_tensor * mm0 = cgraph->nodes[i]; + if (!ggml_sycl_knob("GGML_SYCL_T2_MULTI", 1) || !ggml_sycl_t2_multi_eligible(ctx, mm0)) { + return false; + } + const ggml_tensor * act = mm0->src[1]; + int members[3] = { i, -1, -1 }; + bool staged[3] = { false, false, false }; + int nm = 1; + const int jend = std::min(cgraph->n_nodes, i + 512); + for (int j = i + 1; j < jend && nm < 3; ++j) { + ggml_tensor * mj = cgraph->nodes[j]; + if (done[j] || mj->op != GGML_OP_MUL_MAT || mj->src[1] != act || mj->src[0]->ne[0] != mm0->src[0]->ne[0] || + !ggml_sycl_t2_multi_eligible(ctx, mj)) { + continue; + } + bool safe = true; + for (int k = i + 1; k < j && safe; ++k) { + const ggml_tensor * nk = cgraph->nodes[k]; + bool member = false; + for (int m = 1; m < nm; ++m) { + member = member || members[m] == k; + } + if (member) { + continue; + } + safe = !ggml_sycl_tensors_overlap(mj, nk); + for (int s = 0; s < GGML_MAX_SRC && safe; ++s) { + safe = !ggml_sycl_tensors_overlap(mj, nk->src[s]); + } + } + for (int m = 0; m < nm && safe; ++m) { + safe = !ggml_sycl_tensors_overlap(mj, cgraph->nodes[members[m]]) && !ggml_sycl_tensors_overlap(mj, act); + } + if (safe || ggml_nbytes(mj) <= PTQ1_MULTI_SLOT_BYTES) { + staged[nm] = !safe; + members[nm++] = j; + } + } + if (nm < 2) { + return false; + } + const queue_ptr stream = ctx.stream(); + const void * w[3]; + float * dst[3]; + int64_t N[3]; + for (int m = 0; m < nm; ++m) { + ggml_tensor * t = cgraph->nodes[members[m]]; + w[m] = t->src[0]->data; + dst[m] = (float *) t->data; + N[m] = t->src[0]->ne[1]; + if (staged[m]) { + dst[m] = ggml_sycl_ptq1_multi_slot(stream); + if (!dst[m]) { + return false; + } + } + } + const void * key = act->view_src && act->view_offs == 0 ? act->view_src : act; + if (!ggml_sycl_t2_mul_mat_multi(*stream, nm, w, dst, N, (const float *) act->data, act->nb[1] / sizeof(float), + act->ne[1] * act->ne[2], act->ne[0], key)) { + return false; + } + if (log > 0) { + fprintf(stderr, "[t2-multi] %s", mm0->name); + for (int m = 1; m < nm; ++m) { + fprintf(stderr, " + %s (+%d%s)", cgraph->nodes[members[m]]->name, members[m] - i, staged[m] ? ", staged" : ""); + } + fprintf(stderr, "\n"); + --log; + } + for (int m = 1; m < nm; ++m) { + done[members[m]] = 1; + if (staged[m]) { + pending[members[m]] = dst[m]; + } + } + return true; +} + +// ARC-LAB: see the call in ggml_backend_sycl_graph_compute_impl. Advances i past the nodes it computed. +static bool ggml_sycl_try_add_norm_rot(ggml_backend_sycl_context & ctx, ggml_cgraph * cgraph, int & i, + std::vector & done) { + const int n0 = i; + ggml_tensor * add = cgraph->nodes[n0]->op == GGML_OP_ADD ? cgraph->nodes[n0] : nullptr; + const int in = add ? n0 + 1 : n0; // RMS_NORM + if (in + 1 >= cgraph->n_nodes) { + return false; + } + ggml_tensor * norm = cgraph->nodes[in]; + ggml_tensor * mulw = cgraph->nodes[in + 1]; + if (norm->op != GGML_OP_RMS_NORM || mulw->op != GGML_OP_MUL || mulw->src[0] != norm) { + return false; + } + if (add && norm->src[0] != add) { + return false; + } + const ggml_tensor * x = add ? add : norm->src[0]; + const ggml_tensor * w = mulw->src[1]; + const int64_t K = x->ne[0], M = ggml_nrows(x); + auto f32c = [](const ggml_tensor * t) { return t->type == GGML_TYPE_F32 && ggml_is_contiguous(t); }; + if (!f32c(x) || !f32c(norm) || !f32c(mulw) || !f32c(w) || w->ne[0] != K || ggml_nrows(w) != 1 || x->ne[3] != 1) { + return false; + } + if (add && (!f32c(add->src[0]) || !f32c(add->src[1]) || !ggml_are_same_shape(add->src[0], add) || + !ggml_are_same_shape(add->src[1], add))) { + return false; + } + // rotation tail: MUL(signs) + RESHAPE + MUL_MAT(hint Hadamard 1024) + ggml_tensor * mm = nullptr; + const float * signs = nullptr; + if (in + 4 < cgraph->n_nodes) { + ggml_tensor * ms = cgraph->nodes[in + 2]; + ggml_tensor * rs = cgraph->nodes[in + 3]; + ggml_tensor * mt = cgraph->nodes[in + 4]; + const ggml_tensor * sg = ms->src[1]; + if (ms->op == GGML_OP_MUL && ms->src[0] == mulw && rs->op == GGML_OP_RESHAPE && rs->src[0] == ms && + mt->op == GGML_OP_MUL_MAT && mt->src[1] == rs && ggml_get_op_params_i32(mt, 1) == GGML_HINT_SRC0_IS_HADAMARD && + mt->src[0]->ne[0] == 1024 && f32c(mt) && f32c(ms) && sg && f32c(sg) && sg->ne[0] == K && ggml_nrows(sg) == 1) { + mm = mt; + signs = (const float *) sg->data; + } + } + // DeltaNet layers: the rotation comes a few nodes later (conv-cache ops in between). Fold it in and write its output + // here, early, if no node in between touches that memory (GGML_SYCL_NORM_ROT_FAR, default on). + int ms_far = -1; + if (!mm && add && ggml_sycl_knob("GGML_SYCL_NORM_ROT_FAR", 1)) { + for (int j = in + 2; j + 2 < cgraph->n_nodes && j <= in + 32; ++j) { + ggml_tensor * ms = cgraph->nodes[j]; + if (ms->op != GGML_OP_MUL || ms->src[0] != mulw) { + continue; + } + ggml_tensor * rs = cgraph->nodes[j + 1]; + ggml_tensor * mt = cgraph->nodes[j + 2]; + const ggml_tensor * sg = ms->src[1]; + if (!done[j] && !done[j + 1] && !done[j + 2] && rs->op == GGML_OP_RESHAPE && rs->src[0] == ms && + mt->op == GGML_OP_MUL_MAT && mt->src[1] == rs && ggml_get_op_params_i32(mt, 1) == GGML_HINT_SRC0_IS_HADAMARD && + mt->src[0]->ne[0] == 1024 && f32c(mt) && f32c(ms) && sg && f32c(sg) && sg->ne[0] == K && ggml_nrows(sg) == 1 && + sg->op == GGML_OP_NONE) { // the signs are read early: they must be a weight, not computed in between + bool clear = true; // nodes computed between the norm and the rotation must not touch the rotation output + for (int k = in + 2; k < j && clear; ++k) { + const ggml_tensor * nk = cgraph->nodes[k]; + if (ggml_sycl_is_view_or_noop(nk) || (nk->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) { + continue; + } + clear = !ggml_sycl_tensors_overlap(mt, nk); + for (int t = 0; t < GGML_MAX_SRC && clear; ++t) { + clear = !ggml_sycl_tensors_overlap(mt, nk->src[t]); + } + } + if (clear) { + ms_far = j; + mm = mt; + signs = (const float *) sg->data; + } + } + break; // only the first MUL of the norm output is a candidate + } + } + if (!add && !mm) { + return false; // plain RMS_NORM + MUL: ggml_sycl_op_rms_norm_fused below + } + const int last = mm ? (ms_far >= 0 ? ms_far + 2 : in + 4) : in + 1; + // nodes inside the chain must have no other consumers; write the ADD and the weighted norm only if used after it + std::vector idx; + for (int j = n0; j <= in + 1; ++j) idx.push_back(j); + if (mm) { + const int t0 = ms_far >= 0 ? ms_far : in + 2; + for (int j = t0; j <= t0 + 2; ++j) idx.push_back(j); + } + std::vector outs; + if (add) outs.push_back(n0); + outs.push_back(in + 1); + if (mm) outs.push_back(last); + std::vector ops; + for (int j : idx) ops.push_back(cgraph->nodes[j]->op); + if (!ggml_can_fuse_subgraph_ext(cgraph, idx.data(), (int) idx.size(), ops.data(), outs.data(), (int) outs.size())) { + return false; + } + auto used_after = [&](int j) { + int inside = 0; + for (int k : idx) { + if (k <= j) continue; + for (int s = 0; s < GGML_MAX_SRC; ++s) inside += cgraph->nodes[k]->src[s] == cgraph->nodes[j]; + } + return (cgraph->nodes[j]->flags & GGML_TENSOR_FLAG_OUTPUT) || ggml_node_get_use_count(cgraph, j) > inside; + }; + float * add_out = add && used_after(n0) ? (float *) add->data : nullptr; + float * normed = !mm || used_after(in + 1) ? (float *) mulw->data : nullptr; + // across tokens a work-group may write where another still reads: outputs must miss the inputs or match them exactly + if (M > 1) { + const ggml_tensor * ins[2] = { add ? add->src[0] : x, add ? add->src[1] : nullptr }; + std::pair outr[3] = { { add_out, add_out ? ggml_nbytes(add) : 0 }, + { normed, normed ? ggml_nbytes(mulw) : 0 }, + { mm ? mm->data : nullptr, mm ? ggml_nbytes(mm) : 0 } }; + for (const auto & o : outr) { + for (const ggml_tensor * t : ins) { + if (!o.first || !t) continue; + const char * ob = (const char *) o.first, * tb = (const char *) t->data; + if (ob != tb && ob < tb + ggml_nbytes(t) && tb < ob + o.second) { + return false; + } + } + } + } + float eps; + memcpy(&eps, norm->op_params, sizeof(float)); + if (!ggml_sycl_t2_add_norm_rot(*ctx.stream(), add ? (const float *) add->src[0]->data : (const float *) x->data, + add ? (const float *) add->src[1]->data : nullptr, add_out, (const float *) w->data, eps, + normed, signs, mm ? (float *) mm->data : nullptr, M, K, mm)) { + return false; + } + static bool fired_logged = false; + if (!fired_logged && getenv("GGML_SYCL_FUSION_LOG")) { + fired_logged = true; + fprintf(stderr, "ggml_sycl_try_add_norm_rot: fused (%s%s%s)\n", add ? "add + " : "", "norm", mm ? " + rotation" : ""); + } + if (ms_far >= 0) { + done[ms_far] = done[ms_far + 1] = done[ms_far + 2] = 1; // computed here, skipped when the loop gets there + i = in + 1; + static bool logged = false; + if (!logged) { + logged = true; + fprintf(stderr, "%s: norm + rotation %d nodes apart fused\n", __func__, ms_far - (in + 1)); + } + } else { + i = last; + } + return true; +} + +// ARC-LAB: DeltaNet conv chain for one token: CONCAT(conv state, new qkv) -> CPY(newest d_conv-1 columns -> conv cache) -> +// SSM_CONV -> SILU -> L2_NORM(q/k heads) in one kernel (one work-group per head of S channels), instead of 5 launches. +// The nodes are spread over ~15 graph nodes with cache ops in between; the fused outputs are written at the CONCAT's +// position, so every node computed in between must not touch their memory (checked), and they must not overlap inputs. +// GGML_SYCL_CONV_CHAIN=0 turns it off. +static bool ggml_sycl_try_conv_chain(ggml_backend_sycl_context & ctx, ggml_cgraph * cgraph, int i, std::vector & done) { + if (!ggml_sycl_knob("GGML_SYCL_CONV_CHAIN", 1)) { + return false; + } + ggml_tensor * cat = cgraph->nodes[i]; + if (cat->op != GGML_OP_CONCAT || cat->type != GGML_TYPE_F32 || ggml_get_op_params_i32(cat, 0) != 0) { + return false; + } + const int64_t dc = cat->ne[0], C = cat->ne[1]; + const ggml_tensor * cs = cat->src[0]; + const ggml_tensor * x = cat->src[1]; + if (cat->ne[2] != 1 || cat->ne[3] != 1 || dc < 2 || dc > 8 || !ggml_is_contiguous(cat) || cs->type != GGML_TYPE_F32 || + x->type != GGML_TYPE_F32 || cs->ne[0] != dc - 1 || cs->ne[1] != C || !ggml_is_contiguous(cs) || x->ne[0] != 1 || + x->ne[1] != C) { + return false; + } + int ic = -1, iv = -1, iconv = -1, isilu = -1, il2 = -1; + const int jmax = std::min(cgraph->n_nodes - 1, i + 48); + for (int j = i + 1; j <= jmax && il2 < 0; ++j) { + ggml_tensor * n = cgraph->nodes[j]; + if (n->op == GGML_OP_CPY && n->src[0]->op == GGML_OP_VIEW && n->src[0]->view_src == cat) { + ic = j; + } else if (n->op == GGML_OP_SSM_CONV && n->src[0] == cat) { + iconv = j; + } else if (iconv >= 0 && n->op == GGML_OP_UNARY && ggml_get_unary_op(n) == GGML_UNARY_OP_SILU && + n->src[0] == cgraph->nodes[iconv]) { + isilu = j; + } else if (isilu >= 0 && n->op == GGML_OP_L2_NORM && n->src[0]->op == GGML_OP_VIEW && + n->src[0]->view_src == cgraph->nodes[isilu]) { + il2 = j; + iv = j; + } + } + static int reasons = 0; + auto why = [&](const char * r) { + if (reasons < 3 && getenv("GGML_SYCL_CONV_CHAIN_DEBUG")) { + ++reasons; + fprintf(stderr, "conv chain: not fused at node %d (%s): %s\n", i, cat->name, r); + } + return false; + }; + if (ic < 0 || iconv < 0 || isilu < 0 || il2 < 0 || done[ic] || done[iconv] || done[isilu] || done[il2]) { + return why("chain nodes not found"); + } + ggml_tensor * cpy = cgraph->nodes[ic]; + ggml_tensor * conv = cgraph->nodes[iconv]; + ggml_tensor * silu = cgraph->nodes[isilu]; + ggml_tensor * l2 = cgraph->nodes[il2]; + const ggml_tensor * cv = cpy->src[0]; // view: columns 1..dc-1 of the concat + const ggml_tensor * dst = cpy->src[1]; // conv cache slot + const ggml_tensor * w = conv->src[1]; + const ggml_tensor * l2v = l2->src[0]; + const int64_t S = l2v->ne[0], H = l2v->ne[1]; + if (cv->ne[0] != dc - 1 || cv->ne[1] != C || cv->view_offs != sizeof(float) || cv->nb[1] != cat->nb[1] || + dst->type != GGML_TYPE_F32 || !ggml_is_contiguous(dst) || ggml_nelements(dst) != (dc - 1) * C || + w->type != GGML_TYPE_F32 || !ggml_is_contiguous(w) || w->ne[0] != dc || w->ne[1] != C || + ggml_nelements(conv) != C || !ggml_is_contiguous(conv) || !ggml_is_contiguous(silu) || silu->type != GGML_TYPE_F32 || + l2v->view_offs != 0 || l2v->nb[1] != (size_t) S * sizeof(float) || l2v->ne[2] != 1 || l2v->ne[3] != 1 || + !ggml_is_contiguous(l2) || S < 16 || S > 1024 || S % 16 != 0 || C % S != 0 || S * H > C || + (cpy->flags & GGML_TENSOR_FLAG_OUTPUT) || (conv->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return why("shapes/types"); + } + // the concat and the raw conv output must have no readers outside the chain + for (int j = i + 1; j < cgraph->n_nodes; ++j) { + const ggml_tensor * n = cgraph->nodes[j]; + for (int k = 0; k < GGML_MAX_SRC && n->src[k]; ++k) { + const ggml_tensor * sj = n->src[k]; + if ((sj == cat && j != iconv && n != cv) || (sj == conv && j != isilu) || (sj->view_src == cat && sj != cv)) { + return why("outside reader"); + } + } + if (n->view_src == cat && n != cv) { + return why("outside view"); + } + } + auto overlap = [](const void * a, size_t na, const void * b, size_t nb) { + const char * pa = (const char *) a, * pb = (const char *) b; + return na && nb && pa < pb + nb && pb < pa + na; + }; + const void * outs[3] = { silu->data, l2->data, dst->data }; + const size_t outn[3] = { ggml_nbytes(silu), ggml_nbytes(l2), ggml_nbytes(dst) }; + const ggml_tensor * ins[3] = { cs, x, w }; + // an output may sit exactly on x (the planner reuses its buffer after the concat): x and the silu / l2 outputs keep + // channel c at element c, a lane reads x[c] before writing c, and l2 is written after the work-group barrier + static const char * out_names[3] = { "silu", "l2", "cache" }; + static const char * in_names[3] = { "conv state", "x", "weights" }; + // l2 on top of the conv state (other lanes still read it): leave L2_NORM to its own kernel after this one + const bool l2_in = !overlap(outs[1], outn[1], cs->data, ggml_nbytes(cs)); + for (int o = 0; o < 3; ++o) { + if (o == 1 && !l2_in) { + continue; + } + for (int t = 0; t < 3; ++t) { + if (!overlap(outs[o], outn[o], ins[t]->data, ggml_nbytes(ins[t]))) { + continue; + } + const bool same_slot_ok = t == 1 && o < 2 && outs[o] == ins[t]->data && x->nb[1] == sizeof(float); + if (!same_slot_ok) { + static char msg[96]; + snprintf(msg, sizeof(msg), "output %s overlaps input %s", out_names[o], in_names[t]); + return why(msg); + } + } + } + // nodes computed between the concat and the last fused node must not touch the early-written outputs + for (int j = i + 1; j <= il2; ++j) { + if (j == ic || j == iconv || j == isilu || j == il2) { + continue; + } + const ggml_tensor * n = cgraph->nodes[j]; + if (ggml_sycl_is_view_or_noop(n) || (n->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) { + continue; + } + const int prod[3] = { isilu, il2, ic }; // where each output is produced in the original order + for (int o = 0; o < 3; ++o) { + if (o == 1 && !l2_in) { + continue; + } + if (j > prod[o] && o == 2) { + continue; // the cache slot is final once the original CPY would have run + } + if (overlap(outs[o], outn[o], n->data, ggml_nbytes(n))) { + return why("node in between writes output memory"); + } + const ggml_tensor * own[3] = { silu, l2, cpy }; + for (int k = 0; k < GGML_MAX_SRC && n->src[k]; ++k) { + const ggml_tensor * sk = n->src[k]; + if (j > prod[o] && (sk == own[o] || sk->view_src == own[o])) { + continue; // a consumer of this output, after its producer: reads the right value + } + if (overlap(outs[o], outn[o], sk->data, ggml_nbytes(sk))) { + return why("node in between reads output memory"); + } + } + } + } + float eps; + memcpy(&eps, l2->op_params, sizeof(float)); + const float * csd = (const float *) cs->data; + const char * xd = (const char *) x->data; + const size_t xnb = x->nb[1]; + const float * wd = (const float *) w->data; + float * dd = (float *) dst->data; + float * sd = (float *) silu->data; + float * ld = (float *) l2->data; + const int D = (int) dc; + const int64_t HL = l2_in ? H : 0; + ctx.stream()->parallel_for(sycl::nd_range<1>((size_t) C, (size_t) S), [=](sycl::nd_item<1> it) { + const int64_t c = it.get_global_id(0); + const int64_t g = it.get_group(0); + float v[8]; + for (int t = 0; t < D - 1; ++t) { + v[t] = csd[c * (D - 1) + t]; + } + v[D - 1] = *(const float *) (xd + c * xnb); + float sum = 0.0f; + for (int t = 0; t < D; ++t) { + sum += v[t] * wd[c * D + t]; + } + for (int t = 1; t < D; ++t) { + dd[c * (D - 1) + t - 1] = v[t]; + } + const float y = op_silu(sum); + sd[c] = y; + if (g < HL) { // uniform per work-group + const float ss = sycl::reduce_over_group(it.get_group(), y * y, sycl::plus()); + const float scale = sycl::rsqrt(sycl::fmax(ss, eps * eps)); + ld[g * S + it.get_local_id(0)] = scale * y; + } + }); + done[ic] = done[iconv] = done[isilu] = 1; + if (l2_in) { + done[il2] = 1; + } + static bool logged = false; + if (!logged) { + logged = true; + fprintf(stderr, "%s: fused DeltaNet conv chain active (C=%lld, d_conv=%lld, %lld heads of %lld for L2)\n", __func__, + (long long) C, (long long) dc, (long long) H, (long long) S); + } + GGML_UNUSED(iv); + return true; +} + +// ARC-LAB: DeltaNet gate block in one kernel (GGML_SYCL_DN_GATES, default on): +// alpha = MUL_MAT(bf16 W_a, x) -> RESHAPE -> ADD(dt bias) -> SOFTPLUS -> MUL(ssm_a) -> RESHAPE +// beta = MUL_MAT(bf16 W_b, x) -> RESHAPE -> SIGMOID +// 9 graph nodes / 7 launches per layer for 2 x 48 outputs; here one work-group per row of both matrices, up to 8 tokens. +static bool ggml_sycl_try_dn_gates(ggml_backend_sycl_context & ctx, ggml_cgraph * cgraph, int & i) { + if (!ggml_sycl_knob("GGML_SYCL_DN_GATES", 1) || i + 8 >= cgraph->n_nodes) { + return false; + } + ggml_tensor * const * n = cgraph->nodes + i; + static int dbg = getenv("GGML_SYCL_DN_GATES_DEBUG") ? 6 : 0; + const bool bf16_mm = n[0]->src[0] && n[0]->src[0]->type == GGML_TYPE_BF16; + auto why = [&](const char * r) { + if (dbg > 0 && bf16_mm) { + --dbg; + fprintf(stderr, "dn gates: not fused at node %d (%s, src1 ne %lld %lld %lld): %s; next ops:", i, n[0]->name, + (long long) n[0]->src[1]->ne[0], (long long) n[0]->src[1]->ne[1], (long long) n[0]->src[1]->ne[2], r); + for (int k = 1; k < 9 && i + k < cgraph->n_nodes; ++k) fprintf(stderr, " %s", ggml_op_desc(n[k])); + fprintf(stderr, "\n"); + } + return false; + }; + const ggml_op ops[9] = { GGML_OP_MUL_MAT, GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_UNARY, GGML_OP_MUL, + GGML_OP_RESHAPE, GGML_OP_MUL_MAT, GGML_OP_RESHAPE, GGML_OP_UNARY }; + for (int k = 0; k < 9; ++k) { + if (n[k]->op != ops[k]) { + return why("op sequence"); + } + } + if (ggml_get_unary_op(n[3]) != GGML_UNARY_OP_SOFTPLUS || ggml_get_unary_op(n[8]) != GGML_UNARY_OP_SIGMOID || + n[1]->src[0] != n[0] || n[2]->src[0] != n[1] || n[3]->src[0] != n[2] || n[4]->src[0] != n[3] || + n[5]->src[0] != n[4] || n[7]->src[0] != n[6] || n[8]->src[0] != n[7] || n[6]->src[1] != n[0]->src[1]) { + return why("edges"); + } + const ggml_tensor * wa = n[0]->src[0]; + const ggml_tensor * wb = n[6]->src[0]; + const ggml_tensor * x = n[0]->src[1]; + const ggml_tensor * bias = n[2]->src[1]; + const ggml_tensor * sa = n[4]->src[1]; + const int64_t K = wa->ne[0], R = wa->ne[1], M = x->ne[1] * x->ne[2]; + auto f32c = [](const ggml_tensor * t) { return t->type == GGML_TYPE_F32 && ggml_is_contiguous(t); }; + if (wa->type != GGML_TYPE_BF16 || wb->type != GGML_TYPE_BF16 || !ggml_is_contiguous(wa) || !ggml_is_contiguous(wb) || + wb->ne[0] != K || wb->ne[1] != R || !f32c(x) || x->ne[0] != K || x->ne[3] != 1 || M < 1 || M > 8 || K % 8 != 0 || + !f32c(bias) || ggml_nelements(bias) != R || !f32c(sa) || ggml_nelements(sa) != R || !f32c(n[4]) || !f32c(n[8]) || + ggml_nelements(n[4]) != R * M || ggml_nelements(n[8]) != R * M || !f32c(n[0]) || !f32c(n[6])) { + return why("shapes/types"); + } + { + int idx[9]; + for (int k = 0; k < 9; ++k) idx[k] = i + k; + const int outs[3] = { i + 4, i + 5, i + 8 }; + if (!ggml_can_fuse_subgraph_ext(cgraph, idx, 9, ops, outs, 3)) { + return why("outside readers"); + } + } + auto overlap = [](const ggml_tensor * a, const ggml_tensor * b) { + const char * pa = (const char *) a->data, * pb = (const char *) b->data; + return pa < pb + ggml_nbytes(b) && pb < pa + ggml_nbytes(a); + }; + // every work-group reads all of x: an output on top of an input (the planner reuses x's buffer once its last + // reader is done) goes through a scratch buffer and one small copy kernel + bool staged = false; + for (const ggml_tensor * o : { n[4], n[8] }) { + for (const ggml_tensor * in : { x, wa, wb, bias, sa }) { + staged |= overlap(o, in); + } + } + ggml_sycl_pool_alloc stage(ctx.pool()); + float * sa_out = nullptr; + if (staged) { + sa_out = stage.alloc((size_t) 2 * R * M); + } + const uint16_t * pa = (const uint16_t *) wa->data; + const uint16_t * pb = (const uint16_t *) wb->data; + const float * px = (const float *) x->data; + const float * pbias = (const float *) bias->data; + const float * psa = (const float *) sa->data; + float * oa = staged ? sa_out : (float *) n[4]->data; + float * ob = staged ? sa_out + R * M : (float *) n[8]->data; + const int Ki = (int) K, Ri = (int) R, Mi = (int) M; + constexpr int WG = 256; + ctx.stream()->parallel_for(sycl::nd_range<1>((size_t) 2 * R * WG, WG), [=](sycl::nd_item<1> it) { + const int g = it.get_group(0); + const bool isb = g >= Ri; + const int row = isb ? g - Ri : g; + const uint16_t * w = (isb ? pb : pa) + (size_t) row * Ki; + const int tid = it.get_local_id(0); + float acc[8] = { 0, 0, 0, 0, 0, 0, 0, 0 }; + for (int k = tid * 8; k < Ki; k += WG * 8) { + const sycl::vec wv = *(const sycl::vec *) (w + k); + float wf[8]; +#pragma unroll + for (int j = 0; j < 8; ++j) wf[j] = sycl::bit_cast((uint32_t) wv[j] << 16); + for (int m = 0; m < Mi; ++m) { + const float * xr = px + (size_t) m * Ki + k; +#pragma unroll + for (int j = 0; j < 8; ++j) acc[m] += wf[j] * xr[j]; + } + } + for (int m = 0; m < Mi; ++m) { + const float sum = sycl::reduce_over_group(it.get_group(), acc[m], sycl::plus()); + if (tid == 0) { + if (isb) { + ob[m * Ri + row] = 1.0f / (1.0f + sycl::exp(-sum)); + } else { + const float v = sum + pbias[row]; + const float sp = sycl::fmax(v, 0.0f) + sycl::log1p(sycl::exp(-sycl::fabs(v))); + oa[m * Ri + row] = sp * psa[row]; + } + } + } + }); + if (staged) { + float * da = (float *) n[4]->data, * db = (float *) n[8]->data; + const float * so = sa_out; + const size_t RM = (size_t) R * M; + ctx.stream()->parallel_for(sycl::range<1>(2 * RM), [=](sycl::id<1> j) { + if (j < RM) { + da[j] = so[j]; + } else { + db[j - RM] = so[j]; + } + }); + } + static bool logged = false, logged_staged = false; + if (staged && !logged_staged) { + logged_staged = true; + fprintf(stderr, "%s: fused DeltaNet gates active via scratch (%lld tokens)\n", __func__, (long long) M); + } + if (!logged) { + logged = true; + fprintf(stderr, "%s: fused DeltaNet gates active (K=%lld, 2 x %lld rows, %lld tokens)\n", __func__, (long long) K, + (long long) R, (long long) M); + } + i += 8; + return true; +} + +// ARC-LAB: a kernel whose work-group reads block r of an input and later writes block r of its output may run in place +// (same base address), never with a partial overlap +static bool ggml_sycl_same_or_disjoint(const ggml_tensor * out, const ggml_tensor * in) { + const char * po = (const char *) out->data, * pi = (const char *) in->data; + return po == pi || po + ggml_nbytes(out) <= pi || pi + ggml_nbytes(in) <= po; +} + static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * sycl_ctx, ggml_cgraph * cgraph) { ggml_sycl_set_main_device(sycl_ctx->device); - + ggml_sycl_gdn_gather_reset(*sycl_ctx); + // ARC-LAB: nodes already computed by ggml_sycl_try_ptq1_multi, and staged outputs still to copy in + std::vector done(cgraph->n_nodes, 0); + std::vector pending(cgraph->n_nodes, nullptr); + const bool ht_on = g_ht.on && cgraph->n_nodes > 500; + sycl_host_timing::clk::time_point ht_prev = sycl_host_timing::clk::now(); + const char * ht_op = nullptr; for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; + if (ht_on) { + const auto now = sycl_host_timing::clk::now(); + if (ht_op) { + auto & a = g_ht.per_op[ht_op]; + a.first += std::chrono::duration(now - ht_prev).count(); + a.second++; + } + ht_prev = now; + ht_op = ggml_op_desc(node); + } + if (done[i]) { + if (pending[i]) { + SYCL_CHECK(CHECK_TRY_ERROR(sycl_ctx->stream()->memcpy(node->data, pending[i], ggml_nbytes(node)))); + } + continue; + } if (ggml_sycl_is_view_or_noop(node)) { continue; } @@ -5805,6 +7212,9 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc } } #endif + if (node->op == GGML_OP_GET_ROWS && ggml_sycl_try_gdn_gather_skip(*sycl_ctx, cgraph, i)) { + continue; + } // gated_delta_net -> cpy: scatter recurrent-state snapshots into the cache if (node->op == GGML_OP_GATED_DELTA_NET) { ggml_sycl_gated_delta_net_fused_cache fused_state_cpy; @@ -5815,6 +7225,79 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc continue; } } + if (node->op == GGML_OP_MUL_MAT && ggml_sycl_try_dn_gates(*sycl_ctx, cgraph, i)) { + continue; + } + if (node->op == GGML_OP_CONCAT && ggml_sycl_try_conv_chain(*sycl_ctx, cgraph, i, done)) { + continue; + } + // ARC-LAB: [residual ADD +] RMS_NORM + MUL(weight) [+ MUL(signs) + RESHAPE + FWHT-hint MUL_MAT] in one kernel + // (ggml_sycl_t2_add_norm_rot): the post-attention norm of every layer and the pre-attention norm of the full + // attention layers end in the rotation that feeds the T2 mat-vecs; the DeltaNet layers' norm is followed by + // cache ops first, so there only the ADD folds in. + if ((node->op == GGML_OP_ADD || node->op == GGML_OP_RMS_NORM) && ggml_sycl_try_add_norm_rot(*sycl_ctx, cgraph, i, done)) { + continue; + } + // ARC-LAB: FFN SWIGLU -> MUL(signs) -> RESHAPE -> FWHT-hint MUL_MAT: the rotation computes silu(gate) * up on its + // load (GGML_SYCL_GLU_ROT, default on) + if (node->op == GGML_OP_GLU && i + 3 < cgraph->n_nodes && ggml_sycl_knob("GGML_SYCL_GLU_ROT", 1)) { + ggml_tensor * ms = cgraph->nodes[i + 1]; + ggml_tensor * rs = cgraph->nodes[i + 2]; + ggml_tensor * mt = cgraph->nodes[i + 3]; + const ggml_tensor * g = node->src[0]; + const ggml_tensor * u = node->src[1]; + const ggml_tensor * sg = ms->src[1]; + const ggml_op gops[4] = { GGML_OP_GLU, GGML_OP_MUL, GGML_OP_RESHAPE, GGML_OP_MUL_MAT }; + const int gidx[4] = { i, i + 1, i + 2, i + 3 }; + const int gout[1] = { i + 3 }; + if (ggml_get_glu_op(node) == GGML_GLU_OP_SWIGLU && u && ggml_get_op_params_i32(node, 1) == 0 && + ms->op == GGML_OP_MUL && ms->src[0] == node && rs->op == GGML_OP_RESHAPE && rs->src[0] == ms && + mt->op == GGML_OP_MUL_MAT && mt->src[1] == rs && + ggml_get_op_params_i32(mt, 1) == GGML_HINT_SRC0_IS_HADAMARD && mt->src[0]->ne[0] == 1024 && + g->type == GGML_TYPE_F32 && u->type == GGML_TYPE_F32 && node->type == GGML_TYPE_F32 && + ggml_is_contiguous(g) && ggml_is_contiguous(u) && ggml_are_same_shape(g, u) && + ggml_are_same_shape(g, node) && g->ne[3] == 1 && sg && sg->type == GGML_TYPE_F32 && + ggml_is_contiguous(sg) && sg->ne[0] == g->ne[0] && ggml_nrows(sg) == 1 && ggml_is_contiguous(mt) && + ggml_can_fuse_subgraph_ext(cgraph, gidx, 4, gops, gout, 1) && + ggml_sycl_same_or_disjoint(mt, g) && ggml_sycl_same_or_disjoint(mt, u) && + ggml_sycl_t2_fwht_quant(*sycl_ctx->stream(), nullptr, (const float *) sg->data, (float *) mt->data, + ggml_nrows(g), g->ne[0], mt, (const float *) g->data, (const float *) u->data)) { + static bool glu_logged = false; + if (!glu_logged) { + glu_logged = true; + fprintf(stderr, "ggml_sycl: SWIGLU folded into the rotation (%s)\n", node->name); + } + i += 3; + continue; + } + } + // Hadamard sign flip + reshape + FWHT-hint matmul: apply the signs in the transform's load + if (node->op == GGML_OP_MUL && + ggml_can_fuse_subgraph(cgraph, i, { GGML_OP_MUL, GGML_OP_RESHAPE, GGML_OP_MUL_MAT }, { i + 2 })) { + const ggml_tensor * reshape = cgraph->nodes[i + 1]; + ggml_tensor * mm = cgraph->nodes[i + 2]; + const ggml_tensor * x = node->src[0]; + const ggml_tensor * signs = node->src[1]; + + const bool pattern_ok = ggml_get_op_params_i32(mm, 1) == GGML_HINT_SRC0_IS_HADAMARD && + mm->src[1] == reshape && reshape->src[0] == node && signs->ne[1] == 1 && signs->ne[2] == 1 && + signs->ne[3] == 1 && signs->type == GGML_TYPE_F32 && + (x->type == GGML_TYPE_F32 || x->type == GGML_TYPE_F16) && node->type == x->type && + ggml_is_contiguous(x) && ggml_is_contiguous(signs) && signs->ne[0] == x->ne[0] && + signs->ne[0] % mm->src[0]->ne[0] == 0; + // ARC-LAB: the rotation that feeds T2 mat-vecs also quantizes for them (they read mm through a reshape) + if (pattern_ok && x->type == GGML_TYPE_F32 && mm->src[0]->ne[0] == 1024 && x->ne[3] == 1 && + ggml_is_contiguous(mm) && + ggml_sycl_t2_fwht_quant(*sycl_ctx->stream(), (const float *) x->data, (const float *) signs->data, + (float *) mm->data, ggml_nrows(x), x->ne[0], mm)) { + i += 2; + continue; + } + if (pattern_ok && ggml_sycl_op_fwht_signed(*sycl_ctx, x, signs, mm)) { + i += 2; + continue; + } + } if (node->op == GGML_OP_RMS_NORM && ggml_sycl_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL }, {})) { ggml_sycl_op_rms_norm_fused(*sycl_ctx, node, cgraph->nodes[i + 1]); @@ -5832,6 +7315,12 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc i += 2; continue; } + if (node->op == GGML_OP_MUL_MAT && ggml_sycl_try_t2_multi(*sycl_ctx, cgraph, i, done, pending)) { + continue; + } + if (node->op == GGML_OP_MUL_MAT && ggml_sycl_try_ptq1_multi(*sycl_ctx, cgraph, i, done, pending)) { + continue; + } bool ok = ggml_sycl_compute_forward(*sycl_ctx, node); if (!ok) { @@ -5854,11 +7343,6 @@ static bool check_graph_compatibility(ggml_cgraph * cgraph) { switch (node_op) { default: break; - case GGML_OP_CONCAT: - // ggml_sycl_op_concat() does a blocking host wait after memcpy operations, - // but wait() can't be called on the events returned by a queue recording - // to a graph. - [[fallthrough]]; case GGML_OP_MUL_MAT_ID: // ggml_sycl_mul_mat_id() does a blocking host wait on the sycl queue after // submitting a memcpy operation, but wait() can't be called on a queue that @@ -5867,6 +7351,10 @@ static bool check_graph_compatibility(ggml_cgraph * cgraph) { ggml_op_name(node_op)); return false; case GGML_OP_MUL_MAT: + // oneMKL gemm (multi-token batches) cannot be recorded to a graph. + if (cgraph->nodes[i]->src[1]->ne[1] > 1) { + return false; + } // We cannot use graphs with ggml_sycl_mul_mat() when SYCL async memory allocation extensions are not available, // as SYCL malloc / free and host wait calls are not supported when recording to a graph which are all present // in reordering. @@ -5884,8 +7372,90 @@ static bool check_graph_compatibility(ggml_cgraph * cgraph) { } #endif +namespace { +std::mutex g_knob_mtx; +std::unordered_map g_knob_file; // from GGML_SYCL_KNOBS_FILE (wins) +std::unordered_map g_knob_env; // environment, read once per name +} // namespace + +int ggml_sycl_knob(const char * name, int def) { + std::lock_guard lock(g_knob_mtx); + auto f = g_knob_file.find(name); + if (f != g_knob_file.end()) { + return f->second; + } + auto e = g_knob_env.find(name); + if (e != g_knob_env.end()) { + return e->second; + } + const char * v = getenv(name); + const int r = v ? atoi(v) : def; + g_knob_env[name] = r; + return r; +} + +void ggml_sycl_knobs_refresh() { + static const char * path = getenv("GGML_SYCL_KNOBS_FILE"); + if (!path) { + return; + } + static std::filesystem::file_time_type last_time{}; + static std::uintmax_t last_size = 0; + std::error_code ec; + const auto t = std::filesystem::last_write_time(path, ec); + if (ec) { + return; + } + const auto sz = std::filesystem::file_size(path, ec); + if (ec || (t == last_time && sz == last_size)) { + return; + } + last_time = t; + last_size = sz; + std::unordered_map m; + if (FILE * fp = fopen(path, "r")) { + char line[256]; + while (fgets(line, sizeof(line), fp)) { + char * eq = strchr(line, '='); + if (!eq || line[0] == '#') { + continue; + } + *eq = 0; + std::string k(line); + k.erase(k.find_last_not_of(" \t") + 1); + m[k] = atoi(eq + 1); + } + fclose(fp); + } + std::string desc; + for (auto & kv : m) { + desc += " " + kv.first + "=" + std::to_string(kv.second); + } + GGML_LOG_INFO("%s: knobs reloaded from %s:%s\n", __func__, path, desc.c_str()); + std::lock_guard lock(g_knob_mtx); + g_knob_file = std::move(m); +} + static ggml_status ggml_backend_sycl_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { auto * sycl_ctx = static_cast(backend->context); + g_ht.in_big = g_ht.on && cgraph->n_nodes > 500; + if (g_ht.in_big) { + g_ht.t_in = sycl_host_timing::clk::now(); + if (g_ht.last_sync_end.time_since_epoch().count()) { + g_ht.between += std::chrono::duration(g_ht.t_in - g_ht.last_sync_end).count(); + } + g_ht.nodes += cgraph->n_nodes; + } + struct ht_exit { + ~ht_exit() { + if (g_ht.in_big) { + g_ht.t_enq = sycl_host_timing::clk::now(); + g_ht.enqueue += std::chrono::duration(g_ht.t_enq - g_ht.t_in).count(); + } + } + } ht_exit_guard; + ggml_sycl_knobs_refresh(); + ggml_sycl_t2_new_graph(); #ifdef GGML_SYCL_GRAPH bool use_sycl_graph = false; diff --git a/ggml/src/ggml-sycl/mmvq.cpp b/ggml/src/ggml-sycl/mmvq.cpp index 36c4c1778769..7aaef60032ac 100644 --- a/ggml/src/ggml-sycl/mmvq.cpp +++ b/ggml/src/ggml-sycl/mmvq.cpp @@ -1,3 +1,4 @@ +#include "xe2-dpas.hpp" #include "mmvq.hpp" #include "ggml.h" @@ -737,6 +738,752 @@ static void reorder_mul_mat_vec_q4_0_q8_1_sycl_switch_ncols( } } +static void reorder_mul_mat_vec_pq2_0_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, + const int nrows, dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_PQ2_0 == 0); + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_reorder>(vx, vy, dst, ncols, nrows, + nd_item); + }); + }); +} + +template +static void reorder_mul_mat_vec_pq2_0_q8_1_sycl_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_PQ2_0 == 0); + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_reorder_ncols, ncols_dst>( + vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, + /*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item); + }); + }); +} + +static void reorder_mul_mat_vec_pq2_0_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + switch (ncols_dst) { + case 1: reorder_mul_mat_vec_pq2_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; + case 2: reorder_mul_mat_vec_pq2_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 3: reorder_mul_mat_vec_pq2_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 4: reorder_mul_mat_vec_pq2_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 5: reorder_mul_mat_vec_pq2_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 6: reorder_mul_mat_vec_pq2_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 7: reorder_mul_mat_vec_pq2_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 8: reorder_mul_mat_vec_pq2_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + default: GGML_ABORT("unsupported ncols_dst=%d for PQ2_0 reorder multi-col MMVQ", ncols_dst); + } +} + +// PTQ1_0 single-column: one lane takes a whole block, so a sub-group takes R rows and spreads all +// R * blocks_per_row blocks over its lanes. Rows of 40 blocks leave no idle lane at R = 2. +template +static void mul_mat_vec_ptq1_0_reorder_rows(const void * __restrict__ vx, const void * __restrict__ vy, + float * __restrict__ dst, const int ncols, const int nrows, + const sycl::nd_item<3> & nd_item) { + using block_type = ggml_sycl_reordered::block_q_t; + + const auto sg = nd_item.get_sub_group(); + const int row0 = (nd_item.get_group_linear_id() * sg.get_group_linear_range() + sg.get_group_linear_id()) * R; + if (row0 >= nrows) { + return; + } + + const int blocks_per_row = ncols / QK_PTQ1_0; + const int nblocks = nrows * blocks_per_row; + const int lane = sg.get_local_linear_id(); + + float acc[R] = {}; + for (int i = lane; i < R * blocks_per_row; i += WARP_SIZE) { + const int r = i / blocks_per_row; + const int b = i - r * blocks_per_row; + if (row0 + r >= nrows) { + continue; + } + const int ibx = (row0 + r) * blocks_per_row + b; + const int8_t * q8 = (const int8_t *) vy + b * QK_PTQ1_0; + const sycl::half2 * ds = (const sycl::half2 *) ((const char *) vy + ncols + b * block_type::block_to_q8_1_ratio() * sizeof(sycl::half2)); + const float v = reorder_vec_dot_q_sycl()( + vx, block_type::get_block_offset(ibx, nblocks), block_type::get_d_offset(nrows, ncols, ibx), q8, ds, 0); +#pragma unroll + for (int k = 0; k < R; ++k) { + acc[k] += r == k ? v : 0.0f; + } + } + +#pragma unroll + for (int k = 0; k < R; ++k) { + const float sum = sycl::reduce_over_group(sg, acc[k], std::plus<>()); + if (lane == k && row0 + k < nrows) { + dst[row0 + k] = sum; + } + } +} + +template +static void launch_mul_mat_vec_ptq1_0_reorder_rows(const void * vx, const void * vy, float * dst, const int ncols, + const int nrows, dpct::queue_ptr stream) { + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, R * GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_ptq1_0_reorder_rows(vx, vy, dst, ncols, nrows, nd_item); + }); + }); +} + +// ARC-LAB: single-column PTQ1_0 mat-vec in the layout of the Vulkan shader (mul_mat_vecq_ptq1_0.comp, 47 us vs 60 on the +// 17408x5120 FFN matrix): two lanes per 128-weight block (lane 0: qs dwords 0..2, lane 1: dwords 3..5 + qh); a block's +// activations are loaded ONCE and reused for the NR rows of the subgroup (the per-lane-block kernel reloads 128 B of +// activations per row, over 4x the weight bytes); trits decoded per level on two bytes at a time in 16-bit lanes and fed +// straight to dp4a against pre-masked activation words, so the residue byte multiplies a zero (no repacking). +// Core: rows row0 .. row0 + NR - 1 of NM same-shape matrices (NM = 2: gate and up) against one activation column; +// temp[m][n] holds this lane's partial sums (not yet reduced over the subgroup). +template +static inline void ptq1_0_pairs_core(const uint8_t * const (&bases)[NM], const void * __restrict__ vy, const int ncols, + const int nrows, const int row0, const int lane, float (&temp)[NM][NR]) { + const int bpr = ncols / QK_PTQ1_0; + const int nblocks = nrows * bpr; + const int sub = lane & 1; + + const uint32_t * y32 = static_cast(vy); + const sycl::half2 * yds = (const sycl::half2 *) ((const char *) vy + ncols); + +#pragma unroll + for (int m = 0; m < NM; ++m) { +#pragma unroll + for (int n = 0; n < NR; ++n) { + temp[m][n] = 0.0f; + } + } + + for (int b = lane >> 1; b < bpr; b += WARP_SIZE / 2) { + const uint32_t * yb = y32 + b * (QK_PTQ1_0 / 4); + const sycl::float2 ds0 = yds[b * 4 + 0].convert(); + const sycl::float2 ds1 = yds[b * 4 + 1].convert(); + const sycl::float2 ds2 = yds[b * 4 + 2].convert(); + const sycl::float2 ds3 = yds[b * 4 + 3].convert(); + // accA: this lane's first qs dword (sub-block t/2); accB: the other two (lane 0: t/2, lane 1: 2,2,3,3,3) + const float sA0 = ds0.x(), sA1 = ds1.x(), sA2 = ds2.x(); + const float sB0 = sub ? ds2.x() : ds0.x(); + const float sB1 = sub ? ds3.x() : ds1.x(); + const float sB2 = sub ? ds3.x() : ds2.x(); + const float bias = sub ? (ds2.y() + ds3.y()) : (ds0.y() + ds1.y()); + + // activation words per trit level, pre-masked: yh = bytes 1,3; yl = bytes 0,2 moved to 1,3 + int yl0[5], yh0[5], yl1[5], yh1[5], yl2[5], yh2[5]; +#pragma unroll + for (int t = 0; t < 5; ++t) { + const uint32_t a0 = sub ? yb[4 * t + 3] : yb[4 * t]; + const uint32_t a1 = sub ? yb[20 + 2 * t] : yb[4 * t + 1]; + const uint32_t a2 = sub ? yb[21 + 2 * t] : yb[4 * t + 2]; + yh0[t] = (int) (a0 & 0xFF00FF00u); + yh1[t] = (int) (a1 & 0xFF00FF00u); + yh2[t] = (int) (a2 & 0xFF00FF00u); + yl0[t] = (int) ((a0 << 8) & 0xFF00FF00u); + yl1[t] = (int) ((a1 << 8) & 0xFF00FF00u); + yl2[t] = (int) ((a2 << 8) & 0xFF00FF00u); + } + const int yq0 = (int) yb[30]; + const int yq1 = (int) yb[31]; + +#pragma unroll + for (int m = 0; m < NM; ++m) { + const uint8_t * base = bases[m]; +#pragma unroll + for (int n = 0; n < NR; ++n) { + const int row = row0 + n; + if (row >= nrows) { + break; + } + const int ibx = row * bpr + b; + const uint32_t * q32 = (const uint32_t *) (base + ibx * 24) + 3 * sub; + const uint32_t w0 = q32[0], w1 = q32[1], w2 = q32[2]; + const uint8_t * qh = base + nblocks * 24 + ibx * 2; + const float da = (float) *(reinterpret_cast(base + nblocks * 26 + ibx * 2)); + + uint32_t v0l = w0 & 0x00FF00FFu, v0h = (w0 >> 8) & 0x00FF00FFu; + uint32_t v1l = w1 & 0x00FF00FFu, v1h = (w1 >> 8) & 0x00FF00FFu; + uint32_t v2l = w2 & 0x00FF00FFu, v2h = (w2 >> 8) & 0x00FF00FFu; + int accA0 = 0, accA1 = 0, accA2 = 0, accB0 = 0, accB1 = 0, accB2 = 0; + auto level = [](uint32_t & vl, uint32_t & vh, int yl, int yh, int & acc) { + const uint32_t wl = vl * 3u, wh = vh * 3u; + vl = wl & 0x00FF00FFu; + vh = wh & 0x00FF00FFu; + acc = dpct::dp4a((int) wl, yl, acc); + acc = dpct::dp4a((int) wh, yh, acc); + }; + level(v0l, v0h, yl0[0], yh0[0], accA0); level(v1l, v1h, yl1[0], yh1[0], accB0); level(v2l, v2h, yl2[0], yh2[0], accB0); + level(v0l, v0h, yl0[1], yh0[1], accA0); level(v1l, v1h, yl1[1], yh1[1], accB0); level(v2l, v2h, yl2[1], yh2[1], accB0); + level(v0l, v0h, yl0[2], yh0[2], accA1); level(v1l, v1h, yl1[2], yh1[2], accB1); level(v2l, v2h, yl2[2], yh2[2], accB1); + level(v0l, v0h, yl0[3], yh0[3], accA1); level(v1l, v1h, yl1[3], yh1[3], accB1); level(v2l, v2h, yl2[3], yh2[3], accB1); + level(v0l, v0h, yl0[4], yh0[4], accA2); level(v1l, v1h, yl1[4], yh1[4], accB2); level(v2l, v2h, yl2[4], yh2[4], accB2); + { + // qh trits -> elements 120 + 2m + h, sub-block 3 (lane 1 only; lane 0 decodes zeros) + uint32_t vq = sub ? ((uint32_t) qh[0] | ((uint32_t) qh[1] << 16)) : 0u; + uint32_t wq = vq * 3u; + vq = wq & 0x00FF00FFu; + int q01 = (int) (((wq >> 8) & 0x3u) | ((wq >> 16) & 0x300u)); + wq = vq * 3u; + vq = wq & 0x00FF00FFu; + q01 |= (int) (((wq << 8) & 0x30000u) | (wq & 0x03000000u)); + wq = vq * 3u; + vq = wq & 0x00FF00FFu; + int q23 = (int) (((wq >> 8) & 0x3u) | ((wq >> 16) & 0x300u)); + wq = vq * 3u; + q23 |= (int) (((wq << 8) & 0x30000u) | (wq & 0x03000000u)); + accB2 = dpct::dp4a(q01, yq0, accB2); + accB2 = dpct::dp4a(q23, yq1, accB2); + } + const float sum = sA0 * accA0 + sA1 * accA1 + sA2 * accA2 + sB0 * accB0 + sB1 * accB1 + sB2 * accB2 - bias; + temp[m][n] += da * sum; + } + } + } +} + +template +static void mul_mat_vec_ptq1_0_pairs(const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, + const int ncols, const int nrows, const sycl::nd_item<3> & nd_item) { + const auto sg = nd_item.get_sub_group(); + const int lane = sg.get_local_linear_id(); + const int row0 = (nd_item.get_group_linear_id() * sg.get_group_linear_range() + sg.get_group_linear_id()) * NR; + if (row0 >= nrows) { + return; + } + const uint8_t * const bases[1] = { static_cast(vx) }; + float temp[1][NR]; + ptq1_0_pairs_core(bases, vy, ncols, nrows, row0, lane, temp); +#pragma unroll + for (int n = 0; n < NR; ++n) { + const float s = sycl::reduce_over_group(sg, temp[0][n], std::plus<>()); + if (lane == 0 && row0 + n < nrows) { + dst[row0 + n] = s; + } + } +} + +// ARC-LAB: up to 3 matrices that share one activation (GDN qkv + z, attention q + k + v) in ONE launch: the row-group +// index space is the matrices stacked, each subgroup serves NR rows of one of them. A launch costs ~10 us beyond its +// weight bytes on the B580 (in-model trace: 1 MB attn_k 12 us), so merging saves the tail of every extra launch and +// one activation quantization. Row counts must be multiples of NR. +struct ptq1_0_multi_args { + const void * vx[3]; + float * dst[3]; + int nrows[3]; +}; + +template +static void mul_mat_vec_ptq1_0_pairs_multi(const ptq1_0_multi_args a, const void * __restrict__ vy, const int ncols, + const int nrows_total, const sycl::nd_item<3> & nd_item) { + const auto sg = nd_item.get_sub_group(); + const int lane = sg.get_local_linear_id(); + int row0 = (nd_item.get_group_linear_id() * sg.get_group_linear_range() + sg.get_group_linear_id()) * NR; + if (row0 >= nrows_total) { + return; + } + int m = 0; + if (row0 >= a.nrows[0]) { + row0 -= a.nrows[0]; + m = 1; + if (row0 >= a.nrows[1]) { + row0 -= a.nrows[1]; + m = 2; + } + } + const int nrows = a.nrows[m]; + const uint8_t * const bases[1] = { static_cast(a.vx[m]) }; + float * const dst = a.dst[m]; + float temp[1][NR]; + ptq1_0_pairs_core(bases, vy, ncols, nrows, row0, lane, temp); +#pragma unroll + for (int n = 0; n < NR; ++n) { + const float s = sycl::reduce_over_group(sg, temp[0][n], std::plus<>()); + if (lane == 0 && row0 + n < nrows) { + dst[row0 + n] = s; + } + } +} + +// ARC-LAB: single-column gate/up + GLU with the pairs core: each subgroup dots NR rows of BOTH matrices against one load +// of the activations and writes act(gate) * up. +template +static void mul_mat_vec_ptq1_0_pairs_glu(const void * __restrict__ vup, const void * __restrict__ vgate, + const void * __restrict__ vy, float * __restrict__ dst, const int ncols, + const int nrows, const ggml_glu_op glu_op, const sycl::nd_item<3> & nd_item) { + const auto sg = nd_item.get_sub_group(); + const int lane = sg.get_local_linear_id(); + const int row0 = (nd_item.get_group_linear_id() * sg.get_group_linear_range() + sg.get_group_linear_id()) * NR; + if (row0 >= nrows) { + return; + } + const uint8_t * const bases[2] = { static_cast(vup), static_cast(vgate) }; + float temp[2][NR]; + ptq1_0_pairs_core(bases, vy, ncols, nrows, row0, lane, temp); +#pragma unroll + for (int n = 0; n < NR; ++n) { + const float u = sycl::reduce_over_group(sg, temp[0][n], std::plus<>()); + const float g = sycl::reduce_over_group(sg, temp[1][n], std::plus<>()); + if (lane == 0 && row0 + n < nrows) { + dst[row0 + n] = u * (glu_op == GGML_GLU_OP_SWIGLU ? op_silu(g) : op_gelu(g)); + } + } +} + +template +static void launch_ptq1_0_pairs(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, + dpct::queue_ptr stream) { + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, NR * GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_ptq1_0_pairs(vx, vy, dst, ncols, nrows, nd_item); + }); + }); +} + +template +static void launch_ptq1_0_pairs_multi(const ptq1_0_multi_args & a, const void * vy, const int ncols, const int nrows_total, + dpct::queue_ptr stream) { + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows_total, NR * GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + const ptq1_0_multi_args args = a; + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_ptq1_0_pairs_multi(args, vy, ncols, nrows_total, nd_item); + }); + }); +} + +template +static void launch_ptq1_0_pairs_glu(const void * vup, const void * vgate, const void * vy, float * dst, const int ncols, + const int nrows, const ggml_glu_op glu_op, dpct::queue_ptr stream) { + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, NR * GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_ptq1_0_pairs_glu(vup, vgate, vy, dst, ncols, nrows, glu_op, nd_item); + }); + }); +} + +// ARC-LAB: n matrices (<= 3) sharing one SoA-quantized activation column, one launch. False = caller runs them apart. +bool ggml_sycl_ptq1_0_mul_mat_vec_multi(const void * const * vx, float * const * dst, const int * nrows, const int nm, + const void * vy, const int ncols, dpct::queue_ptr stream) { + constexpr int NR = 4; + if (nm < 2 || nm > 3 || ncols % QK_PTQ1_0 != 0) { + return false; + } + ptq1_0_multi_args a = {}; + int total = 0; + for (int m = 0; m < nm; ++m) { + if (nrows[m] % NR != 0) { + return false; + } + a.vx[m] = vx[m]; + a.dst[m] = dst[m]; + a.nrows[m] = nrows[m]; + total += nrows[m]; + } + launch_ptq1_0_pairs_multi(a, vy, ncols, total, stream); + return true; +} + +// ARC-LAB: single-column fused gate/up + GLU (pairs core). GGML_SYCL_PTQ1_GLU1=0 declines (unfused path). +bool ggml_sycl_ptq1_0_glu_n1_enabled() { + static const bool on = !getenv("GGML_SYCL_PTQ1_GLU1") || atoi(getenv("GGML_SYCL_PTQ1_GLU1")) != 0; + return on; +} + +static void reorder_mul_mat_vec_ptq1_0_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, + const int nrows, dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_PTQ1_0 == 0); + // ARC-LAB: Vulkan-style kernel (two lanes per block, activations reused over NR rows), 4 rows per subgroup by default. + // Whole model (llama-bench tg128, B580): old 36.96, NR 2 36.62, NR 4 37.54, NR 8 34.99 t/s. test-backend-ops favours + // NR 8 (10240x5120 36.0 -> 27.0 us) but reruns one matrix that stays in the 18 MB L2; in the model the weights stream + // from memory and NR 8 lost 7% of mat-vec time (unitrace). GGML_SYCL_PTQ1_PAIRS=0 restores the per-lane-block kernel. + static const int pairs_env = getenv("GGML_SYCL_PTQ1_PAIRS") ? atoi(getenv("GGML_SYCL_PTQ1_PAIRS")) : -1; + const int pairs = pairs_env >= 0 ? pairs_env : 4; + if (pairs == 2) { launch_ptq1_0_pairs<2>(vx, vy, dst, ncols, nrows, stream); return; } + if (pairs == 4) { launch_ptq1_0_pairs<4>(vx, vy, dst, ncols, nrows, stream); return; } + if (pairs == 8) { launch_ptq1_0_pairs<8>(vx, vy, dst, ncols, nrows, stream); return; } + // two rows per sub-group only when one row leaves lanes idle in the last round + if ((ncols / QK_PTQ1_0) % WARP_SIZE == 0) { + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_reorder>(vx, vy, dst, ncols, + nrows, nd_item); + }); + }); + } else { + launch_mul_mat_vec_ptq1_0_reorder_rows<2>(vx, vy, dst, ncols, nrows, stream); + } +} + +bool ggml_sycl_ptq1_0_ncols_dec_enabled() { + static const bool off = getenv("GGML_SYCL_PTQ1_NCOLS_DEC_OFF") != nullptr; + return !off; +} + +// ARC-LAB: PTQ1_0 multi-column mat-vec (MTP verify, n-gram drafts, fused gate/up) that decodes each block's trits +// ONCE and reuses them for every column; mul_mat_vec_q_reorder_ncols re-decodes per column (at 4 columns a matrix +// cost 2.5x the single-column time). R rows per sub-group keeps every lane busy when a row's block count is not a +// multiple of the sub-group (Bonsai K 5120 = 40 blocks). +template // PROBE: lab-only compile-time timing variants +static inline void ptq1_0_ncols_dec_body(const void * __restrict__ vx, const void * __restrict__ vgate, + const void * __restrict__ vy, float * __restrict__ dst, const int ncols, + const int nrows, const int stride_col_y_bytes, const int stride_col_dst, + const ggml_glu_op glu_op, const sycl::sub_group & sg, const int row0) { + using block_type = ggml_sycl_reordered::block_q_t; + + const int blocks_per_row = ncols / QK_PTQ1_0; + const int nblocks = nrows * blocks_per_row; + const int lane = sg.get_local_linear_id(); + + float acc[R][ncols_dst] = {}; + [[maybe_unused]] float accg[has_fusion ? R : 1][has_fusion ? ncols_dst : 1] = {}; + + auto load_decode = [&](const void * w, const std::pair bx, const std::pair dof, int * qw) -> float { + const uint8_t * base = static_cast(w); + uint8_t qs[24]; +#pragma unroll + for (int k = 0; k < 3; ++k) { + *(sycl::uint2 *) (qs + 8 * k) = ((const sycl::uint2 *) (base + bx.first))[k]; + } + const uint8_t * qh = base + bx.second; + if constexpr (PROBE == 1) { // no trit decode +#pragma unroll + for (int k = 0; k < 32; ++k) { + qw[k] = (int) qs[k % 24] | ((int) qh[k & 1] << 8); + } + } else { + ptq1_0_decode_block(qs, (uint32_t) qh[0] | ((uint32_t) qh[1] << 16), qw); + } + return (float) *(reinterpret_cast(base + dof.first)); + }; + + for (int i = lane; i < R * blocks_per_row; i += WARP_SIZE) { + const int r = i / blocks_per_row; + const int b = i - r * blocks_per_row; + if (row0 + r >= nrows) { + continue; + } + const int ibx = (row0 + r) * blocks_per_row + b; + const auto bx = block_type::get_block_offset(ibx, nblocks); + const auto dof = block_type::get_d_offset(nrows, ncols, ibx); + + int qw[32]; + const float d = load_decode(vx, bx, dof, qw); + [[maybe_unused]] int qg[has_fusion ? 32 : 1]; + [[maybe_unused]] float dg = 0.0f; + if constexpr (has_fusion) { + dg = load_decode(vgate, bx, dof, qg); + } + + // activations in the quantize_and_reorder_q8_1_ptq1_il layout: chunk c of block b at [c * blocks_per_row + b] + auto dot_col = [&](const int * q, const sycl::int4 * xb, const sycl::half2 * dsb, const float dd) -> float { + int si[4] = { 0, 0, 0, 0 }; +#pragma unroll + for (int c = 0; c < 8; ++c) { + const sycl::int4 xv = xb[c * blocks_per_row]; +#pragma unroll + for (int u = 0; u < 4; ++u) { + si[c >> 1] = dpct::dp4a(q[4 * c + u], xv[u], si[c >> 1]); + } + } + float a = 0.0f; +#pragma unroll + for (int k = 0; k < 4; ++k) { + const sycl::float2 dsf = dsb[k * blocks_per_row].convert(); + a += si[k] * dsf.x() - dsf.y(); + } + return dd * a; + }; + +#pragma unroll + for (int j = 0; j < ncols_dst; ++j) { + const char * vy_j = (const char *) vy + j * stride_col_y_bytes; + const sycl::int4 * xb = (const sycl::int4 *) vy_j + (PROBE == 2 ? 0 : b); // PROBE 2: hot activations + const sycl::half2 * dsb = (const sycl::half2 *) (vy_j + ncols) + b; + float v; + if constexpr (PROBE == 3) { // dot only for column 0; later columns still load their activations + if (j == 0) { + v = dot_col(qw, xb, dsb, d); + } else { + int sink = 0; +#pragma unroll + for (int c = 0; c < 8; ++c) { + const sycl::int4 xv = xb[c * blocks_per_row]; + sink ^= xv[0] ^ xv[1] ^ xv[2] ^ xv[3]; + } + v = (float) sink * 1e-30f + (float) dsb[0].x() * 1e-30f; + } + } else { + v = dot_col(qw, xb, dsb, d); + } +#pragma unroll + for (int k = 0; k < R; ++k) { + acc[k][j] += r == k ? v : 0.0f; + } + if constexpr (has_fusion) { + const float vg = dot_col(qg, xb, dsb, dg); +#pragma unroll + for (int k = 0; k < R; ++k) { + accg[k][j] += r == k ? vg : 0.0f; + } + } + } + } + +#pragma unroll + for (int k = 0; k < R; ++k) { +#pragma unroll + for (int j = 0; j < ncols_dst; ++j) { + float sum = sycl::reduce_over_group(sg, acc[k][j], std::plus<>()); + if constexpr (has_fusion) { + const float gate = sycl::reduce_over_group(sg, accg[k][j], std::plus<>()); + sum *= glu_op == GGML_GLU_OP_SWIGLU ? op_silu(gate) : op_gelu(gate); + } + if (lane == 0 && row0 + k < nrows) { + dst[j * stride_col_dst + row0 + k] = sum; + } + } + } +} + +template +static void mul_mat_vec_ptq1_0_ncols_dec(const void * __restrict__ vx, const void * __restrict__ vgate, + const void * __restrict__ vy, float * __restrict__ dst, const int ncols, + const int nrows, const int stride_col_y_bytes, const int stride_col_dst, + const ggml_glu_op glu_op, const sycl::nd_item<3> & nd_item) { + const auto sg = nd_item.get_sub_group(); + const int row0 = (nd_item.get_group_linear_id() * sg.get_group_linear_range() + sg.get_group_linear_id()) * R; + if (row0 >= nrows) { + return; + } + ptq1_0_ncols_dec_body(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, + stride_col_dst, glu_op, sg, row0); +} + +static bool ptq1_0_ncols_dec_enabled() { + return ggml_sycl_ptq1_0_ncols_dec_enabled(); +} + +template +static void launch_ptq1_0_ncols_dec(const void * vx, const void * vgate, const void * vy, float * dst, const int ncols, + const int nrows, const int stride_col_y_bytes, const int stride_col_dst, + const ggml_glu_op glu_op, dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_PTQ1_0 == 0); + constexpr size_t num_subgroups = WARP_SIZE; + auto go = [&](auto rc) { + constexpr int R = decltype(rc)::value; + const int block_num_y = ceil_div(nrows, R * GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_ptq1_0_ncols_dec( + vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op, nd_item); + }); + }); + }; + static const int probe = getenv("GGML_SYCL_PTQ1_PROBE") ? atoi(getenv("GGML_SYCL_PTQ1_PROBE")) : 0; // lab timing only + if (!has_fusion && probe >= 1 && probe <= 3 && (ncols / QK_PTQ1_0) % WARP_SIZE != 0) { + auto launch_probe = [&](auto pc) { + using PC = decltype(pc); + const int block_num_y = ceil_div(nrows, 2 * GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_ptq1_0_ncols_dec<2, ncols_dst, false, PC::value>(vx, vgate, vy, dst, ncols, nrows, + stride_col_y_bytes, stride_col_dst, glu_op, nd_item); + }); + }); + }; + if (probe == 1) { + launch_probe(std::integral_constant{}); + } else if (probe == 2) { + launch_probe(std::integral_constant{}); + } else { + launch_probe(std::integral_constant{}); + } + return; + } + static const bool fused_r1 = getenv("GGML_SYCL_PTQ1_FUSED_R1") != nullptr; // ARC-LAB lab knob + if ((ncols / QK_PTQ1_0) % WARP_SIZE == 0 || (has_fusion && fused_r1)) { + go(std::integral_constant{}); + } else { + go(std::integral_constant{}); + } +} + +// ARC-LAB: the same-input merge (ggml_sycl_ptq1_0_mul_mat_vec_multi) for 2..8 columns (MTP verify / n-gram drafts): +// subgroups over the stacked rows of up to 3 matrices, each served by the decode-once body; activations in the ptq1_il +// layout, every dst contiguous (column stride = its row count). Row counts must be multiples of R. +template +static void mul_mat_vec_ptq1_0_ncols_dec_multi(const ptq1_0_multi_args a, const void * __restrict__ vy, const int ncols, + const int nrows_total, const int stride_col_y_bytes, + const sycl::nd_item<3> & nd_item) { + const auto sg = nd_item.get_sub_group(); + int row0 = (nd_item.get_group_linear_id() * sg.get_group_linear_range() + sg.get_group_linear_id()) * R; + if (row0 >= nrows_total) { + return; + } + int m = 0; + if (row0 >= a.nrows[0]) { + row0 -= a.nrows[0]; + m = 1; + if (row0 >= a.nrows[1]) { + row0 -= a.nrows[1]; + m = 2; + } + } + ptq1_0_ncols_dec_body(a.vx[m], nullptr, vy, a.dst[m], ncols, a.nrows[m], stride_col_y_bytes, + a.nrows[m], GGML_GLU_OP_SWIGLU, sg, row0); +} + +template +static void launch_ptq1_0_ncols_dec_multi(const ptq1_0_multi_args & a, const void * vy, const int ncols, const int total, + const int stride_col_y_bytes, const bool r1, dpct::queue_ptr stream) { + constexpr size_t num_subgroups = WARP_SIZE; + const ptq1_0_multi_args args = a; + auto go = [&](auto rc) { + constexpr int R = decltype(rc)::value; + const int block_num_y = ceil_div(total, R * GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_ptq1_0_ncols_dec_multi(args, vy, ncols, total, + stride_col_y_bytes, nd_item); + }); + }); + }; + if (r1) { + go(std::integral_constant{}); + } else { + go(std::integral_constant{}); + } +} + +bool ggml_sycl_ptq1_0_mul_mat_vec_multi_ncols(const void * const * vx, float * const * dst, const int * nrows, + const int nm, const void * vy, const int ncols, const int ncols_dst, + const int stride_col_y_bytes, dpct::queue_ptr stream) { + if (nm < 2 || nm > 3 || ncols % QK_PTQ1_0 != 0 || ncols_dst < 2 || ncols_dst > 8 || !ptq1_0_ncols_dec_enabled()) { + return false; + } + const bool r1 = (ncols / QK_PTQ1_0) % WARP_SIZE == 0; // as launch_ptq1_0_ncols_dec: 2 rows when lanes would idle + const int R = r1 ? 1 : 2; + ptq1_0_multi_args a = {}; + int total = 0; + for (int m = 0; m < nm; ++m) { + if (nrows[m] % R != 0) { + return false; + } + a.vx[m] = vx[m]; + a.dst[m] = dst[m]; + a.nrows[m] = nrows[m]; + total += nrows[m]; + } + switch (ncols_dst) { + case 2: launch_ptq1_0_ncols_dec_multi<2>(a, vy, ncols, total, stride_col_y_bytes, r1, stream); break; + case 3: launch_ptq1_0_ncols_dec_multi<3>(a, vy, ncols, total, stride_col_y_bytes, r1, stream); break; + case 4: launch_ptq1_0_ncols_dec_multi<4>(a, vy, ncols, total, stride_col_y_bytes, r1, stream); break; + case 5: launch_ptq1_0_ncols_dec_multi<5>(a, vy, ncols, total, stride_col_y_bytes, r1, stream); break; + case 6: launch_ptq1_0_ncols_dec_multi<6>(a, vy, ncols, total, stride_col_y_bytes, r1, stream); break; + case 7: launch_ptq1_0_ncols_dec_multi<7>(a, vy, ncols, total, stride_col_y_bytes, r1, stream); break; + case 8: launch_ptq1_0_ncols_dec_multi<8>(a, vy, ncols, total, stride_col_y_bytes, r1, stream); break; + } + return true; +} + +template +static void reorder_mul_mat_vec_ptq1_0_q8_1_sycl_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_PTQ1_0 == 0); + if (ptq1_0_ncols_dec_enabled()) { + launch_ptq1_0_ncols_dec(vx, nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, + GGML_GLU_OP_SWIGLU, stream); + return; + } + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_reorder_ncols, ncols_dst>( + vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, + /*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item); + }); + }); +} + +static void reorder_mul_mat_vec_ptq1_0_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + switch (ncols_dst) { + case 1: reorder_mul_mat_vec_ptq1_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; + case 2: reorder_mul_mat_vec_ptq1_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 3: reorder_mul_mat_vec_ptq1_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 4: reorder_mul_mat_vec_ptq1_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 5: reorder_mul_mat_vec_ptq1_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 6: reorder_mul_mat_vec_ptq1_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 7: reorder_mul_mat_vec_ptq1_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 8: reorder_mul_mat_vec_ptq1_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + default: GGML_ABORT("unsupported ncols_dst=%d for PTQ1_0 reorder multi-col MMVQ", ncols_dst); + } +} + static void mul_mat_vec_q4_0_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, dpct::queue_ptr stream) { GGML_ASSERT(ncols % QK4_0 == 0); @@ -1960,8 +2707,198 @@ static void reorder_mul_mat_vec_q5_k_q8_1_sycl_switch_ncols( } } +// ARC-LAB: Q6_K reordered single-column mat-vec with more loads in flight (GGML_SYCL_Q6K_GEMV=1; the MTP draft head's +// eh_proj / q / wo / ffn weights are Q6_K and ran at ~70% of the B580's bandwidth). Same arithmetic as +// mul_mat_vec_q_reorder>: one sub-group per row, lane l handles quant ints l and l + 16 of +// every block, blocks in order, `partial_sum += vec_dot_q6_K_q8_1_impl_mmvq_scalar(...)` per (block, int) in the same +// order, then the same sub-group reduction. What changes: +// - U blocks' operands are loaded before any of them is used, so U x 2 groups of loads are in flight per lane instead +// of the ones of one block (the loop body was latency bound); +// - ql / qh words are read as aligned 32-bit loads (the generic helper assembles them from two 16-bit loads; the +// reordered planes are 4-byte aligned, so the int is the same); +// - the work-group holds GGML_SYCL_Q6K_SG sub-groups (rows) instead of 16. +// GGML_SYCL_Q6K_U (1, 2, 4; default 2) and GGML_SYCL_Q6K_SG (4, 8, 16; default 8) are runtime knobs for a sweep. +// GGML_SYCL_Q6K_CHECK=1 also runs the original kernel and counts rows whose result differs in any bit (stderr, every +// 256 launches): the identity check for the GPU run. +namespace { +struct q6k_ops { + int vl, vh, u0, u1; + int sc0, sc1; + float d80, d81; +}; + +static __dpct_inline__ void q6k_load(const uint8_t * __restrict__ base, const int8_t * __restrict__ q8, + const sycl::half2 * __restrict__ ds, const int ql_byte, const int qh_byte, + const int sc_byte, const int iqs, q6k_ops & o) { + const int bq8_offset = 2 * QR6_K * (iqs / (QI6_K / 2)) + (iqs % (QI6_K / 2)) / (QI6_K / 4); + const int scale_offset = (QI6_K / 4) * (iqs / (QI6_K / 2)) + (iqs % (QI6_K / 2)) / (QI6_K / 8); + const int vh_shift = 2 * ((iqs % (QI6_K / 2)) / (QI6_K / 4)); + const int qh_int = (QI6_K / 4) * (iqs / (QI6_K / 2)) + iqs % (QI6_K / 4); + + o.vl = *(const int *) (base + ql_byte + 4 * iqs); + o.vh = *(const int *) (base + qh_byte + 4 * qh_int) >> vh_shift; + const int8_t * scs = (const int8_t *) (base + sc_byte) + scale_offset; + o.sc0 = scs[0]; + o.sc1 = scs[4]; + o.u0 = *(const int *) (q8 + bq8_offset * QK8_1 + 4 * (iqs % QI8_1)); + o.u1 = *(const int *) (q8 + (bq8_offset + 2) * QK8_1 + 4 * (iqs % QI8_1)); + o.d80 = ds[bq8_offset + 0][0]; + o.d81 = ds[bq8_offset + 2][0]; +} +} // namespace + +template +static void mul_mat_vec_q6_k_reorder_pf(const void * __restrict__ vx, const void * __restrict__ vy, + float * __restrict__ dst, const int ncols, const int nrows, + const sycl::nd_item<3> & nd_item) { + using block_type = ggml_sycl_reordered::block_q_t; + static_assert(QI6_K == 2 * WARP_SIZE, "two quant ints per lane per block"); + + const auto sg = nd_item.get_sub_group(); + const int row = nd_item.get_group_linear_id() * sg.get_group_linear_range() + sg.get_group_linear_id(); + if (row >= nrows) { + return; + } + const int lane = sg.get_local_linear_id(); + + const int blocks_per_row = ncols / QK_K; + const int nblocks = nrows * blocks_per_row; + const uint8_t * base = (const uint8_t *) vx; + + float partial_sum = 0.0f; + int i = 0; + for (; i + U <= blocks_per_row; i += U) { + q6k_ops o[U][2]; + float d[U]; +#pragma unroll + for (int u = 0; u < U; ++u) { + const int ibx = row * blocks_per_row + i + u; + const auto bx = block_type::get_block_offset(ibx, nblocks); + const auto doff = block_type::get_d_offset(nrows, ncols, ibx); + const int iby = (i + u) * block_type::block_to_q8_1_ratio(); + const int8_t * q8 = (const int8_t *) vy + iby * QK8_1; + const sycl::half2 * ds = (const sycl::half2 *) ((const char *) vy + ncols + iby * sizeof(sycl::half2)); + d[u] = *(const ggml_half *) (base + doff.second); +#pragma unroll + for (int e = 0; e < 2; ++e) { + q6k_load(base, q8, ds, bx.first, bx.second, doff.first, e * WARP_SIZE + lane, o[u][e]); + } + } +#pragma unroll + for (int u = 0; u < U; ++u) { +#pragma unroll + for (int e = 0; e < 2; ++e) { + partial_sum += vec_dot_q6_K_q8_1_impl_mmvq_scalar(o[u][e].vl, o[u][e].vh, o[u][e].u0, o[u][e].u1, + o[u][e].sc0, o[u][e].sc1, d[u], o[u][e].d80, + o[u][e].d81); + } + } + } + for (; i < blocks_per_row; ++i) { + const int ibx = row * blocks_per_row + i; + const auto bx = block_type::get_block_offset(ibx, nblocks); + const auto doff = block_type::get_d_offset(nrows, ncols, ibx); + const int iby = i * block_type::block_to_q8_1_ratio(); + const int8_t * q8 = (const int8_t *) vy + iby * QK8_1; + const sycl::half2 * ds = (const sycl::half2 *) ((const char *) vy + ncols + iby * sizeof(sycl::half2)); + const float d = *(const ggml_half *) (base + doff.second); +#pragma unroll + for (int e = 0; e < 2; ++e) { + q6k_ops o; + q6k_load(base, q8, ds, bx.first, bx.second, doff.first, e * WARP_SIZE + lane, o); + partial_sum += vec_dot_q6_K_q8_1_impl_mmvq_scalar(o.vl, o.vh, o.u0, o.u1, o.sc0, o.sc1, d, o.d80, o.d81); + } + } + + const float sum = sycl::reduce_over_group(sg, partial_sum, std::plus<>()); + if (sg.leader()) { + dst[row] = sum; + } +} + +template +static void launch_q6_k_reorder_pf(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, + const int n_sg, dpct::queue_ptr stream) { + const sycl::range<3> block_nums(1, 1, ceil_div(nrows, n_sg)); + const sycl::range<3> block_dims(1, 1, (size_t) n_sg * WARP_SIZE); + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q6_k_reorder_pf(vx, vy, dst, ncols, nrows, nd_item); + }); + }); +} + +static void reorder_mul_mat_vec_q6_k_q8_1_sycl_orig(const void * vx, const void * vy, float * dst, const int ncols, + const int nrows, dpct::queue_ptr stream); + +// returns false when the tuned kernel is off (GGML_SYCL_Q6K_GEMV=0) +static bool reorder_mul_mat_vec_q6_k_q8_1_sycl_tuned(const void * vx, const void * vy, float * dst, const int ncols, + const int nrows, dpct::queue_ptr stream) { + if (!ggml_sycl_knob("GGML_SYCL_Q6K_GEMV", 0)) { + return false; + } + const int u = ggml_sycl_knob("GGML_SYCL_Q6K_U", 2); + int n_sg = ggml_sycl_knob("GGML_SYCL_Q6K_SG", 8); + if (n_sg != 4 && n_sg != 8 && n_sg != 16) { + n_sg = 8; + } + switch (u) { + case 1: launch_q6_k_reorder_pf<1>(vx, vy, dst, ncols, nrows, n_sg, stream); break; + case 4: launch_q6_k_reorder_pf<4>(vx, vy, dst, ncols, nrows, n_sg, stream); break; + default: launch_q6_k_reorder_pf<2>(vx, vy, dst, ncols, nrows, n_sg, stream); break; + } + + if (ggml_sycl_knob("GGML_SYCL_Q6K_CHECK", 0)) { + // the original kernel into a scratch row vector, then a bitwise compare (debug only: allocates and waits) + static float * ref = nullptr; + static int ref_n = 0; + static int * n_diff = nullptr; + static long launches = 0, rows = 0, diffs = 0; + if (ref_n < nrows) { + if (ref) { + sycl::free(ref, *stream); + } + ref = sycl::malloc_device(nrows, *stream); + ref_n = nrows; + } + if (!n_diff) { + n_diff = sycl::malloc_shared(1, *stream); + } + reorder_mul_mat_vec_q6_k_q8_1_sycl_orig(vx, vy, ref, ncols, nrows, stream); + *n_diff = 0; + stream->wait(); + int * nd = n_diff; + float * r = ref; + stream->parallel_for(sycl::range<1>(nrows), [=](sycl::id<1> k) { + const uint32_t a = sycl::bit_cast(dst[k]); + const uint32_t b = sycl::bit_cast(r[k]); + if (a != b) { + sycl::atomic_ref c(*nd); + c.fetch_add(1); + } + }).wait(); + launches++; + rows += nrows; + diffs += *n_diff; + if (launches % 256 == 0) { + fprintf(stderr, "q6k-check: %ld launches, %ld rows, %ld rows differ (U=%d, SG=%d)\n", launches, rows, diffs, u, + n_sg); + } + } + return true; +} + static void reorder_mul_mat_vec_q6_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, dpct::queue_ptr stream) { + if (reorder_mul_mat_vec_q6_k_q8_1_sycl_tuned(vx, vy, dst, ncols, nrows, stream)) { + return; + } + reorder_mul_mat_vec_q6_k_q8_1_sycl_orig(vx, vy, dst, ncols, nrows, stream); +} + +static void reorder_mul_mat_vec_q6_k_q8_1_sycl_orig(const void * vx, const void * vy, float * dst, const int ncols, + const int nrows, dpct::queue_ptr stream) { GGML_ASSERT(ncols % QK_K == 0); // Round up to a whole number of subgroup-sized workgroups; out-of-range rows are skipped inside the kernel. constexpr size_t num_subgroups = WARP_SIZE; @@ -2461,22 +3398,36 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tens mul_mat_vec_q1_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); } break; - case GGML_TYPE_PTQ1_0: + case GGML_TYPE_Q2_0: if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) { const int stride_col_y = src1_padded_col_size / QK8_1; const int stride_col_dst = dst->ne[0]; - GGML_SYCL_DEBUG("Calling mul_mat_vec_ptq1_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols); - mul_mat_vec_ptq1_0_q8_1_sycl_switch_ncols( + GGML_SYCL_DEBUG("Calling mul_mat_vec_q2_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols); + mul_mat_vec_q2_0_q8_1_sycl_switch_ncols( src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, stride_col_y, stride_col_dst, stream); return; } else if (i == 0 || src1_ncols == 1) { - GGML_SYCL_DEBUG("Calling mul_mat_vec_ptq1_0_q8_1_sycl\n"); - mul_mat_vec_ptq1_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); + GGML_SYCL_DEBUG("Calling mul_mat_vec_q2_0_q8_1_sycl\n"); + mul_mat_vec_q2_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); } break; case GGML_TYPE_PQ2_0: - if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) { + if ((ggml_tensor_extra_gpu *) dst->src[0]->extra && + ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { + if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) { + const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs; + const int stride_col_dst = dst->ne[0]; + GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_pq2_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols); + reorder_mul_mat_vec_pq2_0_q8_1_sycl_switch_ncols( + src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, + src1_ncols, stride_col_y_bytes, stride_col_dst, stream); + return; + } else { + GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_pq2_0_q8_1_sycl\n"); + reorder_mul_mat_vec_pq2_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); + } + } else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) { const int stride_col_y = src1_padded_col_size / QK8_1; const int stride_col_dst = dst->ne[0]; GGML_SYCL_DEBUG("Calling mul_mat_vec_pq2_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols); @@ -2489,18 +3440,32 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tens mul_mat_vec_pq2_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); } break; - case GGML_TYPE_Q2_0: - if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) { + case GGML_TYPE_PTQ1_0: + if ((ggml_tensor_extra_gpu *) dst->src[0]->extra && + ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { + if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) { + const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs; + const int stride_col_dst = dst->ne[0]; + GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_ptq1_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols); + reorder_mul_mat_vec_ptq1_0_q8_1_sycl_switch_ncols( + src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, + src1_ncols, stride_col_y_bytes, stride_col_dst, stream); + return; + } else { + GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_ptq1_0_q8_1_sycl\n"); + reorder_mul_mat_vec_ptq1_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); + } + } else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) { const int stride_col_y = src1_padded_col_size / QK8_1; const int stride_col_dst = dst->ne[0]; - GGML_SYCL_DEBUG("Calling mul_mat_vec_q2_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols); - mul_mat_vec_q2_0_q8_1_sycl_switch_ncols( + GGML_SYCL_DEBUG("Calling mul_mat_vec_ptq1_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols); + mul_mat_vec_ptq1_0_q8_1_sycl_switch_ncols( src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, stride_col_y, stride_col_dst, stream); return; } else if (i == 0 || src1_ncols == 1) { - GGML_SYCL_DEBUG("Calling mul_mat_vec_q2_0_q8_1_sycl\n"); - mul_mat_vec_q2_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); + GGML_SYCL_DEBUG("Calling mul_mat_vec_ptq1_0_q8_1_sycl\n"); + mul_mat_vec_ptq1_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); } break; case GGML_TYPE_Q2_K: @@ -2992,7 +3957,7 @@ static void launch_mul_mat_vec_q_reorder_glu(const void * vx, const void * vgate const int ncols, const int nrows, const int stride_col_y_bytes, const int stride_col_dst, const ggml_glu_op glu_op, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); + GGML_ASSERT(ncols % ggml_sycl_reordered::block_q_t::traits::qk == 0); constexpr size_t num_subgroups = WARP_SIZE; @@ -3010,18 +3975,11 @@ static void launch_mul_mat_vec_q_reorder_glu(const void * vx, const void * vgate }); } -bool ggml_sycl_mul_mat_vec_q_glu_reorder(enum ggml_type src0_type, enum ggml_glu_op glu_op, const void * vx, - const void * vgate, const void * vy, float * dst, int ncols, int nrows, - int ncols_dst, int stride_col_y_bytes, int stride_col_dst, - dpct::queue_ptr stream) { - if (src0_type != GGML_TYPE_Q4_K) { - return false; - } - if (glu_op != GGML_GLU_OP_SWIGLU && glu_op != GGML_GLU_OP_GEGLU) { - return false; - } - - using vec_dot = reorder_vec_dot_q_sycl; +template +static bool mul_mat_vec_q_glu_reorder_type(enum ggml_glu_op glu_op, const void * vx, const void * vgate, const void * vy, + float * dst, int ncols, int nrows, int ncols_dst, int stride_col_y_bytes, + int stride_col_dst, dpct::queue_ptr stream) { + using vec_dot = reorder_vec_dot_q_sycl; switch (ncols_dst) { case 1: @@ -3059,4 +4017,160 @@ bool ggml_sycl_mul_mat_vec_q_glu_reorder(enum ggml_type src0_type, enum ggml_glu default: return false; } + return false; +} + +bool ggml_sycl_mul_mat_vec_q_glu_reorder(enum ggml_type src0_type, enum ggml_glu_op glu_op, const void * vx, + const void * vgate, const void * vy, float * dst, int ncols, int nrows, + int ncols_dst, int stride_col_y_bytes, int stride_col_dst, + dpct::queue_ptr stream) { + if (glu_op != GGML_GLU_OP_SWIGLU && glu_op != GGML_GLU_OP_GEGLU) { + return false; + } + switch (src0_type) { + case GGML_TYPE_Q4_K: + return mul_mat_vec_q_glu_reorder_type(glu_op, vx, vgate, vy, dst, ncols, nrows, ncols_dst, + stride_col_y_bytes, stride_col_dst, stream); + case GGML_TYPE_PQ2_0: + return mul_mat_vec_q_glu_reorder_type(glu_op, vx, vgate, vy, dst, ncols, nrows, ncols_dst, + stride_col_y_bytes, stride_col_dst, stream); + case GGML_TYPE_Q6_K: // ARC-LAB: GGML_SYCL_Q6K_GLU (fusion.cpp) + return mul_mat_vec_q_glu_reorder_type(glu_op, vx, vgate, vy, dst, ncols, nrows, ncols_dst, + stride_col_y_bytes, stride_col_dst, stream); + case GGML_TYPE_PTQ1_0: + if (ncols_dst == 1 && ggml_sycl_ptq1_0_glu_n1_enabled()) { + static const int glu_nr = getenv("GGML_SYCL_PTQ1_GLU1_NR") ? atoi(getenv("GGML_SYCL_PTQ1_GLU1_NR")) : 2; + if (glu_nr == 4) { + launch_ptq1_0_pairs_glu<4>(vx, vgate, vy, dst, ncols, nrows, glu_op, stream); + } else { + launch_ptq1_0_pairs_glu<2>(vx, vgate, vy, dst, ncols, nrows, glu_op, stream); + } + return true; + } + if (ptq1_0_ncols_dec_enabled() && ncols_dst >= 1 && ncols_dst <= 8) { +#define PTQ1_DEC_GLU(N) case N: launch_ptq1_0_ncols_dec(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, \ + stride_col_dst, glu_op, stream); return true; + switch (ncols_dst) { + PTQ1_DEC_GLU(1) PTQ1_DEC_GLU(2) PTQ1_DEC_GLU(3) PTQ1_DEC_GLU(4) + PTQ1_DEC_GLU(5) PTQ1_DEC_GLU(6) PTQ1_DEC_GLU(7) PTQ1_DEC_GLU(8) + } +#undef PTQ1_DEC_GLU + } + return mul_mat_vec_q_glu_reorder_type(glu_op, vx, vgate, vy, dst, ncols, nrows, ncols_dst, + stride_col_y_bytes, stride_col_dst, stream); + default: + return false; + } +} + +// ARC-LAB: reordered q4_0 x f32 for 5..64 columns (speculative verify / n-gram drafts) on XMX via s8 x s4 DPAS +// (inline vISA, xe2-dpas.hpp; Xe2). The MMVQ column kernels re-read the weights per 8 columns and the dequantize +// + GEMM path converts the whole matrix; here each weight row streams once for all columns. One lane per weight row: B = +// the row's 16 q4_0 bytes of a block XOR 0x88888888 (u4 n+8 -> s4 n), A = 8 activation rows as int8 (per row and 32-block +// RNE scale; q4_0 byte i holds elements i and i+16, so A's k = 2i+h pairs elements l and l+16). GGML_SYCL_Q4_0_DPAS=1. +namespace { +typedef short q4d_short8 __attribute__((ext_vector_type(8))); +typedef int q4d_int4 __attribute__((ext_vector_type(4))); +typedef int q4d_int8 __attribute__((ext_vector_type(8))); +typedef float q4d_float8 __attribute__((ext_vector_type(8))); +typedef unsigned q4d_uint4 __attribute__((ext_vector_type(4))); +} + +template +static void q4_0_dpas_gemm_launch(const void * vx, const q4d_short8 * xq, const q4d_float8 * xd, float * dst, const int nrows, + const int ncols, const int ntok, dpct::queue_ptr stream) { + const int NB = ncols / QK4_0; + constexpr int WG = 256; + const size_t nsg = (size_t) nrows / 16; + const size_t gsz = (nsg + WG / 16 - 1) / (WG / 16) * WG; + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<1>(gsz, WG), [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(16)]] { + const auto sg = it.get_sub_group(); + const int lane = sg.get_local_linear_id(); + const int sgi = (int) (it.get_group(0) * (WG / 16) + sg.get_group_linear_id()); + if (sgi * 16 >= nrows) { + return; + } + const int n = sgi * 16 + lane; // this lane's weight row + const uint8_t * qs = (const uint8_t *) vx + (size_t) n * NB * (QK4_0 / 2); + const sycl::half * dw = (const sycl::half *) ((const uint8_t *) vx + (size_t) ncols / 2 * nrows) + (size_t) n * NB; + q4d_float8 acc[MT]; +#pragma unroll + for (int t = 0; t < MT; ++t) { + acc[t] = 0.0f; + } + for (int b = 0; b < NB; ++b) { + const q4d_uint4 q = *(const q4d_uint4 *) (qs + b * (QK4_0 / 2)); + const q4d_int4 bv = { (int) (q[0] ^ 0x88888888u), (int) (q[1] ^ 0x88888888u), + (int) (q[2] ^ 0x88888888u), (int) (q[3] ^ 0x88888888u) }; + const float d = static_cast(dw[b]); +#pragma unroll + for (int t = 0; t < MT; ++t) { + const q4d_int8 ia = xe2dp::dpas_s4s8_r8(xq[((size_t) t * NB + b) * 16 + lane], bv); + acc[t] += __builtin_convertvector(ia, q4d_float8) * (xd[(size_t) t * NB + b] * d); + } + } +#pragma unroll + for (int t = 0; t < MT; ++t) { +#pragma unroll + for (int m = 0; m < 8; ++m) { + const int tok = t * 8 + m; + if (tok < ntok) { + dst[(size_t) tok * nrows + n] = acc[t][m]; + } + } + } + }); + }); +} + +bool ggml_sycl_q4_0_dpas_gemm(ggml_backend_sycl_context & ctx, const void * vx, const float * x, int64_t x_stride, + float * dst, const int nrows, const int ncols, const int ntok, dpct::queue_ptr stream) { + if (ntok < 1 || nrows % 16 != 0 || ncols % QK4_0 != 0) { + return false; + } + if (ntok > 64) { // longer n-gram verify batches: chunks of 64 columns, the weights stream once per chunk + for (int c0 = 0; c0 < ntok; c0 += 64) { + if (!ggml_sycl_q4_0_dpas_gemm(ctx, vx, x + (size_t) c0 * x_stride, x_stride, dst + (size_t) c0 * nrows, nrows, + ncols, std::min(64, ntok - c0), stream)) { + return false; + } + } + return true; + } + const int MT = (ntok + 7) / 8, NB = ncols / QK4_0; + ggml_sycl_pool_alloc xq(ctx.pool(), (size_t) MT * NB * 16); + ggml_sycl_pool_alloc xd(ctx.pool(), (size_t) MT * NB); + q4d_short8 * xq_p = xq.get(); + q4d_float8 * xd_p = xd.get(); + // activations -> int8 A layout: [tile][block][lane] -> 8 rows, one scale per row and block; padded rows are zero + stream->parallel_for(sycl::range<1>((size_t) MT * 8 * NB), [=](sycl::item<1> it) { + const int e = (int) it[0], m = e / NB, b = e % NB, t = m / 8, mm = m % 8; + float v[QK4_0]; + float amax = 0.0f; +#pragma unroll + for (int i = 0; i < QK4_0; ++i) { + v[i] = m < ntok ? x[(size_t) m * x_stride + b * QK4_0 + i] : 0.0f; + amax = sycl::fmax(amax, sycl::fabs(v[i])); + } + const float iq = amax > 0.0f ? 127.0f / amax : 0.0f; + ((float *) &xd_p[(size_t) t * NB + b])[mm] = amax / 127.0f; + short * row = (short *) &xq_p[((size_t) t * NB + b) * 16]; +#pragma unroll + for (int l = 0; l < 16; ++l) { + const int lo = (int) sycl::rint(v[l] * iq), hi = (int) sycl::rint(v[l + 16] * iq); + row[l * 8 + mm] = (short) ((lo & 0xFF) | ((hi & 0xFF) << 8)); + } + }); + switch (MT) { + case 1: q4_0_dpas_gemm_launch<1>(vx, xq_p, xd_p, dst, nrows, ncols, ntok, stream); break; + case 2: q4_0_dpas_gemm_launch<2>(vx, xq_p, xd_p, dst, nrows, ncols, ntok, stream); break; + case 3: q4_0_dpas_gemm_launch<3>(vx, xq_p, xd_p, dst, nrows, ncols, ntok, stream); break; + case 4: q4_0_dpas_gemm_launch<4>(vx, xq_p, xd_p, dst, nrows, ncols, ntok, stream); break; + case 5: q4_0_dpas_gemm_launch<5>(vx, xq_p, xd_p, dst, nrows, ncols, ntok, stream); break; + case 6: q4_0_dpas_gemm_launch<6>(vx, xq_p, xd_p, dst, nrows, ncols, ntok, stream); break; + case 7: q4_0_dpas_gemm_launch<7>(vx, xq_p, xd_p, dst, nrows, ncols, ntok, stream); break; + default: q4_0_dpas_gemm_launch<8>(vx, xq_p, xd_p, dst, nrows, ncols, ntok, stream); break; + } + return true; } diff --git a/ggml/src/ggml-sycl/mmvq.hpp b/ggml/src/ggml-sycl/mmvq.hpp index 9d2f5645ecf3..1dd14b46e0af 100644 --- a/ggml/src/ggml-sycl/mmvq.hpp +++ b/ggml/src/ggml-sycl/mmvq.hpp @@ -73,4 +73,18 @@ bool ggml_sycl_mul_mat_vec_q_glu_reorder( int stride_col_dst, // floats between output columns in dst dpct::queue_ptr stream); +// ARC-LAB: PTQ1_0 decode-once multi-column mat-vec on (GGML_SYCL_PTQ1_NCOLS_DEC_OFF unset); its activations must be +// quantized with quantize_and_reorder_q8_1_ptq1_il (plain path: 2..8 columns; fused gate/up path: 1..8 columns) +bool ggml_sycl_ptq1_0_ncols_dec_enabled(); +// ARC-LAB: single-column PTQ1_0 helpers (pairs kernel) +bool ggml_sycl_ptq1_0_glu_n1_enabled(); +bool ggml_sycl_ptq1_0_mul_mat_vec_multi(const void * const * vx, float * const * dst, const int * nrows, const int nm, + const void * vy, const int ncols, dpct::queue_ptr stream); +bool ggml_sycl_ptq1_0_mul_mat_vec_multi_ncols(const void * const * vx, float * const * dst, const int * nrows, + const int nm, const void * vy, const int ncols, const int ncols_dst, + const int stride_col_y_bytes, dpct::queue_ptr stream); + +// ARC-LAB: reordered q4_0 x f32, 1..64 columns, weights streamed once on XMX (s8 x s4 DPAS, Xe2) +bool ggml_sycl_q4_0_dpas_gemm(ggml_backend_sycl_context & ctx, const void * vx, const float * x, int64_t x_stride, + float * dst, int nrows, int ncols, int ntok, dpct::queue_ptr stream); #endif // GGML_SYCL_MMVQ_HPP diff --git a/ggml/src/ggml-sycl/ptq1-t2.cpp b/ggml/src/ggml-sycl/ptq1-t2.cpp new file mode 100644 index 000000000000..7fb12a6e951f --- /dev/null +++ b/ggml/src/ggml-sycl/ptq1-t2.cpp @@ -0,0 +1,847 @@ +// ARC-LAB: PTQ1_0 on XMX through the TernSYCL int2 x int8 DPAS kernels (ternsycl/, BSD 3-Clause, libxsmm/TernSYCL +// @8b0ae42: inline vISA only, no IGC builtins; builds with icpx's default -fp-model=fast). The base-3 PTQ1_0 packing (1.625 + 0.125 bits/weight) has to be decoded on the ALUs before any DPAS, which +// makes the 4-column verify mat-vec compute-bound (~190 GB/s on the B580); 2-bit two's-complement codes (2.125 bits) +// feed the s8 x s2 DPAS directly. B580 standalone, 2 GiB rotating weights: 17408x5120 m=1 56 us, m=4 61 us (vs 70 / +// ~100 in-model for PTQ1_0). The weight grows 31%; activations become int8 with one fp16 scale per 128 (q8_1 is per 32). +#include "ptq1-t2.hpp" +#include "common.hpp" + +#include "ternsycl/int2_int8_dpas.hpp" + +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr int QK = 128; +constexpr int64_t BLK_BYTES = 28; // block_ptq1_0: qs[24], qh[2], fp16 d + +int t2_mode() { + static const int mode = [] { + const char * e = getenv("GGML_SYCL_PTQ1_T2"); + if (!e) { + return 0; + } + if (!strcmp(e, "all")) { + return 2; + } + if (!strcmp(e, "ffn")) { + return 1; + } + return 0; + }(); + return mode; +} + +// growing per-process scratch for the int8 activations and their scales (in-order queue: reuse is safe) +struct scratch { + void * p = nullptr; + size_t bytes = 0; +}; + +void * scratch_get(sycl::queue & q, size_t bytes) { + static std::mutex mtx; + static scratch s; + std::lock_guard lock(mtx); + if (bytes > s.bytes) { + if (s.p) { + q.wait(); + sycl::free(s.p, q); + } + s.bytes = bytes + bytes / 4; + s.p = sycl::malloc_device(s.bytes, q); + if (!s.p) { + fprintf(stderr, "ptq1-t2: no device memory for %zu bytes of activation scratch\n", s.bytes); + abort(); + } + } + return s.p; +} + +} // namespace + +// the kernels use 16-lane DPAS (Xe2 and newer); Xe-LPG(+) / Alchemist XMX is 8 lanes and has no such kernels, so fall back +// to the regular path with one warning instead of failing at the first launch. GGML_SYCL_PTQ1_T2_ANYGPU=1 skips the check. +bool ggml_sycl_device_is_xe2() { + const auto & hw = ggml_sycl_info().devices[ggml_sycl_get_device()].hw_info; + return hw.arch == gpu_arch::intel_gpu_bmg_g21 || hw.arch == gpu_arch::intel_gpu_bmg_g31 || + hw.arch == gpu_arch::intel_gpu_lnl_m || hw.arch == gpu_arch::intel_gpu_ptl_h || + hw.arch == gpu_arch::intel_gpu_ptl_u || hw.arch == gpu_arch::intel_gpu_wcl; +} + +// Xe3 (Panther Lake, Wildcat Lake), for the defaults measured on an Arc B390 +bool ggml_sycl_device_is_xe3(int device) { + const auto & hw = ggml_sycl_info().devices[device].hw_info; + return hw.arch == gpu_arch::intel_gpu_ptl_h || hw.arch == gpu_arch::intel_gpu_ptl_u || + hw.arch == gpu_arch::intel_gpu_wcl; +} + +static bool t2_device_ok() { + static const bool ok = [] { + if (getenv("GGML_SYCL_PTQ1_T2_ANYGPU")) { + return true; + } + const auto & hw = ggml_sycl_info().devices[ggml_sycl_get_device()].hw_info; + const bool xe2 = ggml_sycl_device_is_xe2(); + if (!xe2) { + GGML_LOG_WARN("%s: GGML_SYCL_PTQ1_T2 needs an Xe2 or newer GPU (Arc B-series, Lunar Lake, Panther Lake); " + "%s is not one, using the regular path\n", __func__, hw.name.c_str()); + } + return xe2; + }(); + return ok; +} + +bool ggml_sycl_t2_wants(const char * name, int64_t K, int64_t N) { + const int mode = t2_mode(); + if (mode == 0 || K % QK != 0 || N % 16 != 0 || !t2_device_ok()) { + return false; + } + if (mode == 2) { + return true; + } + return name && (strstr(name, "ffn_gate.") || strstr(name, "ffn_up.") || strstr(name, "ffn_down.")); +} + +size_t ggml_sycl_t2_bytes(int64_t K, int64_t N) { + return (size_t) (K / 16) * N * 4 + (size_t) (K / QK) * N * 2; +} + +namespace { +void t2_precompile(sycl::queue & q); +} + +bool ggml_sycl_t2_repack(sycl::queue & q, void * data, int64_t K, int64_t N, bool pq2) { + static std::once_flag compiled; + std::call_once(compiled, [&] { t2_precompile(q); }); + const int64_t nb = K / QK; + const int64_t blk_bytes = pq2 ? 34 : BLK_BYTES; // block_pq2_0: fp16 d, then 32 bytes of 2-bit codes + const size_t src_bytes = (size_t) N * nb * blk_bytes; + uint8_t * tmp = (uint8_t *) sycl::malloc_device(src_bytes, q); + if (!tmp) { + fprintf(stderr, "%s: no device memory for a %zu-byte repack buffer\n", __func__, src_bytes); + abort(); + } + q.memcpy(tmp, data, src_bytes).wait(); + uint32_t * B = (uint32_t *) data; + uint16_t * SB = (uint16_t *) ((char *) data + (size_t) (K / 16) * N * 4); + int * bad = sycl::malloc_device(1, q); + q.memset(bad, 0, sizeof(int)).wait(); + q.parallel_for(sycl::range<2>((size_t) (K / 16), (size_t) N), [=](sycl::item<2> it) { + const int64_t kp = it[0], n = it[1]; + const int64_t g = kp / 8; + const uint8_t * b = tmp + (n * nb + g) * blk_bytes; + const int e0 = (int) (kp % 8) * 16; + uint32_t w = 0; + for (int j = 0; j < 16; ++j) { + const int e = e0 + j; + uint32_t digit; // weight = digit - 1 + if (pq2) { + digit = (b[2 + e / 4] >> (2 * (e % 4))) & 3u; + if (digit == 3u) { + *bad = 1; + digit = 2u; + } + } else { + uint32_t v; + int lvl; + if (e < 80) { + v = b[e % 16]; + lvl = e / 16; + } else if (e < 120) { + v = b[16 + (e - 80) % 8]; + lvl = (e - 80) / 8; + } else { + v = b[24 + (e - 120) % 2]; + lvl = (e - 120) / 2; + } + digit = 0; + for (int t = 0; t <= lvl; ++t) { // base-3 fixed-point digits, as ptq1_0_decode_block + const uint32_t x = v * 3; + digit = x >> 8; + v = x & 0xFF; + } + } + const uint32_t code = digit == 2 ? 1u : (digit == 1 ? 0u : 3u); // 2-bit two's complement of digit - 1 + w |= code << (2 * j); + } + B[kp * N + n] = w; + if (kp % 8 == 0) { + SB[g * N + n] = pq2 ? (uint16_t) (b[0] | (b[1] << 8)) : (uint16_t) (b[26] | (b[27] << 8)); + } + }).wait(); + int bad_h = 0; + q.memcpy(&bad_h, bad, sizeof(int)).wait(); + sycl::free(bad, q); + sycl::free(tmp, q); + // ARC-LAB experiment: GGML_SYCL_T2_ZERO_WEIGHTS=1 zeroes every T2 weight after the repack (output is garbage). The Xe2 + // memory compression then makes weight reads almost free, so a profile shows everything else a token costs (2026-10-05). + static const bool zero_w = getenv("GGML_SYCL_T2_ZERO_WEIGHTS") && atoi(getenv("GGML_SYCL_T2_ZERO_WEIGHTS")); + if (zero_w) { + q.memset(B, 0, (size_t) (K / 16) * N * 4).wait(); + } + return bad_h == 0; +} + +namespace { + +// fp32 activations -> int8 Aq [M, K] + fp16 SA [K/128, ldsa(M)] (TernSYCL QMODE 0: SA = 127 / absmax, saturate) +// G = 128 (one scale per 16 lanes x 8 values) or 32 (a scale per 4 lanes); SA [K/G, ldsa(M)] +void quant_a(sycl::queue & q, const float * x, int64_t x_stride, int64_t M, int64_t K, int8_t * Aq, uint16_t * SA, + const int G = QK) { + const int lda = int8dpas::ldsa((int) M); + q.parallel_for(sycl::nd_range<2>({ (size_t) M, (size_t) (K / QK) * 16 }, { 1, 16 }), + [=](sycl::nd_item<2> it) [[sycl::reqd_sub_group_size(16)]] { + const auto sg = it.get_sub_group(); + const int lane = sg.get_local_linear_id(); + const int64_t m = it.get_global_id(0); + const int64_t g = it.get_global_id(1) / 16; + const float * a = x + m * x_stride + g * QK + 8 * lane; + float v[8], mx = 0.0f; +#pragma unroll + for (int i = 0; i < 8; ++i) { + v[i] = a[i]; + mx = sycl::fmax(mx, sycl::fabs(v[i])); + } + if (G == QK) { + mx = sycl::reduce_over_group(sg, mx, sycl::maximum()); + } else { // 32-value groups = 4 lanes + mx = sycl::fmax(mx, sycl::permute_group_by_xor(sg, mx, 1)); + mx = sycl::fmax(mx, sycl::permute_group_by_xor(sg, mx, 2)); + } + const sycl::half sh = sycl::half(127.0f / sycl::fmax(mx, int8dpas::EPS)); + if (G == QK ? lane == 0 : (lane & 3) == 0) { + const int64_t gi = G == QK ? g : g * 4 + lane / 4; + SA[gi * lda + m] = sycl::bit_cast(sh); + } + const float s = (float) sh; + uint64_t p = 0; +#pragma unroll + for (int i = 0; i < 8; ++i) { + // round to nearest (TernSYCL truncates: RTZ biases every product toward zero) + p |= (uint64_t) (uint8_t) (int8_t) sycl::rint(sycl::clamp(v[i] * s, -128.0f, 127.0f)) << (8 * i); + } + *(uint64_t *) (Aq + m * K + g * QK + 8 * lane) = p; + }); +} + +// GGML_SYCL_PTQ1_T2_NSG: sub-groups per GEMV work-group for 2..8 rows (verify batches); 2 (default) or 4. The B580 tile +// sweep (TernSYCL main, Bonsai 27B shapes, M=4, qmode 0) had 4 fastest on 5 of 6 shapes (+0-8%); M=1 keeps 2. +int t2_nsg() { + const int v = ggml_sycl_knob("GGML_SYCL_PTQ1_T2_NSG", 2) == 4 ? 4 : 2; + return v; +} + +template +void gemv(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, const uint32_t * B, const uint16_t * SB, float * C, + int M, int N, int K) { + using Kern = int8dpas::Gemv; + const size_t wgn = 16 * NSG; + const Epi epi{ nullptr, nullptr, 0, 1 }; + const sycl::range<2> local(1, Kern::WG); + const sycl::range<2> global((M + SGM - 1) / SGM, (N + wgn - 1) / wgn * Kern::WG); + q.parallel_for(sycl::nd_range<2>(global, local), + Kern{ nullptr, (const signed char *) Aq, SA, B, SB, C, epi, M, N, K }); +} + +template +void gemm_tile(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, const uint32_t * B, const uint16_t * SB, float * C, + int M, int N, int K) { + const size_t tm = MT_M * WG_M, tn = MT_N * WG_N; + const Epi epi{ nullptr, nullptr, 0, 1 }; + const sycl::range<2> local(1, 16 * WG_M * WG_N); + const sycl::range<2> global((M + tm - 1) / tm, (N + tn - 1) / tn * local[1]); + q.parallel_for(sycl::nd_range<2>(global, local), + int8dpas::GemmMT{ + nullptr, (const signed char *) Aq, SA, B, SB, C, epi, M, N, K }); +} + +// ARC-LAB lab knob GGML_SYCL_PTQ1_T2_TILE = index into TernSYCL's large-M tile table (mt_m, mt_n, wg_m, wg_n): +// 0 {8,128,8,2} (default) 1 {8,128,4,2} 2 {8,128,4,4} 3 {8,128,16,1} 4 {8,128,2,4} 5 {16,64,4,2} 6 {16,64,8,2} +// 7 {8,64,8,2} 8 {32,32,4,2} +void gemm(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, const uint32_t * B, const uint16_t * SB, float * C, + int M, int N, int K) { + // per-batch-size tile (arctune sweep, B580, 2026-10-05): {8,128,2,4} is 7-11% faster for 12-32 rows (n-gram verify + // batches), {8,128,8,2} from 48 rows up (prompts). GGML_SYCL_PTQ1_T2_TILE forces one tile for every size; + // GGML_SYCL_PTQ1_T2_TILE_SMALL_MAX = largest batch on the small tile (0 = never). + const int forced = ggml_sycl_knob("GGML_SYCL_PTQ1_T2_TILE", -1); + const int tile = forced >= 0 ? forced : (M <= ggml_sycl_knob("GGML_SYCL_PTQ1_T2_TILE_SMALL_MAX", 32) ? 4 : 0); + switch (tile) { + case 1: gemm_tile<8, 128, 4, 2>(q, Aq, SA, B, SB, C, M, N, K); break; + case 2: gemm_tile<8, 128, 4, 4>(q, Aq, SA, B, SB, C, M, N, K); break; + case 3: gemm_tile<8, 128, 16, 1>(q, Aq, SA, B, SB, C, M, N, K); break; + case 4: gemm_tile<8, 128, 2, 4>(q, Aq, SA, B, SB, C, M, N, K); break; + case 5: gemm_tile<16, 64, 4, 2>(q, Aq, SA, B, SB, C, M, N, K); break; + case 6: gemm_tile<16, 64, 8, 2>(q, Aq, SA, B, SB, C, M, N, K); break; + case 7: gemm_tile<8, 64, 8, 2>(q, Aq, SA, B, SB, C, M, N, K); break; + case 8: gemm_tile<32, 32, 4, 2>(q, Aq, SA, B, SB, C, M, N, K); break; + default: gemm_tile<8, 128, 8, 2>(q, Aq, SA, B, SB, C, M, N, K); break; + } +} + +} // namespace + +namespace { +// up to 3 weight matrices with the same activations in ONE launch: work-groups [gs[m], gs[m+1]) along dim 1 run +// matrix m (Gemv::run takes the group indices). All members use the same Gemv instantiation (same SLM, same shape). +template +struct GemvMulti { + using Kern = int8dpas::Gemv; + Kern k0, k1, k2; + int gs1, gs2; + void operator()(sycl::nd_item<2> it) const { + // one call site: three inlined copies of the body spilled (34.6 vs 46.3 t/s) + const int g0 = (int) it.get_group(0), g1 = (int) it.get_group(1); + const bool a = g1 < gs1, b = !a && g1 < gs2; + const Kern k = a ? k0 : (b ? k1 : k2); + k.run(it, g0, g1 - (a ? 0 : (b ? gs1 : gs2))); + } + auto get(int8dpas::syclex::properties_tag) const { return k0.get(int8dpas::syclex::properties_tag{}); } +}; + +template +void gemv_multi(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, int nm, const uint32_t * const * B, + const uint16_t * const * SB, float * const * C, const int * N, int M, int K) { + using GM = GemvMulti; + using Kern = typename GM::Kern; + const int wgn = 16 * NSG; + const Epi epi{ nullptr, nullptr, 0, 1 }; + int gs[4] = { 0, 0, 0, 0 }; + Kern k[3]; + for (int m = 0; m < 3; ++m) { + const int mm = m < nm ? m : 0; + k[m] = Kern{ nullptr, (const signed char *) Aq, SA, B[mm], SB[mm], C[mm], epi, M, N[mm], K }; + gs[m + 1] = gs[m] + (m < nm ? (N[m] + wgn - 1) / wgn : 0); + } + const GM f{ k[0], k[1], k[2], gs[1], nm > 2 ? gs[2] : gs[3] + 1 }; + const sycl::range<2> local(1, Kern::WG); + const sycl::range<2> global((M + SGM - 1) / SGM, (size_t) gs[nm] * Kern::WG); + q.parallel_for(sycl::nd_range<2>(global, local), f); +} +} // namespace + +namespace { +// ARC-LAB: GGML_SYCL_PTQ1_T2_TUNED=1 (default: 1 on a B580, else 0): per-shape GEMV tiles for 1..8 rows from the B580 sweep +// (/lab/exp/gemv-sweep, README "Mapping a winner into the fork"). A row of t2_tuned_table matches a launch by +// (N, K, rows, multi), N = the sum of the members' N for a same-input merged launch; no match = the default tiles above. +// =2 is a plumbing self-test (default tiles through the tuned launcher, outputs must not change). +// The tile is Gemv under a GRF mode (0 = no property, as Gemv; 128 / 256; 1 = grf_size_automatic) and +// a work-group order (0 = row-group outer, as the default path; 1 = the row-groups of a column block adjacent, so with +// SGM < rows the repeated weight reads hit L2). Same struct as gsw::Sweep in the sweep (sweep_kern.hpp): keep them equal. +template constexpr auto t2_tuned_props() { + namespace syclex = sycl::ext::oneapi::experimental; + namespace intelex = sycl::ext::intel::experimental; + if constexpr (GRF == 128 || GRF == 256) { + return syclex::properties{ syclex::sub_group_size<16>, syclex::work_group_size<1, WG>, intelex::grf_size }; + } else if constexpr (GRF == 1) { + return syclex::properties{ syclex::sub_group_size<16>, syclex::work_group_size<1, WG>, + intelex::grf_size_automatic }; + } else { + return syclex::properties{ syclex::sub_group_size<16>, syclex::work_group_size<1, WG> }; + } +} + +template +struct GemvTuned { + using Kern = int8dpas::Gemv; + Kern k0, k1, k2; + int gs1, gs2; // first column-group of members 1 and 2 (past the end when absent) + int R, C; // row-groups, column-groups (all members) + int ord; + void operator()(sycl::nd_item<2> it) const { + const int g = (int) it.get_group(1); + const int g0 = ord ? g % R : g / C; + const int g1 = ord ? g / R : g % C; + const bool a = g1 < gs1, b = !a && g1 < gs2; + const Kern k = a ? k0 : (b ? k1 : k2); // one call site of run (see GemvMulti) + k.run(it, g0, g1 - (a ? 0 : (b ? gs1 : gs2))); + } + auto get(int8dpas::syclex::properties_tag) const { return t2_tuned_props(); } +}; + +template +void gemv_tuned(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, int nm, const uint32_t * const * B, + const uint16_t * const * SB, float * const * C, const int * N, int M, int K, int ord) { + using T = GemvTuned; + using Kern = typename T::Kern; + const int wgn = 16 * NSG; + const Epi epi{ nullptr, nullptr, 0, 1 }; + int gs[4] = { 0, 0, 0, 0 }; + Kern k[3]; + for (int m = 0; m < 3; ++m) { + const int mm = m < nm ? m : 0; + k[m] = Kern{ nullptr, (const signed char *) Aq, SA, B[mm], SB[mm], C[mm], epi, M, N[mm], K }; + gs[m + 1] = gs[m] + (m < nm ? (N[m] + wgn - 1) / wgn : 0); + } + const int R = (M + SGM - 1) / SGM, Ct = gs[nm]; + const T f{ k[0], k[1], k[2], nm > 1 ? gs[1] : Ct + 1, nm > 2 ? gs[2] : Ct + 1, R, Ct, ord }; + q.parallel_for(sycl::nd_range<2>({ 1, (size_t) R * Ct * Kern::WG }, { 1, (size_t) Kern::WG }), f); +} + +// the compiled tiles: X(SGM, NSG, LS, U, GRF), one line per tile a table row uses (a row whose tile is not listed here is +// skipped with a warning) +#define T2_TUNED_KERNELS(X) \ + X(4, 8, 4, 2, 0) \ + X(4, 1, 2, 2, 0) \ + X(4, 1, 2, 4, 0) \ + X(1, 2, 2, 1, 0) + +struct t2_tuned_kernel { + int sgm, nsg, ls, u, grf; + void (*fn)(sycl::queue &, const int8_t *, const uint16_t *, int, const uint32_t * const *, const uint16_t * const *, + float * const *, const int *, int, int, int); +}; +#define T2_TUNED_X(a, b, c, d, e) t2_tuned_kernel{ a, b, c, d, e, gemv_tuned }, +const std::vector & t2_tuned_kernels() { + static const std::vector k = { T2_TUNED_KERNELS(T2_TUNED_X) }; + return k; +} +#undef T2_TUNED_X + +struct t2_tuned_row { + int N, K, M, multi; // launch: N (sum over members if merged), K, rows (1..8), merged launch (0/1) + int sgm, nsg, ls, u, grf, ord; // tile +}; +// rows from the sweep's summary ("fork table rows"); empty = GGML_SYCL_PTQ1_T2_TUNED=1 changes nothing +const std::vector & t2_tuned_table() { + static const std::vector t = { + // { N, K, M, multi, SGM, NSG, LS, U, GRF, ord }, + // B580 sweep 2026-10-10 (/lab/exp/gemv-sweep jit.log, JIT build, medians of 5 interleaved rounds): bit-exact + // winners only (same LS as the default tile), > 2% faster, launches the model issues. Time vs the default tile: + { 5120, 17408, 5, 0, 4, 8, 4, 2, 0, 1 }, // ffn_down, 5-row verify 0.775 + { 34816, 5120, 5, 1, 4, 1, 2, 2, 0, 1 }, // ffn gate+up merged, 5 rows 0.896 + { 248320, 5120, 5, 0, 4, 1, 2, 4, 0, 1 }, // output head, 5 rows 0.882 + { 14336, 5120, 5, 1, 4, 1, 2, 2, 0, 1 }, // attention q+k+v merged, 5 rows 0.923 + { 14336, 5120, 1, 1, 1, 2, 2, 1, 0, 0 }, // attention q+k+v merged, 1 row 0.974 + }; + return t; +} + +// the tiles were swept on a B580 (20 Xe cores): on by default there only; GGML_SYCL_PTQ1_T2_TUNED=1 / 0 forces it +static int t2_tuned_default() { + static const int d = [] { + const std::string name = dpct::dev_mgr::instance().get_device(ggml_sycl_get_device()).get_info(); + return name.find("B580") != std::string::npos ? 1 : 0; + }(); + return d; +} + +bool t2_tuned_on() { + return ggml_sycl_knob("GGML_SYCL_PTQ1_T2_TUNED", t2_tuned_default()) != 0; +} + +// GGML_SYCL_PTQ1_T2_TUNED=2: plumbing self-test, every 1..8-row GEMV through gemv_tuned with the default tile (same LS, +// so outputs must be token-identical to =0) +template +void t2_tuned_selftest_ls(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, int nm, const uint32_t * const * B, + const uint16_t * const * SB, float * const * C, const int * N, int M, int K, bool ls4) { + ls4 ? gemv_tuned(q, Aq, SA, nm, B, SB, C, N, M, K, 0) : + gemv_tuned(q, Aq, SA, nm, B, SB, C, N, M, K, 0); +} + +template +void t2_tuned_selftest(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, int nm, const uint32_t * const * B, + const uint16_t * const * SB, float * const * C, const int * N, int M, int K, bool ls4) { + if (M == 1) { + t2_tuned_selftest_ls<1, NSG>(q, Aq, SA, nm, B, SB, C, N, M, K, ls4); + } else if (M == 2) { + t2_tuned_selftest_ls<2, NSG>(q, Aq, SA, nm, B, SB, C, N, M, K, ls4); + } else if (M <= 4) { + t2_tuned_selftest_ls<4, NSG>(q, Aq, SA, nm, B, SB, C, N, M, K, ls4); + } else { + t2_tuned_selftest_ls<8, NSG>(q, Aq, SA, nm, B, SB, C, N, M, K, ls4); + } +} + +// launches the tuned tile for this GEMV if the table has one; false = take the default path +bool t2_tuned(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, int nm, const uint32_t * const * B, + const uint16_t * const * SB, float * const * C, const int * N, int M, int K) { + const int mode = ggml_sycl_knob("GGML_SYCL_PTQ1_T2_TUNED", t2_tuned_default()); + if (mode == 0) { + return false; + } + int ntot = 0, nmax = 0; + for (int m = 0; m < nm; ++m) { + ntot += N[m]; + nmax = std::max(nmax, N[m]); + } + if (mode == 2) { + // NSG as the default path: 2, or GGML_SYCL_PTQ1_T2_NSG=4 for 2..8 rows + (M > 1 && t2_nsg() == 4 ? t2_tuned_selftest<4> : t2_tuned_selftest<2>)(q, Aq, SA, nm, B, SB, C, N, M, K, + nmax <= 8192); + return true; + } + for (const t2_tuned_row & r : t2_tuned_table()) { + if (r.N != ntot || r.K != K || r.M != M || r.multi != (nm > 1)) { + continue; + } + for (const t2_tuned_kernel & k : t2_tuned_kernels()) { + if (k.sgm == r.sgm && k.nsg == r.nsg && k.ls == r.ls && k.u == r.u && k.grf == r.grf) { + k.fn(q, Aq, SA, nm, B, SB, C, N, M, K, r.ord); + return true; + } + } + static std::atomic warned{ false }; + if (!warned.exchange(true)) { + GGML_LOG_WARN("%s: tuned tile %d.%d.%d.%d.%d for N=%d K=%d M=%d is not in T2_TUNED_KERNELS, default tile\n", + __func__, r.sgm, r.nsg, r.ls, r.u, r.grf, r.N, r.K, r.M); + } + return false; + } + return false; +} + +void t2_dispatch(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, const uint32_t * B, const uint16_t * SB, + float * dst, int m, int n, int k, bool use_gemm) { + if (use_gemm) { + gemm(q, Aq, SA, B, SB, dst, m, n, k); + } else if (t2_tuned(q, Aq, SA, 1, &B, &SB, &dst, &n, m, k)) { + // tuned tile + } else if (m == 1) { + n <= 8192 ? gemv<1, 4>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<1, 2>(q, Aq, SA, B, SB, dst, m, n, k); + } else if (t2_nsg() == 4) { + if (m == 2) { + n <= 8192 ? gemv<2, 4, 4>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<2, 2, 4>(q, Aq, SA, B, SB, dst, m, n, k); + } else if (m <= 4) { + n <= 8192 ? gemv<4, 4, 4>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<4, 2, 4>(q, Aq, SA, B, SB, dst, m, n, k); + } else { + n <= 8192 ? gemv<8, 4, 4>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<8, 2, 4>(q, Aq, SA, B, SB, dst, m, n, k); + } + } else if (m == 2) { + n <= 8192 ? gemv<2, 4>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<2, 2>(q, Aq, SA, B, SB, dst, m, n, k); + } else if (m <= 4) { + n <= 8192 ? gemv<4, 4>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<4, 2>(q, Aq, SA, B, SB, dst, m, n, k); + } else { + n <= 8192 ? gemv<8, 4>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<8, 2>(q, Aq, SA, B, SB, dst, m, n, k); + } +} +} // namespace + +namespace { +// what the activation scratch holds: the tensor it was quantized from and in which graph (one queue, in order) +struct quant_key { + const void * key = nullptr; const float * x = nullptr; int64_t stride = 0, M = 0, K = 0; uint64_t graph = 0; + bool operator==(const quant_key & o) const { + return key == o.key && x == o.x && stride == o.stride && M == o.M && K == o.K && graph == o.graph; + } +}; +quant_key g_quant_have; +std::atomic g_graph{ 1 }; +} // namespace + +void ggml_sycl_t2_new_graph() { g_graph.fetch_add(1); } + +namespace { +// the activation scratch: int8 Aq [M, K], then fp16 SA [K/QK, ldsa(M)] +void scratch_aq(sycl::queue & q, int64_t M, int64_t K, int8_t *& Aq, uint16_t *& SA) { + const int lda = int8dpas::ldsa((int) M); + const size_t aq_bytes = (size_t) M * K; + const size_t sa_off = (aq_bytes + 255) & ~(size_t) 255; + const size_t sa_bytes = (size_t) (K / QK) * lda * 2 + 256; + char * s = (char *) scratch_get(q, sa_off + sa_bytes); + Aq = (int8_t *) s; + SA = (uint16_t *) (s + sa_off); +} +} // namespace + +namespace { +// one 1024 block of the rotation, already loaded and signed: v[j] = element lane + 16 j + 128 sgi (sgi = sub-group 0..7 of +// the block's 128 lanes), s = the block's 1024 floats of SLM. Writes the rotated block to dst (1024 floats) and its 8 +// quant groups to Aq (1024 int8) + SA (column m, groups g0..g0+7). Butterflies in fwht.cpp's order (h = 1, 2, 4 ... 512). +// Every lane of the work-group must call it (barriers). +template +inline void rot_quant_block(float (&v)[8], float * s, const Item & it, const sycl::sub_group & sg, int lane, int sgi, + float * dst, int8_t * aq, uint16_t * SA, int64_t g0, int lda, int64_t m) { +#pragma unroll + for (int h = 1; h < 16; h *= 2) { +#pragma unroll + for (int j = 0; j < 8; ++j) { + const float o = sycl::permute_group_by_xor(sg, v[j], h); + v[j] = (lane & h) == 0 ? v[j] + o : o - v[j]; + } + } +#pragma unroll + for (int st = 1; st < 8; st *= 2) { +#pragma unroll + for (int j = 0; j < 8; ++j) { + if ((j & st) == 0) { + const float a = v[j], b = v[j + st]; + v[j] = a + b; v[j + st] = a - b; + } + } + } + // layout 2: element t + 128 i (t = 16 sgi + lane); butterflies h = 128, 256, 512 in registers + const int t = 16 * sgi + lane; +#pragma unroll + for (int j = 0; j < 8; ++j) s[lane + 16 * j + 128 * sgi] = v[j]; + it.barrier(sycl::access::fence_space::local_space); +#pragma unroll + for (int i = 0; i < 8; ++i) v[i] = s[t + 128 * i]; +#pragma unroll + for (int st = 1; st < 8; st *= 2) { +#pragma unroll + for (int i = 0; i < 8; ++i) { + if ((i & st) == 0) { + const float a = v[i], b = v[i + st]; + v[i] = a + b; v[i + st] = a - b; + } + } + } + it.barrier(sycl::access::fence_space::local_space); +#pragma unroll + for (int i = 0; i < 8; ++i) s[t + 128 * i] = v[i]; + it.barrier(sycl::access::fence_space::local_space); + // layout 1 again: sub-group sgi holds quant group sgi (128 values, 8 per lane) -> quant_a's numbers + float mx = 0.0f; +#pragma unroll + for (int j = 0; j < 8; ++j) { + v[j] = s[lane + 16 * j + 128 * sgi]; + mx = sycl::fmax(mx, sycl::fabs(v[j])); + dst[lane + 16 * j + 128 * sgi] = v[j]; + } + mx = sycl::reduce_over_group(sg, mx, sycl::maximum()); + const sycl::half sh = sycl::half(127.0f / sycl::fmax(mx, int8dpas::EPS)); + const float sc = (float) sh; + if (lane == 0) { + SA[(g0 + sgi) * lda + m] = sycl::bit_cast(sh); + } +#pragma unroll + for (int j = 0; j < 8; ++j) { + aq[128 * sgi + lane + 16 * j] = (int8_t) sycl::rint(sycl::clamp(v[j] * sc, -128.0f, 127.0f)); + } +} +} // namespace + +// signed 1024-point Hadamard rotation (as fwht.cpp: dst = H (x * signs) / 32, same butterfly order, so the same floats) +// that also quantizes its output for the T2 mat-vecs which read it next: one 128-lane work-group per 1024 block where +// fwht.cpp gives a block one 16-lane sub-group (5 sub-groups in all for a decode token), and no quant_a launch after it. +bool ggml_sycl_t2_fwht_quant(sycl::queue & q, const float * x, const float * signs, float * dst, int64_t M, int64_t K, + const void * key, const float * glu_g, const float * glu_u) { + constexpr int N = 1024; + if (K % N != 0 || M <= 0 || M > 512 || !ggml_sycl_knob("GGML_SYCL_PTQ1_T2_FWHTQ", 1)) { + return false; + } + int8_t * Aq; + uint16_t * SA; + scratch_aq(q, M, K, Aq, SA); + const int lda = int8dpas::ldsa((int) M); + const int64_t nblk = K / N, rows = M * nblk; + q.submit([&](sycl::handler & cgh) { + sycl::local_accessor sl(sycl::range<1>(N), cgh); + cgh.parallel_for(sycl::nd_range<1>(rows * 128, 128), [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(16)]] { + const auto sg = it.get_sub_group(); + const int lane = sg.get_local_linear_id(); + const int sgi = sg.get_group_linear_id(); + const int64_t r = it.get_group(0); + const int64_t m = r / nblk, kb = r % nblk; + const float * sgn = signs + kb * N; + float v[8]; + if (glu_g) { // SWIGLU folded in: x = silu(g) * u, as ggml_sycl_op_swiglu computes it + const float * gp = glu_g + r * N; + const float * up = glu_u + r * N; +#pragma unroll + for (int j = 0; j < 8; ++j) { + const int e = lane + 16 * j + 128 * sgi; + const float g = gp[e]; + const float xv = g / (1.0f + sycl::exp(-g)) * up[e]; + v[j] = xv * (1.0f / 32.0f) * sgn[e]; + } + } else { + const float * src = x + r * N; +#pragma unroll + for (int j = 0; j < 8; ++j) { + const int e = lane + 16 * j + 128 * sgi; + v[j] = src[e] * (1.0f / 32.0f) * sgn[e]; + } + } + rot_quant_block(v, sl.get_multi_ptr().get(), it, sg, lane, sgi, dst + r * N, + Aq + m * K + kb * N, SA, kb * (N / QK), lda, m); + }); + }); + g_quant_have = quant_key{ key, dst, K, M, K, g_graph.load() }; + return true; +} + +// residual add + RMS norm + norm weight (+ the signed rotation + quantize of ggml_sycl_t2_fwht_quant): one work-group +// per token, K / 1024 blocks of 128 lanes. Every read of a and b happens before the row's sum of squares is reduced +// (a work-group barrier), so outputs may reuse an input's memory within a token; the caller checks across tokens. +bool ggml_sycl_t2_add_norm_rot(sycl::queue & q, const float * a, const float * b, float * add_out, const float * w, + float eps, float * normed, const float * signs, float * mm, int64_t M, int64_t K, + const void * key) { + constexpr int N = 1024; + if (K % N != 0 || K > 8 * N || M <= 0 || M > 512 || !ggml_sycl_knob("GGML_SYCL_PTQ1_T2_NORMQ", 1)) { + return false; + } + const bool rot = signs != nullptr; + int8_t * Aq = nullptr; + uint16_t * SA = nullptr; + if (rot) { + scratch_aq(q, M, K, Aq, SA); + } + const int lda = int8dpas::ldsa((int) M); + const int nt = (int) (K / N) * 128; + q.submit([&](sycl::handler & cgh) { + sycl::local_accessor sl(sycl::range<1>(rot ? K : 1), cgh); + cgh.parallel_for(sycl::nd_range<1>(M * nt, nt), [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(16)]] { + const auto sg = it.get_sub_group(); + const int lane = sg.get_local_linear_id(); + const int sgid = sg.get_group_linear_id(); + const int kb = sgid / 8, sgi = sgid % 8; + const int64_t m = it.get_group(0); + const int64_t base = m * K + kb * N; + float v[8], ss = 0.0f; +#pragma unroll + for (int j = 0; j < 8; ++j) { + const int64_t e = base + lane + 16 * j + 128 * sgi; + v[j] = b ? a[e] + b[e] : a[e]; + ss += v[j] * v[j]; + } + ss = sycl::reduce_over_group(it.get_group(), ss, sycl::plus()); + const float scale = sycl::rsqrt(ss / K + eps); +#pragma unroll + for (int j = 0; j < 8; ++j) { + const int c = kb * N + lane + 16 * j + 128 * sgi; + const int64_t e = m * K + c; + if (add_out) { + add_out[e] = v[j]; + } + v[j] = scale * v[j] * w[c]; + if (normed) { + normed[e] = v[j]; + } + if (rot) { + v[j] = v[j] * (1.0f / 32.0f) * signs[c]; + } + } + if (rot) { + rot_quant_block(v, sl.get_multi_ptr().get() + kb * N, it, sg, lane, sgi, + mm + base, Aq + base, SA, (int64_t) kb * (N / QK), lda, m); + } + }); + }); + if (rot) { + g_quant_have = quant_key{ key, mm, K, M, K, g_graph.load() }; + } + return true; +} + +void ggml_sycl_t2_mul_mat(sycl::queue & q, const void * w, const float * x, int64_t x_stride, float * dst, int64_t M, + int64_t N, int64_t K, const void * x_key) { + // batches > 8 (n-gram verify, prompts): GemmMT reads each weight once per 64 rows (TernSYCL main: the 2D block I/O + // that misbehaved in the JIT build with the old IGC builtins validates under inline vISA, so it is upstream's kernel). + const bool gemm_off = ggml_sycl_knob("GGML_SYCL_PTQ1_T2_GEMM_OFF", 0) != 0; + // GGML_SYCL_PTQ1_T2_GEMV_MAX: largest batch on the GEMV (default 8; above it GemmMT). GEMV re-reads the weights once per + // 8 rows, GemmMT once per 64 but less efficiently: `arctune sweep` measures the crossover. + const bool use_gemm = M > ggml_sycl_knob("GGML_SYCL_PTQ1_T2_GEMV_MAX", 8) && !gemm_off; + + int8_t * Aq; + uint16_t * SA; + scratch_aq(q, M, K, Aq, SA); + const uint32_t * B = (const uint32_t *) w; + const uint16_t * SB = (const uint16_t *) ((const char *) w + (size_t) (K / 16) * N * 4); + + // GGML_SYCL_PTQ1_T2_QCACHE=0: quantize for every call + const quant_key want{ x_key, x, x_stride, M, K, g_graph.load() }; + if (!x_key || !ggml_sycl_knob("GGML_SYCL_PTQ1_T2_QCACHE", 1) || !(g_quant_have == want)) { + quant_a(q, x, x_stride, M, K, Aq, SA, QK); + g_quant_have = x_key ? want : quant_key{}; + } + t2_dispatch(q, Aq, SA, B, SB, dst, (int) M, (int) N, (int) K, use_gemm); +} + +// several weights [K, N_m] on the same activations x [M, K] (M <= 8): one quantization (or the cached one) and one launch +bool ggml_sycl_t2_mul_mat_multi(sycl::queue & q, int nm, const void * const * w, float * const * dst, const int64_t * N, + const float * x, int64_t x_stride, int64_t M, int64_t K, const void * x_key) { + if (nm < 2 || nm > 3 || M < 1 || M > 8 || !ggml_sycl_knob("GGML_SYCL_T2_MULTI", 1)) { + return false; + } + int8_t * Aq; + uint16_t * SA; + scratch_aq(q, M, K, Aq, SA); + const quant_key want{ x_key, x, x_stride, M, K, g_graph.load() }; + if (!x_key || !ggml_sycl_knob("GGML_SYCL_PTQ1_T2_QCACHE", 1) || !(g_quant_have == want)) { + quant_a(q, x, x_stride, M, K, Aq, SA, QK); + g_quant_have = x_key ? want : quant_key{}; + } + const uint32_t * B[3]; + const uint16_t * SB[3]; + float * C[3]; + int Ni[3]; + int64_t nmax = 0; + for (int m = 0; m < nm; ++m) { + B[m] = (const uint32_t *) w[m]; + SB[m] = (const uint16_t *) ((const char *) w[m] + (size_t) (K / 16) * N[m] * 4); + C[m] = dst[m]; + Ni[m] = (int) N[m]; + nmax = std::max(nmax, N[m]); + } + const int m_ = (int) M, k_ = (int) K; + const bool ls4 = nmax <= 8192; + if (t2_tuned(q, Aq, SA, nm, B, SB, C, Ni, m_, k_)) { + return true; + } + if (M == 1) { + ls4 ? gemv_multi<1, 4>(q, Aq, SA, nm, B, SB, C, Ni, m_, k_) : gemv_multi<1, 2>(q, Aq, SA, nm, B, SB, C, Ni, m_, k_); + } else if (t2_nsg() == 4) { + if (M == 2) { + ls4 ? gemv_multi<2, 4, 4>(q, Aq, SA, nm, B, SB, C, Ni, m_, k_) : gemv_multi<2, 2, 4>(q, Aq, SA, nm, B, SB, C, Ni, m_, k_); + } else if (M <= 4) { + ls4 ? gemv_multi<4, 4, 4>(q, Aq, SA, nm, B, SB, C, Ni, m_, k_) : gemv_multi<4, 2, 4>(q, Aq, SA, nm, B, SB, C, Ni, m_, k_); + } else { + ls4 ? gemv_multi<8, 4, 4>(q, Aq, SA, nm, B, SB, C, Ni, m_, k_) : gemv_multi<8, 2, 4>(q, Aq, SA, nm, B, SB, C, Ni, m_, k_); + } + } else if (M == 2) { + ls4 ? gemv_multi<2, 4>(q, Aq, SA, nm, B, SB, C, Ni, m_, k_) : gemv_multi<2, 2>(q, Aq, SA, nm, B, SB, C, Ni, m_, k_); + } else if (M <= 4) { + ls4 ? gemv_multi<4, 4>(q, Aq, SA, nm, B, SB, C, Ni, m_, k_) : gemv_multi<4, 2>(q, Aq, SA, nm, B, SB, C, Ni, m_, k_); + } else { + ls4 ? gemv_multi<8, 4>(q, Aq, SA, nm, B, SB, C, Ni, m_, k_) : gemv_multi<8, 2>(q, Aq, SA, nm, B, SB, C, Ni, m_, k_); + } + return true; +} + +namespace { +// run every kernel once on a tiny problem while the model loads (the first weight repack happens in the warm-up +// decode): the JIT of GemmMT alone took ~35 s at the first prompt otherwise +void t2_precompile(sycl::queue & q) { + constexpr int K = 128, N = 32, M = 16; + char * w = (char *) sycl::malloc_device(ggml_sycl_t2_bytes(K, N), q); + float * x = sycl::malloc_device(M * K, q); + float * d = sycl::malloc_device(M * N, q); + int8_t * aq = sycl::malloc_device(M * K, q); + uint16_t * sa = sycl::malloc_device(int8dpas::ldsa(M) * (K / 32) + 64, q); + q.memset(w, 0, ggml_sycl_t2_bytes(K, N)); + q.memset(x, 0, sizeof(float) * M * K); + const uint32_t * B = (const uint32_t *) w; + const uint16_t * SB = (const uint16_t *) (w + (K / 16) * N * 4); + quant_a(q, x, K, M, K, aq, sa, QK); + for (int m : { 1, 2, 4, 8, M }) { // the N <= 8192 GEMV tiles + GemmMT (the N > 8192 ones are called below) + t2_dispatch(q, aq, sa, B, SB, d, m, N, K, m > 8); + } + gemv<1, 2>(q, aq, sa, B, SB, d, 1, N, K); + gemv<2, 2>(q, aq, sa, B, SB, d, 2, N, K); + gemv<4, 2>(q, aq, sa, B, SB, d, 4, N, K); + gemv<8, 2>(q, aq, sa, B, SB, d, 8, N, K); + if (t2_nsg() == 4) { // the N > 8192 GEMV tiles with 4 sub-groups (t2_dispatch above covered the N <= 8192 ones) + gemv<2, 2, 4>(q, aq, sa, B, SB, d, 2, N, K); + gemv<4, 2, 4>(q, aq, sa, B, SB, d, 4, N, K); + gemv<8, 2, 4>(q, aq, sa, B, SB, d, 8, N, K); + } + if (t2_tuned_on()) { // GGML_SYCL_PTQ1_T2_TUNED: JIT the table's tiles now, not at the first verify + for (const t2_tuned_kernel & t : t2_tuned_kernels()) { + const int nn = N; + t.fn(q, aq, sa, 1, &B, &SB, &d, &nn, 8, K, 0); + } + } + q.wait(); + sycl::free(w, q); + sycl::free(x, q); + sycl::free(d, q); + sycl::free(aq, q); + sycl::free(sa, q); +} +} // namespace diff --git a/ggml/src/ggml-sycl/ptq1-t2.hpp b/ggml/src/ggml-sycl/ptq1-t2.hpp new file mode 100644 index 000000000000..f6f73bb2b62f --- /dev/null +++ b/ggml/src/ggml-sycl/ptq1-t2.hpp @@ -0,0 +1,39 @@ +// ARC-LAB: PTQ1_0 weights repacked to the TernSYCL 2-bit layout and multiplied on XMX (s8 x s2 DPAS). +// See ptq1-t2.cpp. +#pragma once + +#include + +#include +#include + +// GGML_SYCL_PTQ1_T2: "ffn" = ffn_gate/ffn_up/ffn_down weights, "all" = every PTQ1_0 weight (tests), unset = off +bool ggml_sycl_t2_wants(const char * name, int64_t K, int64_t N); +// Xe2 or newer (16-lane DPAS incl. the int8 x int2 / int8 x int4 builtins): Arc B-series, Lunar Lake, Panther Lake +bool ggml_sycl_device_is_xe2(); +bool ggml_sycl_device_is_xe3(int device); +// device bytes of the repacked weight: 2-bit codes [K/16, N] u32 + fp16 scales [K/128, N] +size_t ggml_sycl_t2_bytes(int64_t K, int64_t N); +// in place: data holds N rows of K/128 block_ptq1_0 (pq2 = false) or block_pq2_0 (pq2 = true) as uploaded, and at least +// ggml_sycl_t2_bytes(K, N) bytes. PQ2_0 code 3 (+2) has no 2-bit two's-complement form: returns false if one is found. +bool ggml_sycl_t2_repack(sycl::queue & q, void * data, int64_t K, int64_t N, bool pq2 = false); +// dst[m * N + n] = sum_k x[m * x_stride + k] * w[n, k], M tokens (fp32 in and out). x_key (optional) names the +// activation within the current graph (its tensor + ggml_sycl_t2_new_graph epoch): back-to-back calls on the same x_key +// reuse the int8 activations instead of quantizing again (q/k/v, gate/up, the DeltaNet input projections). +void ggml_sycl_t2_mul_mat(sycl::queue & q, const void * w, const float * x, int64_t x_stride, float * dst, int64_t M, + int64_t N, int64_t K, const void * x_key = nullptr); +// nm (2..3) weights on the same activations (M <= 8 tokens) in one launch; false = not served (caller runs them singly) +bool ggml_sycl_t2_mul_mat_multi(sycl::queue & q, int nm, const void * const * w, float * const * dst, const int64_t * N, + const float * x, int64_t x_stride, int64_t M, int64_t K, const void * x_key); +// a new graph compute starts: activations quantized for the previous one are stale +void ggml_sycl_t2_new_graph(); +// dst [M, K] = 1024-point Hadamard of (x * signs) per block / 32 (signs: K floats), quantized for the T2 mat-vecs at once; +// key = the tensor they will read it through (see x_key). false: shape not served (K % 1024, M > 512) or knob off. +// glu_g / glu_u (optional, then x is unused): x = silu(glu_g) * glu_u, both [M, K] contiguous (SWIGLU folded in) +bool ggml_sycl_t2_fwht_quant(sycl::queue & q, const float * x, const float * signs, float * dst, int64_t M, int64_t K, + const void * key, const float * glu_g = nullptr, const float * glu_u = nullptr); +// [add_out =] a (+ b); normed = rms_norm(.) * w; [mm = rotation of normed * signs, quantized as above]. b, add_out, +// normed, signs (and then mm, key) may be null. One work-group per token: K % 1024 == 0, K <= 8192, M <= 512. +bool ggml_sycl_t2_add_norm_rot(sycl::queue & q, const float * a, const float * b, float * add_out, const float * w, + float eps, float * normed, const float * signs, float * mm, int64_t M, int64_t K, + const void * key); diff --git a/ggml/src/ggml-sycl/quantize.hpp b/ggml/src/ggml-sycl/quantize.hpp index b5c7a54b7924..36305314a493 100644 --- a/ggml/src/ggml-sycl/quantize.hpp +++ b/ggml/src/ggml-sycl/quantize.hpp @@ -91,6 +91,41 @@ template struct quantize_and_reorder_q8_1_soa { } }; +// ARC-LAB: q8_1 activations for the PTQ1_0 decode-once mat-vec (one lane per 128-weight block). Same values as +// quantize_and_reorder_q8_1_soa, but within a row the 16-byte chunk c of block b sits at (c * nb128 + b) * 16 and the +// half2 (d, sum) of sub-block k at kx + (k * nb128 + b) * 4, so the 16 lanes' loads of one chunk are one contiguous +// 256-byte read (the SoA layout made every such load touch 16 cache lines). Needs kx % 128 == 0. +template struct quantize_and_reorder_q8_1_ptq1_il { + __dpct_inline__ void operator()(const float * __restrict__ x, void * reordered_q8_tensor, const int kx, + const int kx_padded, const sycl::nd_item<1> & it) const { + auto subgroup_id = it.get_group(0); + auto wi_id = it.get_local_id(0); + + sycl::vec quantized_values; + float d = 0.0f; + float sum = 0.0f; + quantize_q8_1_impl(x, quantized_values, d, sum, it); + + const int num_blocks_per_row = kx / QK8_1; + const int nb128 = kx / 128; + const int row = subgroup_id / num_blocks_per_row; + const int col = subgroup_id % num_blocks_per_row; // q8_1 block within the row + const auto row_offset = (size_t) row * (kx_padded / QK8_1) * sizeof(block_q8_1); + const int b = col / 4; // 128-block + const int w = 32 * (col % 4) + wi_id * ElementsPerWI; + const int c = w / 16; + const int byte = w % 16; + + auto quant_ptr = (int8_t *) ((char *) reordered_q8_tensor + row_offset + (size_t) (c * nb128 + b) * 16 + byte); + *reinterpret_cast *>(quant_ptr) = quantized_values; + + auto ds_ptr = (sycl::half2 *) ((char *) reordered_q8_tensor + row_offset + kx) + ((col % 4) * nb128 + b); + if (wi_id == 0) { + *ds_ptr = sycl::half2(sycl::half(d), sycl::half(sum)); + } + } +}; + template struct quantize_q8_1 { __dpct_inline__ void operator()(const float * __restrict__ x, void * q8_tensor, const int kx, const int kx_padded, const sycl::nd_item<1> & it) const { diff --git a/ggml/src/ggml-sycl/quants.hpp b/ggml/src/ggml-sycl/quants.hpp index a26a6ce6e6d7..eb98039dc243 100644 --- a/ggml/src/ggml-sycl/quants.hpp +++ b/ggml/src/ggml-sycl/quants.hpp @@ -58,6 +58,45 @@ template <> struct block_q_t { static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; } }; +template <> struct block_q_t { + struct traits { + static constexpr uint32_t qk = QK_PQ2_0; + static constexpr uint32_t qi = QK_PQ2_0 / 16; // 16 two-bit quants per int + static constexpr uint32_t qr = 4; + static constexpr uint32_t vdr_mmvq = 4; + }; + + static constexpr std::pair get_block_offset(const int block_index, const int /* nblocks */) { + return { block_index * (QK_PQ2_0 / 4), 0 }; + } + + static constexpr std::pair get_d_offset(int nrows, int ncols, const int block_index) { + return { (ncols / 4 * nrows) + block_index * sizeof(ggml_half), 0 }; + } + + static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; } +}; + +template <> struct block_q_t { + struct traits { + static constexpr uint32_t qk = QK_PTQ1_0; + static constexpr uint32_t qi = 4; + static constexpr uint32_t qr = 1; + static constexpr uint32_t vdr_mmvq = 4; // one lane takes a whole block + }; + + // first: qs offset, second: qh offset + static constexpr std::pair get_block_offset(const int block_index, const int nblocks) { + return { block_index * 24, nblocks * 24 + block_index * 2 }; + } + + static constexpr std::pair get_d_offset(int nrows, int ncols, const int block_index) { + return { (ncols / QK_PTQ1_0 * nrows) * 26 + block_index * sizeof(ggml_half), 0 }; + } + + static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; } +}; + template <> struct block_q_t { struct traits { static constexpr uint32_t qk = QK_K; diff --git a/ggml/src/ggml-sycl/ternsycl/LICENSE.md b/ggml/src/ggml-sycl/ternsycl/LICENSE.md new file mode 100644 index 000000000000..37e99b93421a --- /dev/null +++ b/ggml/src/ggml-sycl/ternsycl/LICENSE.md @@ -0,0 +1,29 @@ +# BSD 3-Clause License + +Copyright (c) 2026, Intel Corporation +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/ggml/src/ggml-sycl/ternsycl/NOTICE b/ggml/src/ggml-sycl/ternsycl/NOTICE new file mode 100644 index 000000000000..44128779c2e6 --- /dev/null +++ b/ggml/src/ggml-sycl/ternsycl/NOTICE @@ -0,0 +1,8 @@ +TernSYCL +Copyright (c) 2026, Intel Corporation. BSD 3-Clause License, see LICENSE.md. + +The SYCL kernels are ports of the TernOCL OpenCL kernels +(https://github.com/libxsmm/TernOCL), which port Intel XeTLA kernels +(int2_fp16_upcvt_xmx_xe.hpp, int2_fp16_dpas_xmx_xe.hpp, +int2_bf16_dpas_xmx_xe.hpp) and follow their data layouts, dequantization and +quantization schemes. diff --git a/ggml/src/ggml-sycl/ternsycl/epilogue_dev.hpp b/ggml/src/ggml-sycl/ternsycl/epilogue_dev.hpp new file mode 100644 index 000000000000..c85661cbda23 --- /dev/null +++ b/ggml/src/ggml-sycl/ternsycl/epilogue_dev.hpp @@ -0,0 +1,71 @@ +// Fused epilogues on the fp32 accumulator, numbered like the xetla vLLM plugin +// and OpenVINO integration (same as TernOCL's epilogue.clh): +// postop 0 none, 1 silu(acc) * other (SwiGLU gate), 2 acc + other (residual), +// 3 acc + bias[n], 4 sigmoid(acc) +// f32: C (and bias) in fp32 instead of DT (lm_head logits) +// Unlike the OpenCL -D build, postop and f32 are uniform kernel arguments, so +// one kernel binary serves every epilogue; they only branch at the store. +#pragma once + +#include "xe2.hpp" + +struct Epi { + const unsigned short *other; // DT [M, N] + const void *bias; // DT or fp32 [N] + int postop; + int f32; +}; + +template struct epi_dev { + using D = xe2::dt; + + // xetla_sigmoid; the clamp is a multiply (IGC miscompiles the ?: form in OpenCL) + static float sigmoid(float x) { + return sycl::native::recip(sycl::native::exp(-x) + 1.0f) * (float)(x > -10.0f); + } + static float apply(float v, float e, int p) { + if (p == 1) return v * sigmoid(v) * e; + if (p == 2 || p == 3) return v + e; + if (p == 4) return sigmoid(v); + return v; + } + // epilogue operand of element i = m * N + n + static float in(const Epi &e, size_t i, int n) { + if (e.postop == 1 || e.postop == 2) return D::tof(e.other[i]); + if (e.postop == 3) + return e.f32 ? ((const float *)e.bias)[n] : D::tof(((const unsigned short *)e.bias)[n]); + return 0.0f; + } + static void store(void *C, const Epi &e, size_t i, int n, float v) { + v = apply(v, in(e, i, n), e.postop); + if (e.f32) ((float *)C)[i] = v; + else ((unsigned short *)C)[i] = D::from(v); + } + + // epilogue + 2D block store of one 8 x 16 tile at (m0, n0); out of range + // rows and columns are zero-filled on read and clipped on write. + // P / F >= 0 fix postop / f32 at compile time (-1: read them from e). + template + static void store8x16(void *C, const Epi &e, int M, int N, int m0, int n0, xe2::float8 v) { + using namespace xe2; + const int postop = P >= 0 ? P : e.postop; + const bool f32 = F >= 0 ? F : e.f32; + if (postop == 1 || postop == 2) { + const ushort8 t = rd_16b_8r16(surf(e.other, N * 2, M, N * 2), n0, m0); + for (int r = 0; r < 8; ++r) v[r] = apply(v[r], D::tof(t[r]), postop); + } else if (postop == 3) { + const float b = f32 ? sycl::bit_cast(rd_32b_1r16(surf(e.bias, N * 4, 1, N * 4), n0, 0)) + : D::tof(rd_16b_1r16(surf(e.bias, N * 2, 1, N * 2), n0, 0)); + for (int r = 0; r < 8; ++r) v[r] += b; + } else if (postop == 4) { + for (int r = 0; r < 8; ++r) v[r] = sigmoid(v[r]); + } + if (f32) { + wr_32b_8r16(surf(C, N * 4, M, N * 4), n0, m0, __builtin_bit_cast(uint8, v)); + } else { + ushort8 o; + for (int r = 0; r < 8; ++r) o[r] = D::from(v[r]); + wr_16b_8r16(surf(C, N * 2, M, N * 2), n0, m0, o); + } + } +}; diff --git a/ggml/src/ggml-sycl/ternsycl/int2_int8_dpas.hpp b/ggml/src/ggml-sycl/ternsycl/int2_int8_dpas.hpp new file mode 100644 index 000000000000..50bdd2521ded --- /dev/null +++ b/ggml/src/ggml-sycl/ternsycl/int2_int8_dpas.hpp @@ -0,0 +1,411 @@ +// SYCL port of TernOCL's int2_int8_dpas.cl, itself a port of the xetla int2 x +// int8 DPAS GEMM/GEMV (int2_fp16_dpas_xmx_xe.hpp, int2_bf16_dpas_xmx_xe.hpp): +// +// C[m,n] = DT( epi( sum_g float(sum_{k in g} q[m,k] * code(B[k,n])) +// * (SB[g,n] * (1 / SA[g,m])) ) ) +// q[m,k] = sat_int8_rtz(A[m,k] * SA[g,m]), SA[g,m] = DT(127 / absmax_g(A[m,:])) +// +// A : DT [M, K] row-major SA : DT [K/128, LDSA(M)] +// B : uint32 [K/16, N], 2-bit two's-complement codes of K rows +// 16*kp .. 16*kp+15 of column n (row 16*kp+j at bits 2j..2j+1) +// SB : DT [K/128, N] C : DT (or fp32) [M, N] +// Aq : int8 [M, K] (QMODE 0 only) +// +// B never gets unpacked: the DPAS takes it natively (s8 x s2, K = 32); per +// lane (= column) the int2 B operand is two consecutive words of the layout. +// int32 accumulation within a 128-group, fp32 rescale per group. +// +// QMODE (A quantization): +// 0 upfront : quant_a writes SA and int8 Aq; the GEMM reads int8 A +// 1 xetla : quant_a writes SA only (xetla external_scale_a_calc=1); the +// GEMM quantizes the DT A tile on the fly (large-M: xetla's instruction +// sequence as inline vISA, quant8x32) +#pragma once + +#include "epilogue_dev.hpp" + +#include + +namespace int8dpas { + +using namespace xe2; +namespace syclex = sycl::ext::oneapi::experimental; +namespace intelex = sycl::ext::intel::experimental; + +constexpr int GS = 128; +// SA row pitch, padded so SA is a valid 2D block surface (pitch % 64 B == 0) +inline int ldsa(int M) { return (M + 31) & ~31; } +constexpr float EPS = 1.1920928955078125e-07f; // FLT_EPSILON, xetla absmax init + +// two DT (low, high half of h2) -> two int8 packed in a short, rounded to nearest +// even: + 1.5 * 2^23 rounds to nearest even into the low mantissa bits +// (|f * sa| <= 127.5), whose low byte is the int8 +template inline short q2(unsigned h2, float sa) { + float f0, f1; + if constexpr (BF16) { + f0 = sycl::bit_cast(h2 << 16); + f1 = sycl::bit_cast(h2 & 0xffff0000u); + } else { + f0 = dt::tof((unsigned short)h2); + f1 = dt::tof((unsigned short)(h2 >> 16)); + } + const unsigned u0 = sycl::bit_cast(sycl::fma(f0, sa, 12582912.0f)); + const unsigned u1 = sycl::bit_cast(sycl::fma(f1, sa, 12582912.0f)); + return (short)((u0 & 0xffu) | ((u1 & 0xffu) << 8)); +} + +// xetla's elemwise_scale_{fp16,bf16}_to_int8 on one 8-row x 32-K block, on the +// row-contiguous registers (SIMT code would split and re-pair every pair). +// a: row r = a[r] (lane l holds K 2l, 2l+1); lane r of sal = scale of row r. +// SIMD32 NoMask like xetla: call only from convergent code. a * sal + 1.5 * 2^23 +// rounds to nearest even into the low mantissa bits (|a * sal| <= 127.5); the +// low byte of each float is the int8. +#define TS_QROW(r, g, orow, ocol) \ + "mad (M1_NM, 32) T(" #g ",0)<1> T(" #g ",0)<1;1,0> %2(0," #r ")<0;1,0> 0x4b400000:f\n" \ + "mov (M1_NM, 32) QB(" #orow "," #ocol ")<1> TB(" #g ",0)<4;1,0>\n" +#define TS_QUANT(A2F) \ + "{\n" \ + ".decl AH v_type=G type=hf num_elts=256 align=GRF alias=<%1,0>\n" \ + ".decl AW v_type=G type=uw num_elts=256 align=GRF alias=<%1,0>\n" \ + ".decl QB v_type=G type=b num_elts=256 align=GRF alias=<%0,0>\n" \ + ".decl T v_type=G type=f num_elts=256 align=GRF\n" \ + ".decl TD v_type=G type=ud num_elts=256 align=GRF alias=\n" \ + ".decl TB v_type=G type=b num_elts=1024 align=GRF alias=\n" \ + A2F(0, 0) TS_QROW(0, 0, 0, 0) A2F(1, 2) TS_QROW(1, 2, 0, 32) \ + A2F(2, 4) TS_QROW(2, 4, 1, 0) A2F(3, 6) TS_QROW(3, 6, 1, 32) \ + A2F(4, 8) TS_QROW(4, 8, 2, 0) A2F(5, 10) TS_QROW(5, 10, 2, 32) \ + A2F(6, 12) TS_QROW(6, 12, 3, 0) A2F(7, 14) TS_QROW(7, 14, 3, 32) \ + "}\n" +#define TS_A2F_BF16(r, g) "shl (M1_NM, 32) TD(" #g ",0)<1> AW(" #r ",0)<1;1,0> 0x10:uw\n" +#define TS_A2F_FP16(r, g) "mov (M1_NM, 32) T(" #g ",0)<1> AH(" #r ",0)<1;1,0>\n" + +template inline short8 quant8x32(uint8 a, float sal) { + short8 q; +#ifdef __SYCL_DEVICE_ONLY__ + if constexpr (BF16) __asm__ volatile(TS_QUANT(TS_A2F_BF16) : "=rw"(q) : "rw"(a), "rw"(sal)); + else __asm__ volatile(TS_QUANT(TS_A2F_FP16) : "=rw"(q) : "rw"(a), "rw"(sal)); +#endif + return q; +} + +// --------------------------------------------------------------------------- +// Pre-kernel: one sub-group per (row, 128-group). SA always, Aq if WRITE_Q. +template +struct QuantA { + const unsigned short *A; + unsigned short *SA; + signed char *Aq; + int M, K; + + void operator()(sycl::nd_item<2> it) const { + using D = dt; + const auto sg = it.get_sub_group(); + const int lane = sg.get_local_linear_id(); + const int g = (int)it.get_global_id(1) / 16; + const int m = (int)it.get_global_id(0); + if (g >= K / GS || m >= M) return; + const size_t off = (size_t)m * K + g * GS + 8 * lane; + const ushort8 h = *(const ushort8 *)(A + off); + float a[8], mx = 0.0f; +#pragma unroll + for (int i = 0; i < 8; ++i) { + a[i] = D::tof(h[i]); + mx = sycl::fmax(mx, sycl::fabs(a[i])); + } + mx = sycl::reduce_over_group(sg, mx, sycl::maximum()); + const unsigned short sh = D::from(127.0f / sycl::fmax(mx, EPS)); + if (lane == 0) SA[(size_t)g * ldsa(M) + m] = sh; + if constexpr (WRITE_Q) { + const float s = D::tof(sh); + unsigned long long q = 0; +#pragma unroll + for (int i = 0; i < 8; ++i) + q |= (unsigned long long)(unsigned char)(signed char)sycl::clamp(sycl::rint(a[i] * s), -128.0f, 127.0f) + << (8 * i); + *(unsigned long long *)(Aq + off) = q; + } + } + + auto get(syclex::properties_tag) const { + return syclex::properties{syclex::sub_group_size<16>, syclex::work_group_size<1, 16>}; + } +}; + +template struct rows; +template <> struct rows<1> { using a_t = short; using ia_t = int; using fa_t = float; }; +template <> struct rows<2> { using a_t = short2; using ia_t = int2; using fa_t = float2; }; +template <> struct rows<4> { using a_t = short4; using ia_t = int4; using fa_t = float4; }; +template <> struct rows<8> { using a_t = short8; using ia_t = int8; using fa_t = float8; }; + +template inline auto el(const V &v, int r) { + if constexpr (SGM == 1) return v; + else return v[r]; +} +template inline void set_el(V &v, int r, T x) { + if constexpr (SGM == 1) v = x; + else v[r] = x; +} + +// --------------------------------------------------------------------------- +// GEMV / small M. A sub-group owns 16 columns and SGM rows and walks its K +// slice in 128-steps: one 2D block read of 8 B words (4 DPAS of K = 32), one +// SB row, SGM A rows (quantized in registers for QMODE 1). +// SGM rows per sub-group (1, 2, 4, 8) NSG_N sub-groups along N +// LS K-slices per column block, reduced through SLM +// U 128-steps whose loads are issued before any compute +template +struct Gemv { + const unsigned short *A; + const signed char *Aq; + const unsigned short *SA; + const unsigned *B; + const unsigned short *SB; + void *C; + Epi epi; + int M, N, K; + + using a_t = typename rows::a_t; + using ia_t = typename rows::ia_t; + using fa_t = typename rows::fa_t; + using D = dt; + static constexpr int WG = 16 * NSG_N * LS; + + // SGM x 128 A tile of step s: aq[c] = K 32c..32c+31, inv[r] = 1 / SA of row r + void load_a(int m0, int s, a_t *aq, float *inv) const { + const int lda = ldsa(M); +#pragma unroll + for (int r = 0; r < SGM; ++r) { + const bool ok = m0 + r < M; + const size_t row = (size_t)sycl::min(m0 + r, M - 1) * K + s * GS; + if constexpr (QMODE == 0) { + const ushort4 l = sg_rd_us4((const unsigned short *)(Aq + row)); + const ushort4 v = ok ? l : ushort4{}; +#pragma unroll + for (int c = 0; c < 4; ++c) set_el(aq[c], r, (short)v[c]); + inv[r] = ok ? sycl::native::recip(D::tof(SA[(size_t)s * lda + m0 + r])) : 0.0f; + } else { + const uint4 l = sg_rd_u4((const unsigned *)(A + row)); + const uint4 v = ok ? l : uint4{}; + const float sa = ok ? D::tof(SA[(size_t)s * lda + m0 + r]) : 0.0f; +#pragma unroll + for (int c = 0; c < 4; ++c) set_el(aq[c], r, q2(v[c], sa)); + inv[r] = ok ? sycl::native::recip(sa) : 0.0f; + } + } + } + + static fa_t step(fa_t acc, const uint8 &w, float sb, const a_t *aq, const float *inv) { + ia_t ia = dpas_s2s8_z(aq[0], int2{(int)w[0], (int)w[1]}); +#pragma unroll + for (int c = 1; c < 4; ++c) + ia = dpas_s2s8(aq[c], int2{(int)w[2 * c], (int)w[2 * c + 1]}, ia); +#pragma unroll + for (int r = 0; r < SGM; ++r) + set_el(acc, r, el(acc, r) + (float)el(ia, r) * (sb * inv[r])); + return acc; + } + + // ARC-LAB (llama.cpp vendored copy): body takes the work-group indices as arguments so one launch can serve several + // weight matrices that share the activations (llama.cpp merges q/k/v, qkv/z and gate/up this way) + void operator()(sycl::nd_item<2> it) const { run(it, (int)it.get_group(0), (int)it.get_group(1)); } + + void run(sycl::nd_item<2> it, int g0, int g1) const { + const auto sgp = it.get_sub_group(); + const int lane = sgp.get_local_linear_id(); + const int sg = sgp.get_group_linear_id(); + const int sgn = sg % NSG_N, sgk = sg / NSG_N; + const int n0 = (g1 * NSG_N + sgn) * 16; + const int m0 = g0 * SGM; + + const int nsteps = K / GS; + const int per = (nsteps + LS - 1) / LS; + const int s_begin = sgk * per; + const int s_end = sycl::min(nsteps, s_begin + per); + const surf sbs(B, N * 4, K / 16, N * 4); + + fa_t acc = 0.0f; + if (n0 < N) { + int s = s_begin; +#pragma unroll 1 + for (; s + U <= s_end; s += U) { + uint8 w[U]; + float sb[U]; + a_t aq[U][4]; + float inv[U][SGM]; +#pragma unroll + for (int u = 0; u < U; ++u) { + w[u] = rd_32b_8r16(sbs, n0, (s + u) * 8); + sb[u] = D::tof(sg_rd_us(SB + (size_t)(s + u) * N + n0)); + load_a(m0, s + u, aq[u], inv[u]); + } +#pragma unroll + for (int u = 0; u < U; ++u) acc = step(acc, w[u], sb[u], aq[u], inv[u]); + } + if constexpr (U > 1) + for (; s < s_end; ++s) { + a_t aq[4]; + float inv[SGM]; + const uint8 w = rd_32b_8r16(sbs, n0, s * 8); + const float sb = D::tof(sg_rd_us(SB + (size_t)s * N + n0)); + load_a(m0, s, aq, inv); + acc = step(acc, w, sb, aq, inv); + } + } + + if constexpr (LS > 1) { + float *red = *sycl::ext::oneapi::group_local_memory_for_overwrite( + it.get_group()); + if (sgk > 0) { + float *dst = red + (((sgk - 1) * NSG_N + sgn) * SGM) * 16; +#pragma unroll + for (int r = 0; r < SGM; ++r) dst[r * 16 + lane] = el(acc, r); + } + barrier_local(); + if (sgk > 0) return; + for (int j = 0; j < LS - 1; ++j) { + const float *src = red + ((j * NSG_N + sgn) * SGM) * 16; +#pragma unroll + for (int r = 0; r < SGM; ++r) set_el(acc, r, el(acc, r) + src[r * 16 + lane]); + } + } + + if (n0 >= N) return; +#pragma unroll + for (int r = 0; r < SGM; ++r) + if (m0 + r < M) + epi_dev::store(C, epi, (size_t)(m0 + r) * N + n0 + lane, n0 + lane, el(acc, r)); + } + + auto get(syclex::properties_tag) const { + return syclex::properties{syclex::sub_group_size<16>, syclex::work_group_size<1, WG>}; + } +}; + +// --------------------------------------------------------------------------- +// Large-M GEMM. A sub-group computes an MT_M x MT_N tile (MT_M % 8 == 0, +// MT_N % 16 == 0); a work-group is WG_M x WG_N sub-groups. Per 128-group: +// B (8 words per 16-column block) and SB are loaded once and reused for all +// MT_M/8 row blocks; each 8 x 128 A block is loaded (and for QMODE 1 +// quantized) once and reused for all MT_N/16 column blocks. 2D block I/O +// zero-fills out-of-range reads and clips writes, so any M works. 256 GRF. +// The epilogue is a template parameter here (as in the OpenCL -D build): a +// runtime-selected one costs ~3% in this kernel. +template +struct GemmMT { + const unsigned short *A; + const signed char *Aq; + const unsigned short *SA; + const unsigned *B; + const unsigned short *SB; + void *C; + Epi epi; + int M, N, K; + + using D = dt; + static constexpr int MB = MT_M / 8, NB = MT_N / 16, WG = 16 * WG_M * WG_N; + + void operator()(sycl::nd_item<2> it) const { + const auto sgp = it.get_sub_group(); + const int sg = sgp.get_group_linear_id(); + const int m0 = ((int)it.get_group(0) * WG_M + sg / WG_N) * MT_M; + const int n0 = ((int)it.get_group(1) * WG_N + sg % WG_N) * MT_N; + const surf sbs(B, N * 4, K / 16, N * 4), ssb(SB, N * 2, K / GS, N * 2); + // lane l gets SA[s, m + l] (pad columns past M are junk) + const surf ssa(SA, ldsa(M) * 2, K / GS, ldsa(M) * 2); + + float8 acc[MB][NB]; +#pragma unroll + for (int i = 0; i < MB; ++i) +#pragma unroll + for (int j = 0; j < NB; ++j) acc[i][j] = 0.0f; + + // 2D payloads built once; each K step only moves y (B, SB, SA) or x (A) + unsigned pb = pl2d(sbs, n0, 0), psb2 = pl2d(ssb, n0, 0), + psb1 = pl2d(ssb, n0, 0), psa = pl2d(ssa, m0, 0), pq; + if constexpr (QMODE == 0) pq = pl2d(surf(Aq, K, M, K), 0, m0); + else pq = pl2d(surf(A, K * 2, M, K * 2), 0, m0); + + for (int s = 0; s < K / GS; ++s) { + // A (and SA) loads first: they feed the quantization / unpacking that the + // first dpas waits on (the asm reads are issued in source order). Row + // block 0 before B and SB, block I + 1 at the start of block I. + pl2d_y(psa, s); + pl2d_x(pq, s * GS / 2); + unsigned short sar[MB]; + std::conditional_t ar[MB]; + auto load_a = [&](auto ii) { + constexpr int I = decltype(ii)::value; + sar[I] = rd2d(psa); + if constexpr (QMODE == 0) + static_for<2>([&](auto h) { ar[I][h] = rd2d(pq); }); + else + static_for<4>([&](auto c) { ar[I][c] = rd2d(pq); }); + }; + load_a(std::integral_constant{}); + uint8 w[NB]; + float sb[NB]; + pl2d_y(pb, s * 8); + static_for([&](auto j) { w[j] = rd2d(pb); }); + // all SB loads first, then convert: converting each (bf16: via acc0) before the + // next load let IGC reuse one load register and serialize the loads + ushort2 sbr[(NB + 1) / 2]; + pl2d_y(psb2, s); + pl2d_y(psb1, s); + static_for<(NB + 1) / 2>([&](auto h) { + constexpr int J = 2 * decltype(h)::value; + if constexpr (J + 1 < NB) sbr[h] = rd2d(psb2); + else sbr[h] = ushort2{rd2d(psb1), 0}; + }); +#pragma unroll + for (int j = 0; j < NB; ++j) sb[j] = D::tof(sbr[j / 2][j % 2]); + static_for([&](auto ii) { + constexpr int I = decltype(ii)::value; + if constexpr (I + 1 < MB) load_a(std::integral_constant{}); + short8 aq[4]; + float inv[8]; + const float sal = D::tof(sar[I]); + if constexpr (QMODE == 0) { +#pragma unroll + for (int h = 0; h < 2; ++h) +#pragma unroll + for (int r = 0; r < 8; ++r) { + aq[2 * h][r] = (short)ar[I][h][r]; + aq[2 * h + 1][r] = (short)ar[I][h][8 + r]; + } + } else { +#pragma unroll + for (int c = 0; c < 4; ++c) aq[c] = quant8x32(ar[I][c], sal); + } + // rows >= M get inf here, but their int32 dot is 0 and the store clips them + const float invl = sycl::native::recip(sal); +#pragma unroll + for (int r = 0; r < 8; ++r) inv[r] = sycl::group_broadcast(sgp, invl, r); +#pragma unroll + for (int j = 0; j < NB; ++j) { + int8 ia = dpas_s2s8_z(aq[0], int2{(int)w[j][0], (int)w[j][1]}); +#pragma unroll + for (int c = 1; c < 4; ++c) + ia = dpas_s2s8(aq[c], int2{(int)w[j][2 * c], (int)w[j][2 * c + 1]}, ia); + // whole-vector convert (OpenCL's convert_float8): per-element casts + // are emitted through one scratch register and stall every mad + const float8 fi = __builtin_convertvector(ia, float8); +#pragma unroll + for (int r = 0; r < 8; ++r) acc[I][j][r] += fi[r] * (sb[j] * inv[r]); + } + }); + } + +#pragma unroll + for (int i = 0; i < MB; ++i) +#pragma unroll + for (int j = 0; j < NB; ++j) + epi_dev::template store8x16(C, epi, M, N, m0 + 8 * i, n0 + 16 * j, acc[i][j]); + } + + auto get(syclex::properties_tag) const { + return syclex::properties{syclex::sub_group_size<16>, syclex::work_group_size<1, WG>, + intelex::grf_size<256>}; + } +}; + +} // namespace int8dpas diff --git a/ggml/src/ggml-sycl/ternsycl/xe2.hpp b/ggml/src/ggml-sycl/ternsycl/xe2.hpp new file mode 100644 index 000000000000..9a25467049bc --- /dev/null +++ b/ggml/src/ggml-sycl/ternsycl/xe2.hpp @@ -0,0 +1,350 @@ +// Xe2 sub-group primitives for SIMT SYCL device code, without OpenCL builtins +// or IGC-internal intrinsics: +// * DPAS, 2D block I/O, cached LSC loads: inline vISA (the instructions IGC +// emits for the OpenCL builtins; operands are the SIMD16 per-lane vectors, +// row r of a block / DPAS operand = vector component r) +// * sub-group block reads: sycl_ext_oneapi_group_load_store (group_load) +#pragma once + +#include +#include + +#include + +namespace xe2 { + +#define XE2_VEC(T, n, name) typedef T name __attribute__((ext_vector_type(n))) +XE2_VEC(short, 2, short2); +XE2_VEC(short, 4, short4); +XE2_VEC(short, 8, short8); +XE2_VEC(unsigned short, 2, ushort2); +XE2_VEC(unsigned short, 4, ushort4); +XE2_VEC(unsigned short, 8, ushort8); +XE2_VEC(unsigned short, 16, ushort16); +XE2_VEC(unsigned short, 32, ushort32); +XE2_VEC(unsigned short, 64, ushort64); +XE2_VEC(int, 2, int2); +XE2_VEC(int, 4, int4); +XE2_VEC(int, 8, int8); +XE2_VEC(unsigned, 2, uint2); +XE2_VEC(unsigned, 3, uint3); +XE2_VEC(unsigned, 4, uint4); +XE2_VEC(unsigned, 8, uint8); +XE2_VEC(float, 2, float2); +XE2_VEC(float, 4, float4); +XE2_VEC(float, 8, float8); +#undef XE2_VEC + +// 2D surface: width and pitch in bytes, height in rows. Addresses are long long: +// long is 32 bits in SYCL device code on Windows (LLP64) and would truncate them +struct surf { + long long base; + int w, h, p; + surf(const void *b, int width_bytes, int height, int pitch_bytes) + : base((long long)b), w(width_bytes - 1), h(height - 1), p(pitch_bytes - 1) {} +}; + +#ifdef __SYCL_DEVICE_ONLY__ +#define XE2_ASM(...) __asm__(__VA_ARGS__) +#define XE2_ASM_V(...) __asm__ volatile(__VA_ARGS__) +#else +#define XE2_ASM(...) +#define XE2_ASM_V(...) +#endif + +// 2D block I/O, lsc_load_block2d / lsc_store_block2d on an explicit surface +// flat[base, width-1, height-1, pitch-1, x (elements), y (rows)]; shape +// d.[V x]C x R: V blocks of C columns x R rows, block-major. +#define XE2_2D_RD(name, T, shape) \ + inline T name(const surf &s, int x, int y) { \ + T v; \ + XE2_ASM("{\n" \ + ".decl SB v_type=G type=q num_elts=1 align=qword alias=<%1,0>\n" \ + ".decl SW v_type=G type=d num_elts=1 align=dword alias=<%2,0>\n" \ + ".decl SH v_type=G type=d num_elts=1 align=dword alias=<%3,0>\n" \ + ".decl SP v_type=G type=d num_elts=1 align=dword alias=<%4,0>\n" \ + ".decl SX v_type=G type=d num_elts=1 align=dword alias=<%5,0>\n" \ + ".decl SY v_type=G type=d num_elts=1 align=dword alias=<%6,0>\n" \ + "lsc_load_block2d.ugm (M1, 1) %0:" shape " flat[SB,SW,SH,SP,SX,SY]\n" \ + "}\n" \ + : "=rw"(v) \ + : "rw.u"(s.base), "rw.u"(s.w), "rw.u"(s.h), "rw.u"(s.p), "rw.u"(x), "rw.u"(y)); \ + return v; \ + } +#define XE2_2D_WR(name, T, shape) \ + inline void name(const surf &s, int x, int y, T v) { \ + XE2_ASM_V("{\n" \ + ".decl SB v_type=G type=q num_elts=1 align=qword alias=<%0,0>\n" \ + ".decl SW v_type=G type=d num_elts=1 align=dword alias=<%1,0>\n" \ + ".decl SH v_type=G type=d num_elts=1 align=dword alias=<%2,0>\n" \ + ".decl SP v_type=G type=d num_elts=1 align=dword alias=<%3,0>\n" \ + ".decl SX v_type=G type=d num_elts=1 align=dword alias=<%4,0>\n" \ + ".decl SY v_type=G type=d num_elts=1 align=dword alias=<%5,0>\n" \ + "lsc_store_block2d.ugm (M1, 1) flat[SB,SW,SH,SP,SX,SY] %6:" shape "\n" \ + "}\n" \ + :: "rw.u"(s.base), "rw.u"(s.w), "rw.u"(s.h), "rw.u"(s.p), "rw.u"(x), "rw.u"(y), "rw"(v)); \ + } +XE2_2D_RD(rd_32b_1r16, unsigned, "d32.16x1nn") +XE2_2D_RD(rd_32b_2r16, uint2, "d32.16x2nn") +XE2_2D_RD(rd_32b_8r16, uint8, "d32.16x8nn") +// a 16x1 d16 block is half a GRF, but the load writes a whole (zero-padded) GRF +inline unsigned short rd_16b_1r16(const surf &s, int x, int y) { + unsigned short v; + XE2_ASM("{\n" + ".decl SB v_type=G type=q num_elts=1 align=qword alias=<%1,0>\n" + ".decl SW v_type=G type=d num_elts=1 align=dword alias=<%2,0>\n" + ".decl SH v_type=G type=d num_elts=1 align=dword alias=<%3,0>\n" + ".decl SP v_type=G type=d num_elts=1 align=dword alias=<%4,0>\n" + ".decl SX v_type=G type=d num_elts=1 align=dword alias=<%5,0>\n" + ".decl SY v_type=G type=d num_elts=1 align=dword alias=<%6,0>\n" + ".decl TP v_type=G type=uw num_elts=32 align=GRF\n" + "lsc_load_block2d.ugm (M1, 1) TP:d16.16x1nn flat[SB,SW,SH,SP,SX,SY]\n" + "mov (M1, 16) %0(0,0)<1> TP(0,0)<1;1,0>\n" + "}\n" + : "=rw"(v) + : "rw.u"(s.base), "rw.u"(s.w), "rw.u"(s.h), "rw.u"(s.p), "rw.u"(x), "rw.u"(y)); + return v; +} +// two adjacent 1x16 blocks = one 1x32 block: [0] = column x + lane, [1] = x + 16 + lane +XE2_2D_RD(rd_16b_1r16x2, ushort2, "d16.32x1nn") +XE2_2D_RD(rd_16b_8r16, ushort8, "d16.16x8nn") +// two adjacent 8x16 blocks, block-major: [0..7] = columns x..x+15, [8..15] = x+16..x+31 +XE2_2D_RD(rd_16b_8r16x2, ushort16, "d16.2x16x8nn") +XE2_2D_WR(wr_16b_8r16, ushort8, "d16.16x8nn") +XE2_2D_WR(wr_32b_8r16, uint8, "d32.16x8nn") +#undef XE2_2D_RD +#undef XE2_2D_WR + +// 2D block reads from a prebuilt address payload plus immediate (DX, DY) offsets, +// as IGC emits for the builtins: one payload per surface and K step instead of +// ~6 movs per load. The vISA text is built at compile time (asm((expr))). +namespace detail { +template struct cstr { + char s[N]{}; + size_t n = 0; + constexpr size_t size() const { return n; } + constexpr const char *data() const { return s; } + constexpr void add(const char *p) { while (*p) s[n++] = *p++; } + constexpr void addi(int v) { + if (v < 0) { s[n++] = '-'; v = -v; } + char t[12]; + int k = 0; + do { t[k++] = char('0' + v % 10); v /= 10; } while (v); + while (k) s[n++] = t[--k]; + } +}; +template constexpr auto rd2d_str() { + cstr<320> c; + c.add("{\n.decl PD v_type=G type=ud num_elts=8 align=GRF alias=<%1,0>\n"); + if constexpr (SH::pad) c.add(".decl TP v_type=G type=uw num_elts=32 align=GRF\n" + "lsc_load_block2d.ugm (M1, 1) TP:"); + else c.add("lsc_load_block2d.ugm (M1, 1) %0:"); + c.add(SH::v); + c.add(" flat[PD + ("); + c.addi(DX); + c.add(","); + c.addi(DY); + c.add(")]\n"); + if constexpr (SH::pad) c.add("mov (M1, 16) %0(0,0)<1> TP(0,0)<1;1,0>\n"); + c.add("}\n"); + return c; +} +} // namespace detail + +// block shape: vISA type string and payload dword 7 = (V-1) << 16 | (R-1) << 8 | (C-1); +// pad: the block is half a GRF but the load writes a whole (zero-padded) GRF +struct b32_16x1 { static constexpr const char *v = "d32.16x1nn"; static constexpr int code = 0x00f; static constexpr bool pad = false; }; +struct b32_16x8 { static constexpr const char *v = "d32.16x8nn"; static constexpr int code = 0x70f; static constexpr bool pad = false; }; +struct b16_16x8 { static constexpr const char *v = "d16.16x8nn"; static constexpr int code = 0x70f; static constexpr bool pad = false; }; +struct b16_2x16x8 { static constexpr const char *v = "d16.2x16x8nn"; static constexpr int code = 0x1070f; static constexpr bool pad = false; }; +struct b16_16x16 { static constexpr const char *v = "d16.16x16nn"; static constexpr int code = 0xf0f; static constexpr bool pad = false; }; +struct b16_2x16x16 { static constexpr const char *v = "d16.2x16x16nn"; static constexpr int code = 0x10f0f; static constexpr bool pad = false; }; +struct b16_16x32 { static constexpr const char *v = "d16.16x32nn"; static constexpr int code = 0x1f0f; static constexpr bool pad = false; }; +struct b16_2x16x32 { static constexpr const char *v = "d16.2x16x32nn"; static constexpr int code = 0x11f0f; static constexpr bool pad = false; }; +struct b16_32x1 { static constexpr const char *v = "d16.32x1nn"; static constexpr int code = 0x01f; static constexpr bool pad = false; }; +struct b16_16x1 { static constexpr const char *v = "d16.16x1nn"; static constexpr int code = 0x00f; static constexpr bool pad = true; }; + +template inline unsigned pl2d(const surf &s, int x, int y) { + unsigned pl; + XE2_ASM("{\n" + ".decl PQ v_type=G type=uq num_elts=4 align=GRF alias=<%0,0>\n" + ".decl PD v_type=G type=ud num_elts=8 align=GRF alias=<%0,0>\n" + "mov (M1_NM, 1) PQ(0,0)<1> %1(0,0)<0;1,0>\n" + "mov (M1_NM, 1) PD(0,2)<1> %2(0,0)<0;1,0>\n" + "mov (M1_NM, 1) PD(0,3)<1> %3(0,0)<0;1,0>\n" + "mov (M1_NM, 1) PD(0,4)<1> %4(0,0)<0;1,0>\n" + "mov (M1_NM, 1) PD(0,5)<1> %5(0,0)<0;1,0>\n" + "mov (M1_NM, 1) PD(0,6)<1> %6(0,0)<0;1,0>\n" + "mov (M1_NM, 1) PD(0,7)<1> %7(0,0)<0;1,0>\n" + "}\n" + : "=rw"(pl) + : "rw.u"(s.base), "rw.u"(s.w), "rw.u"(s.h), "rw.u"(s.p), "rw.u"(x), "rw.u"(y), "rw.u"(SH::code)); + return pl; +} +// in-place update of the block x (dword 5) or y (dword 6) of a payload: build the +// payload once outside the K loop and move it per step (keeps one register per +// payload live across the loop, as IGC does, instead of rebuilding all 8 dwords) +template inline void pl2d_set(unsigned &pl, int v) { + static_assert(D == 5 || D == 6); + if constexpr (D == 5) + XE2_ASM("{\n.decl PD v_type=G type=ud num_elts=8 align=GRF alias=<%0,0>\n" + "mov (M1_NM, 1) PD(0,5)<1> %1(0,0)<0;1,0>\n}\n" : "+rw"(pl) : "rw.u"(v)); + else + XE2_ASM("{\n.decl PD v_type=G type=ud num_elts=8 align=GRF alias=<%0,0>\n" + "mov (M1_NM, 1) PD(0,6)<1> %1(0,0)<0;1,0>\n}\n" : "+rw"(pl) : "rw.u"(v)); +} +inline void pl2d_x(unsigned &pl, int x) { pl2d_set<5>(pl, x); } +inline void pl2d_y(unsigned &pl, int y) { pl2d_set<6>(pl, y); } +template inline T rd2d(unsigned pl) { + T v; + XE2_ASM((detail::rd2d_str()) : "=rw"(v) : "rw"(pl)); + return v; +} + +// compile-time loop: f(std::integral_constant) for I = 0..N-1 +template inline void static_for(F &&f) { + [&](std::integer_sequence) { (f(std::integral_constant{}), ...); }( + std::make_integer_sequence{}); +} + +// Sub-group block reads, striped: element i of lane l = p[l + 16 i]; p must be +// global and 4-byte aligned (16-byte for the vector forms). One transposed LSC +// block load, whose dword order is the SIMD16 vector layout (group_load lowers +// these to per-lane gathers). The scalar form fills half a GRF but the load +// writes a whole one, so it goes through a temp. +inline unsigned short sg_rd_us(const unsigned short *p) { + unsigned short v; + XE2_ASM("{\n.decl TP v_type=G type=uw num_elts=32 align=GRF\n" + ".decl AD v_type=G type=q num_elts=1 align=GRF\n" + "mov (M1_NM, 1) AD(0,0)<1> %1(0,0)<0;1,0>\n" + "lsc_load.ugm (M1_NM, 1) TP:d32x8t flat[AD]:a64\n" + "mov (M1, 16) %0(0,0)<1> TP(0,0)<1;1,0>\n}\n" + : "=rw"(v) : "rw.u"((long long)p)); + return v; +} +// the send address must start a GRF: a uniform scalar is copied into an aligned temp +#define XE2_BLK_RD(name, R, P, shape) \ + inline R name(const P *p) { \ + R v; \ + XE2_ASM("{\n.decl AD v_type=G type=q num_elts=1 align=GRF\n" \ + "mov (M1_NM, 1) AD(0,0)<1> %1(0,0)<0;1,0>\n" \ + "lsc_load.ugm (M1_NM, 1) %0:" shape " flat[AD]:a64\n}\n" \ + : "=rw"(v) : "rw.u"((long long)p)); \ + return v; \ + } +XE2_BLK_RD(sg_rd_us4, ushort4, unsigned short, "d32x32t") +XE2_BLK_RD(sg_rd_us8, ushort8, unsigned short, "d32x64t") +XE2_BLK_RD(sg_rd_u4, uint4, unsigned, "d32x64t") +#undef XE2_BLK_RD + +// per-lane 3-dword load, L1 and L3 cached +inline uint3 ld_u3_cached(const uint3 *p) { + uint3 v; + XE2_ASM("lsc_load.ugm.ca.ca (M1, 16) %0:d32x3 flat[%1]:a64" : "=rw"(v) : "rw"((long long)p)); + return v; +} + +// DPAS, SIMD16, systolic depth 8. Repeat count R = rows of A: A is one +// short per row per lane (K16 fp16/bf16, or K32 s8), B the VNNI weights; +// na / nb = A / B size in dwords. +#define XE2_DPAS(name, prec, A, B, C, R, na, nb) \ + inline C name(A a, B b, C acc) { \ + XE2_ASM("{\n" \ + ".decl DB v_type=G type=ud num_elts=" #nb " align=GRF alias=<%1,0>\n" \ + ".decl DA v_type=G type=ud num_elts=" #na " align=GRF alias=<%2,0>\n" \ + "dpas." prec ".8." #R " (M1, 16) %0.0 %0.0 DB.0 DA(0,0)\n" \ + "}\n" \ + : "+rw"(acc) : "rw"(b), "rw"(a)); \ + return acc; \ + } +#define XE2_DPAS_Z(name, prec, A, B, C, R, na, nb) \ + inline C name(A a, B b) { \ + C d; \ + XE2_ASM("{\n" \ + ".decl DB v_type=G type=ud num_elts=" #nb " align=GRF alias=<%1,0>\n" \ + ".decl DA v_type=G type=ud num_elts=" #na " align=GRF alias=<%2,0>\n" \ + "dpas." prec ".8." #R " (M1, 16) %0.0 %%null.0 DB.0 DA(0,0)\n" \ + "}\n" \ + : "=rw"(d) : "rw"(b), "rw"(a)); \ + return d; \ + } +XE2_DPAS(dpas_hf, "hf.hf", short, int8, float, 1, 8, 128) +XE2_DPAS(dpas_hf, "hf.hf", short2, int8, float2, 2, 16, 128) +XE2_DPAS(dpas_hf, "hf.hf", short4, int8, float4, 4, 32, 128) +XE2_DPAS(dpas_hf, "hf.hf", short8, int8, float8, 8, 64, 128) +XE2_DPAS(dpas_bf, "bf.bf", short, int8, float, 1, 8, 128) +XE2_DPAS(dpas_bf, "bf.bf", short2, int8, float2, 2, 16, 128) +XE2_DPAS(dpas_bf, "bf.bf", short4, int8, float4, 4, 32, 128) +XE2_DPAS(dpas_bf, "bf.bf", short8, int8, float8, 8, 64, 128) +// s8 A (K32) x s2 B: 4 weights per byte of B +XE2_DPAS(dpas_s2s8, "s2.s8", short, int2, int, 1, 8, 32) +XE2_DPAS(dpas_s2s8, "s2.s8", short2, int2, int2, 2, 16, 32) +XE2_DPAS(dpas_s2s8, "s2.s8", short4, int2, int4, 4, 32, 32) +XE2_DPAS(dpas_s2s8, "s2.s8", short8, int2, int8, 8, 64, 32) +// same with a zero accumulator (src0 = null) +XE2_DPAS_Z(dpas_s2s8_z, "s2.s8", short, int2, int, 1, 8, 32) +XE2_DPAS_Z(dpas_s2s8_z, "s2.s8", short2, int2, int2, 2, 16, 32) +XE2_DPAS_Z(dpas_s2s8_z, "s2.s8", short4, int2, int4, 4, 32, 32) +XE2_DPAS_Z(dpas_s2s8_z, "s2.s8", short8, int2, int8, 8, 64, 32) +// s8 A (K32) x s4 B: 2 weights per byte of B, 16 bytes (32 K) per lane; dword d of a lane's B holds K 8d .. 8d+7, +// low nibble first. Same result as the IGC builtin intel_sub_group_i8_i4_matrix_mad_k32 (repeat count 8). +XE2_DPAS(dpas_s4s8, "s4.s8", short, int4, int, 1, 8, 64) +XE2_DPAS(dpas_s4s8, "s4.s8", short2, int4, int2, 2, 16, 64) +XE2_DPAS(dpas_s4s8, "s4.s8", short4, int4, int4, 4, 32, 64) +XE2_DPAS(dpas_s4s8, "s4.s8", short8, int4, int8, 8, 64, 64) +XE2_DPAS_Z(dpas_s4s8_z, "s4.s8", short, int4, int, 1, 8, 64) +XE2_DPAS_Z(dpas_s4s8_z, "s4.s8", short2, int4, int2, 2, 16, 64) +XE2_DPAS_Z(dpas_s4s8_z, "s4.s8", short4, int4, int4, 4, 32, 64) +XE2_DPAS_Z(dpas_s4s8_z, "s4.s8", short8, int4, int8, 8, 64, 64) +#undef XE2_DPAS +#undef XE2_DPAS_Z + +// work-group barrier with a local-memory fence only (OpenCL barrier(CLK_LOCAL_MEM_FENCE)) +inline void barrier_local() { +#ifdef __SYCL_DEVICE_ONLY__ + __spirv_ControlBarrier(__spv::Scope::Workgroup, __spv::Scope::Workgroup, + __spv::MemorySemanticsMask::AcquireRelease | __spv::MemorySemanticsMask::WorkgroupMemory); +#endif +} + +// 16-bit activation dtype: raw bits in, fp32 out, round-to-nearest-even back +template struct dt; +template <> struct dt { + static float tof(unsigned short h) { return (float)sycl::bit_cast(h); } + static unsigned short from(float f) { return sycl::bit_cast((sycl::half)f); } +}; +template <> struct dt { + static float tof(unsigned short h) { return sycl::bit_cast((unsigned)h << 16); } + static unsigned short from(float f) { + return sycl::bit_cast(sycl::ext::oneapi::bfloat16(f)); + } +}; + +template inline C mad_k16(A a, int8 b, C acc) { + if constexpr (BF16) return dpas_bf(a, b, acc); + else return dpas_hf(a, b, acc); +} + +namespace detail { +template constexpr auto dpas_at_str() { + cstr<300> c; + c.add("{\n.decl DB v_type=G type=ud num_elts=128 align=GRF alias=<%1,0>\n" + ".decl DA v_type=G type=ud num_elts="); + c.addi(NA); + c.add(" align=GRF alias=<%2,0>\ndpas."); + c.add(BF16 ? "bf.bf" : "hf.hf"); + c.add(".8.8 (M1, 16) %0.0 %0.0 DB.0 DA("); + c.addi(ROW); + c.add(",0)\n}\n"); + return c; +} +} // namespace detail + +// 8-row DPAS whose A operand starts at GRF row ROW of a larger A block (e.g. a +// 32-row x 2-block 2D read), aliased in place as IGC does for the builtins +template inline float8 mad8_at(const AV &a, int8 b, float8 acc) { + XE2_ASM((detail::dpas_at_str()) : "+rw"(acc) : "rw"(b), "rw"(a)); + return acc; +} + +} // namespace xe2 diff --git a/ggml/src/ggml-sycl/vecdotq.hpp b/ggml/src/ggml-sycl/vecdotq.hpp index dcfded35b2a0..4700506556ab 100644 --- a/ggml/src/ggml-sycl/vecdotq.hpp +++ b/ggml/src/ggml-sycl/vecdotq.hpp @@ -85,6 +85,99 @@ static __dpct_inline__ int get_int_from_uint8_aligned( (const int*)(x8 + sizeof(int) * i32)); // assume at least 4 byte alignment } +// PTQ1_0 block x 128 int8 activations. qs is 4-byte aligned. Byte b holds 5 trits, read with +// w = v * 3, trit = w >> 8, v = w & 0xFF. Two bytes sit in 16-bit lanes so one multiply reads two. +// Lanes take bytes {0,2} and {1,3} so the four trits come out in activation order. +// sumi[k] gets sum(trit * x) over q8_1 block k, with trit in {0,1,2}. +// part 0 does qs ints 0..2, part 1 does qs ints 3..5 and qh; nparts 1 does all. +static __dpct_inline__ void ptq1_0_dot_block(const uint8_t * qs, const uint32_t qh, const int8_t * x, int * sumi, + const int part = 0, const int nparts = 1) { + const int g0 = nparts == 1 ? 0 : 3 * part; + const int g1 = nparts == 1 ? 6 : g0 + 3; +#pragma unroll + for (int g = g0; g < g1; ++g) { + const uint32_t packed = get_int_from_uint8_aligned(qs, g); + const uint32_t v_lo = packed & 0x00FF00FF; + const uint32_t v_hi = (packed >> 8) & 0x00FF00FF; +#pragma unroll + for (int t = 0; t < 5; ++t) { + // trit t of byte b is ((b * 3^t) mod 256) * 3 >> 8; no chain between trits + constexpr uint32_t pow3[5] = { 1, 3, 9, 27, 81 }; + const uint32_t w_lo = ((v_lo * pow3[t]) & 0x00FF00FF) * 3; + const uint32_t w_hi = ((v_hi * pow3[t]) & 0x00FF00FF) * 3; + const int q = (int) (((w_lo >> 8) & 0x00FF00FF) | (w_hi & 0xFF00FF00)); + const int e = g < 4 ? t * 16 + 4 * g : 80 + t * 8 + 4 * (g - 4); + sumi[e >> 5] = dpct::dp4a(q, get_int_from_int8_aligned(x, e / 4), sumi[e >> 5]); + } + } + if (nparts == 1 || part == 1) { + uint32_t v = qh; +#pragma unroll + for (int t = 0; t < 4; t += 2) { + const uint32_t w0 = v * 3; + v = w0 & 0x00FF00FF; + const uint32_t w1 = v * 3; + v = w1 & 0x00FF00FF; + const int q = (int) (((w0 >> 8) & 0xFF) | ((w0 >> 16) & 0xFF00) | ((w1 << 8) & 0xFF0000) | (w1 & 0xFF000000)); + sumi[3] = dpct::dp4a(q, get_int_from_int8_aligned(x, (120 + 2 * t) / 4), sumi[3]); + } + } +} + +// trit - 1 is the weight: subtract the q8_1 sum once per block via ds.y (part 0 only) +static __dpct_inline__ float ptq1_0_finish(const int * sumi, const sycl::half2 * ds, const float d, const int part = 0) { + float acc = 0.0f; +#pragma unroll + for (int k = 0; k < 4; ++k) { + const sycl::float2 dsf = ds[k].convert(); + acc += sumi[k] * dsf.x() - (part == 0 ? dsf.y() : 0.0f); + } + return d * acc; +} + +// ARC-LAB: PTQ1_0 decode-once helpers for multi-column mat-vec. Word w of qw holds trits (as 0..2 bytes) for elements +// 4w .. 4w+3, the same values ptq1_0_dot_block feeds to dp4a, so a block is decoded once and dotted with every column. +static __dpct_inline__ void ptq1_0_decode_block(const uint8_t * qs, const uint32_t qh, int * qw) { + // ARC-LAB: one trit level at a time, x3 as shift + add (a 32-bit integer multiply is two instructions on Xe; + // the old form did two per half-word per level: v * 3^t, then * 3) +#pragma unroll + for (int g = 0; g < 6; ++g) { + const uint32_t packed = get_int_from_uint8_aligned(qs, g); + uint32_t v_lo = packed & 0x00FF00FF; + uint32_t v_hi = (packed >> 8) & 0x00FF00FF; +#pragma unroll + for (int t = 0; t < 5; ++t) { + const uint32_t w_lo = v_lo + (v_lo << 1); + const uint32_t w_hi = v_hi + (v_hi << 1); + v_lo = w_lo & 0x00FF00FF; + v_hi = w_hi & 0x00FF00FF; + const int e = g < 4 ? t * 16 + 4 * g : 80 + t * 8 + 4 * (g - 4); + qw[e / 4] = (int) (((w_lo >> 8) & 0x00FF00FF) | (w_hi & 0xFF00FF00)); + } + } + uint32_t v = qh; +#pragma unroll + for (int t = 0; t < 4; t += 2) { + const uint32_t w0 = v + (v << 1); + v = w0 & 0x00FF00FF; + const uint32_t w1 = v + (v << 1); + v = w1 & 0x00FF00FF; + qw[(120 + 2 * t) / 4] = (int) (((w0 >> 8) & 0xFF) | ((w0 >> 16) & 0xFF00) | ((w1 << 8) & 0xFF0000) | (w1 & 0xFF000000)); + } +} + +// dot of a decoded block with 128 q8 activations, read 16 bytes at a time (sumi: 4 groups of 32 elements) +static __dpct_inline__ void ptq1_0_dot_decoded(const int * qw, const int8_t * x, int * sumi) { +#pragma unroll + for (int c = 0; c < 8; ++c) { + const sycl::int4 xv = ((const sycl::int4 *) x)[c]; +#pragma unroll + for (int u = 0; u < 4; ++u) { + sumi[c >> 1] = dpct::dp4a(qw[4 * c + u], xv[u], sumi[c >> 1]); + } + } +} + static __dpct_inline__ int byte_sub_4(const int a, const int b) { const uint32_t ua = static_cast(a); const uint32_t ub = static_cast(b); @@ -521,6 +614,69 @@ template <> struct reorder_vec_dot_q_sycl { }; }; +template <> struct reorder_vec_dot_q_sycl { + static constexpr ggml_type gtype = GGML_TYPE_PQ2_0; + + using pq2_0_block = ggml_sycl_reordered::block_q_t; + using pq2_0_traits = typename pq2_0_block::traits; + + // Each step takes 2 ints of quants = one q8_1 block of 32 values. Value is (q - 1) * d. + __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair ibx_offset, + const std::pair d_offset, const int8_t * q8_1_quant_ptr, + const sycl::half2 * q8_1_ds, const int & iqs) { + const uint8_t * qs = static_cast(vbq) + ibx_offset.first; + const ggml_half d = *(reinterpret_cast(static_cast(vbq) + d_offset.first)); + + float sum = 0.0f; +#pragma unroll + for (int b = 0; b < (int) pq2_0_traits::vdr_mmvq / 2; ++b) { + const int m = iqs + 2 * b; + int sumi = 0; +#pragma unroll + for (int h = 0; h < 2; ++h) { + const int v = get_int_from_uint8_aligned(qs, m + h); +#pragma unroll + for (int j = 0; j < 4; ++j) { + const int q = (v >> (8 * j)) & 0xFF; + const int vi = (q & 3) | ((q >> 2 & 3) << 8) | ((q >> 4 & 3) << 16) | ((q >> 6 & 3) << 24); + sumi = dpct::dp4a(vi, get_int_from_int8_aligned(q8_1_quant_ptr, 4 * (m + h) + j), sumi); + } + } + const sycl::float2 ds8f = q8_1_ds[m / 2].convert(); + sum += sumi * ds8f.x() - ds8f.y(); + } + return d * sum; + }; +}; + +template <> struct reorder_vec_dot_q_sycl { + static constexpr ggml_type gtype = GGML_TYPE_PTQ1_0; + + __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair ibx_offset, + const std::pair d_offset, const int8_t * q8_1_quant_ptr, + const sycl::half2 * q8_1_ds, const int & iqs) { + const uint8_t * base = static_cast(vbq); + const uint8_t * qh = base + ibx_offset.second; + const ggml_half d = *(reinterpret_cast(base + d_offset.first)); + + GGML_UNUSED(iqs); + // wide loads: 128 activations as 8 x 16 B, 24 quant bytes as 3 x 8 B + int8_t x[QK_PTQ1_0]; +#pragma unroll + for (int j = 0; j < QK_PTQ1_0 / 16; ++j) { + *(sycl::int4 *) (x + 16 * j) = ((const sycl::int4 *) q8_1_quant_ptr)[j]; + } + uint8_t qs[24]; +#pragma unroll + for (int j = 0; j < 3; ++j) { + *(sycl::uint2 *) (qs + 8 * j) = ((const sycl::uint2 *) (base + ibx_offset.first))[j]; + } + int sumi[4] = { 0, 0, 0, 0 }; + ptq1_0_dot_block(qs, (uint32_t) qh[0] | ((uint32_t) qh[1] << 16), x, sumi); + return ptq1_0_finish(sumi, q8_1_ds, d); + }; +}; + template <> struct reorder_vec_dot_q_sycl { static constexpr ggml_type gtype = GGML_TYPE_Q8_0; diff --git a/ggml/src/ggml-sycl/w8a8.cpp b/ggml/src/ggml-sycl/w8a8.cpp new file mode 100644 index 000000000000..e6a714af8889 --- /dev/null +++ b/ggml/src/ggml-sycl/w8a8.cpp @@ -0,0 +1,216 @@ +#include "w8a8.hpp" + +#if GGML_SYCL_DNNL +#include "dequantize.hpp" +#include "gemm.hpp" + +// per row: D = max over groups of d, ratio q_g = round(127 * d_g / D); one work-group (16 lanes) per row +static void w8a8_row_scales(ggml_type type, bool reordered, const void * w, float * row_scale, int8_t * ratio, + int64_t nrows, int64_t K, dpct::queue_ptr stream) { + const int64_t ng = K / 128; + const int64_t nb = nrows * ng; + const uint8_t * base = (const uint8_t *) w; + const bool ptq = type == GGML_TYPE_PTQ1_0; + stream->parallel_for(sycl::nd_range<1>(nrows * 16, 16), [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(16)]] { + const int64_t r = it.get_group(0); + const int lane = it.get_local_id(0); + auto dget = [&](int64_t g) -> float { + const int64_t ib = r * ng + g; + if (ptq) { + return reordered ? (float) ((const sycl::half *) (base + nb * 26))[ib] : (float) ((const block_ptq1_0 *) w)[ib].d; + } + return reordered ? (float) ((const sycl::half *) (base + nb * 32))[ib] : (float) ((const block_pq2_0 *) w)[ib].d; + }; + float m = 0.0f; + for (int64_t g = lane; g < ng; g += 16) { + m = sycl::fmax(m, sycl::fabs(dget(g))); + } + m = sycl::reduce_over_group(it.get_sub_group(), m, sycl::maximum()); + for (int64_t g = lane; g < ng; g += 16) { + ratio[r * ng + g] = m > 0 ? (int8_t) sycl::round(127.0f * dget(g) / m) : 0; + } + if (lane == 0) { + row_scale[r] = m / 127.0f; + } + }); +} + +// int8 weights = (code value) * ratio of its group; 8 elements per work item +static void w8a8_weights(ggml_type type, bool reordered, const void * w, const int8_t * ratio, int8_t * w8, + int64_t nrows, int64_t K, dpct::queue_ptr stream) { + const int64_t nb = nrows * K / 128; + const uint8_t * base = (const uint8_t *) w; + const bool ptq = type == GGML_TYPE_PTQ1_0; + stream->parallel_for(sycl::range<1>(nb * 16), [=](sycl::id<1> id) { + const int64_t ib = id[0] / 16; + const int c = (int) (id[0] % 16); + int v[8]; + if (ptq) { + const uint8_t * qs; + const uint8_t * qh; + if (reordered) { + qs = base + ib * 24; + qh = base + nb * 24 + ib * 2; + } else { + qs = ((const block_ptq1_0 *) w)[ib].qs; + qh = ((const block_ptq1_0 *) w)[ib].qh; + } + constexpr uint32_t pow3[5] = { 1, 3, 9, 27, 81 }; + if (c < 15) { + // 8 source bytes in one load: qs[0..7], qs[8..15] (c < 10) or qs[16..23] + const uint32_t p = pow3[c < 10 ? c / 2 : c - 10]; + const int off = c < 10 ? (c % 2) * 8 : 16; + const uint64_t src = reordered ? *(const uint64_t *) (qs + off) : 0; +#pragma unroll + for (int i = 0; i < 8; ++i) { + const uint32_t byte = reordered ? (uint32_t) (src >> (8 * i)) & 0xFF : qs[off + i]; + v[i] = (int) ((((byte * p) & 0xFF) * 3) >> 8) - 1; + } + } else { +#pragma unroll + for (int i = 0; i < 8; ++i) v[i] = (int) ((((qh[i % 2] * pow3[i / 2]) & 0xFF) * 3) >> 8) - 1; + } + } else { + const uint8_t * qs = reordered ? base + ib * 32 : ((const block_pq2_0 *) w)[ib].qs; + const uint32_t two = reordered ? *(const uint16_t *) (qs + 2 * c) : (uint32_t) qs[2 * c] | ((uint32_t) qs[2 * c + 1] << 8); +#pragma unroll + for (int i = 0; i < 8; ++i) v[i] = (int) ((two >> (2 * i)) & 3) - 1; + } + const int r8 = ratio[ib]; + uint64_t out = 0; +#pragma unroll + for (int i = 0; i < 8; ++i) out |= (uint64_t) (uint8_t) (int8_t) (v[i] * r8) << (8 * i); + ((uint64_t *) w8)[id[0]] = out; + }); +} + +// per token: s = max|x| / 127, x8 = round(x / s); one 256-item work-group per token row, 16-byte loads +static void w8a8_acts(const float * x, int8_t * x8, float * xs, int64_t ntok, int64_t K, dpct::queue_ptr stream) { + constexpr int WG = 256; + GGML_ASSERT(K % 4 == 0); + stream->parallel_for(sycl::nd_range<1>(ntok * WG, WG), [=](sycl::nd_item<1> it) { + const int64_t t = it.get_group(0); + const int lid = it.get_local_id(0); + const int64_t K4 = K / 4; + const sycl::float4 * row4 = (const sycl::float4 *) (x + t * K); + float m = 0.0f; + for (int64_t k4 = lid; k4 < K4; k4 += WG) { + const sycl::float4 v = sycl::fabs(row4[k4]); + m = sycl::fmax(m, sycl::fmax(sycl::fmax(v.x(), v.y()), sycl::fmax(v.z(), v.w()))); + } + m = sycl::reduce_over_group(it.get_group(), m, sycl::maximum()); + const float inv = m > 0 ? 127.0f / m : 0.0f; + sycl::char4 * out4 = (sycl::char4 *) (x8 + t * K); + for (int64_t k4 = lid; k4 < K4; k4 += WG) { + const sycl::float4 v = sycl::round(row4[k4] * inv); + out4[k4] = sycl::char4((int8_t) v.x(), (int8_t) v.y(), (int8_t) v.z(), (int8_t) v.w()); + } + if (lid == 0) { + xs[t] = m / 127.0f; + } + }); +} + +bool ggml_sycl_w8a8_mul_mat(ggml_backend_sycl_context & ctx, ggml_type type, bool reordered, const void * w, + const float * x, float * dst, int64_t nrows, int64_t ncols_x, int64_t K, + dpct::queue_ptr stream) { + if ((type != GGML_TYPE_PQ2_0 && type != GGML_TYPE_PTQ1_0) || K % 128 != 0) { + return false; + } + ggml_sycl_pool_alloc ratio(ctx.pool(), nrows * K / 128); + ggml_sycl_pool_alloc wscale(ctx.pool(), nrows); + ggml_sycl_pool_alloc w8(ctx.pool(), nrows * K); + ggml_sycl_pool_alloc x8(ctx.pool(), ncols_x * K); + ggml_sycl_pool_alloc xscale(ctx.pool(), ncols_x); + + w8a8_row_scales(type, reordered, w, wscale.get(), ratio.get(), nrows, K, stream); + w8a8_weights(type, reordered, w, ratio.get(), w8.get(), nrows, K, stream); + w8a8_acts(x, x8.get(), xscale.get(), ncols_x, K, stream); + DnnlGemmWrapper::gemm_s8(ctx, (int) nrows, (int) ncols_x, (int) K, x8.get(), w8.get(), dst, stream); + + const float * ws = wscale.get(); + const float * xs = xscale.get(); + stream->parallel_for(sycl::range<1>(ncols_x * nrows), [=](sycl::id<1> i) { + dst[i] *= xs[i[0] / nrows] * ws[i[0] % nrows]; + }); + return true; +} + +// ARC-LAB: the same int8 GEMM from the TernSYCL 2-bit layout (PTQ1_0 / PQ2_0 repacked in place by ggml_sycl_t2_repack): +// codes [K/16][N] u32 (16 two-bit two's-complement weights per word, k = 16 w + j at bits 2 j), scales [K/128][N] fp16. +// Large prompt batches: oneDNN's int8 GEMM beat the TernSYCL GemmMT for them before T2 went live (pp512 1013 vs ~900). +bool ggml_sycl_w8a8_mul_mat_t2(ggml_backend_sycl_context & ctx, const void * w, const float * x, int64_t x_stride, + float * dst, int64_t nrows, int64_t ntok, int64_t K, dpct::queue_ptr stream) { + if (K % 128 != 0) { + return false; + } + const int64_t N = nrows, ng = K / 128; + const uint32_t * B = (const uint32_t *) w; + const uint16_t * SB = (const uint16_t *) ((const char *) w + (size_t) (K / 16) * N * 4); + ggml_sycl_pool_alloc ratio(ctx.pool(), N * ng); + ggml_sycl_pool_alloc wscale(ctx.pool(), N); + ggml_sycl_pool_alloc w8(ctx.pool(), N * K); + ggml_sycl_pool_alloc x8(ctx.pool(), ntok * K); + ggml_sycl_pool_alloc xscale(ctx.pool(), ntok); + int8_t * ratio_p = ratio.get(); + float * ws_p = wscale.get(); + int8_t * w8_p = w8.get(); + // per row: D = max group scale, ratio = round(127 d_g / D) + stream->parallel_for(sycl::nd_range<1>(N * 16, 16), [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(16)]] { + const int64_t r = it.get_group(0); + const int lane = it.get_local_id(0); + auto dget = [&](int64_t g) -> float { return (float) sycl::bit_cast(SB[g * N + r]); }; + float m = 0.0f; + for (int64_t g = lane; g < ng; g += 16) { + m = sycl::fmax(m, sycl::fabs(dget(g))); + } + m = sycl::reduce_over_group(it.get_sub_group(), m, sycl::maximum()); + for (int64_t g = lane; g < ng; g += 16) { + ratio_p[r * ng + g] = m > 0 ? (int8_t) sycl::round(127.0f * dget(g) / m) : 0; + } + if (lane == 0) { + ws_p[r] = m / 127.0f; + } + }); + // int8 weights [N][K]: one work item per (16-weight word, row), rows fastest (coalesced word reads) + stream->parallel_for(sycl::range<1>((size_t) (K / 16) * N), [=](sycl::id<1> id) { + const int64_t kw = id[0] / N, n = id[0] % N; + const uint32_t c = B[kw * N + n]; + const int r8 = ratio_p[n * ng + (kw * 16) / 128]; + uint32_t o[4]; +#pragma unroll + for (int q = 0; q < 4; ++q) { + uint32_t pk = 0; +#pragma unroll + for (int j = 0; j < 4; ++j) { + const int v = ((int) ((c >> (2 * (4 * q + j))) << 30)) >> 30; // sign-extend the 2-bit code + pk |= (uint32_t) (uint8_t) (int8_t) (v * r8) << (8 * j); + } + o[q] = pk; + } + uint32_t * dstw = (uint32_t *) (w8_p + n * K + kw * 16); + dstw[0] = o[0]; dstw[1] = o[1]; dstw[2] = o[2]; dstw[3] = o[3]; + }); + GGML_ASSERT(x_stride == K); + w8a8_acts(x, x8.get(), xscale.get(), ntok, K, stream); + DnnlGemmWrapper::gemm_s8(ctx, (int) N, (int) ntok, (int) K, x8.get(), w8_p, dst, stream); + const float * xs = xscale.get(); + stream->parallel_for(sycl::range<1>(ntok * N), [=](sycl::id<1> i) { + dst[i] *= xs[i[0] / N] * ws_p[i[0] % N]; + }); + return true; +} + +#else + +bool ggml_sycl_w8a8_mul_mat(ggml_backend_sycl_context &, ggml_type, bool, const void *, const float *, float *, int64_t, + int64_t, int64_t, dpct::queue_ptr) { + return false; +} + +bool ggml_sycl_w8a8_mul_mat_t2(ggml_backend_sycl_context &, const void *, const float *, int64_t, float *, int64_t, int64_t, + int64_t, dpct::queue_ptr) { + return false; +} + +#endif diff --git a/ggml/src/ggml-sycl/w8a8.hpp b/ggml/src/ggml-sycl/w8a8.hpp new file mode 100644 index 000000000000..492512a5c0bd --- /dev/null +++ b/ggml/src/ggml-sycl/w8a8.hpp @@ -0,0 +1,14 @@ +#pragma once +#include "common.hpp" + +// Prompt-sized mat-mul for PQ2_0 / PTQ1_0 weights on int8 XMX via oneDNN s8 x s8. +// Per-128-group weight scales are requantized to a per-row scale (int8 weight = code * round(127 * d_g / max_g d)), +// activations are quantized to int8 per token; both scales are applied in one pass over the output. +// Returns false when not applicable (caller falls back). +bool ggml_sycl_w8a8_mul_mat(ggml_backend_sycl_context & ctx, ggml_type type, bool reordered, const void * w, + const float * x, float * dst, int64_t nrows, int64_t ncols_x, int64_t K, + dpct::queue_ptr stream); + +// ARC-LAB: int8 oneDNN GEMM straight from the TernSYCL 2-bit layout (large prompt batches) +bool ggml_sycl_w8a8_mul_mat_t2(ggml_backend_sycl_context & ctx, const void * w, const float * x, int64_t x_stride, + float * dst, int64_t nrows, int64_t ntok, int64_t K, dpct::queue_ptr stream); diff --git a/ggml/src/ggml-sycl/xe2-dpas.hpp b/ggml/src/ggml-sycl/xe2-dpas.hpp new file mode 100644 index 000000000000..c814bd6da7db --- /dev/null +++ b/ggml/src/ggml-sycl/xe2-dpas.hpp @@ -0,0 +1,82 @@ +// ARC-LAB: Xe2 s8 x s4 DPAS as inline vISA (as libxsmm/TernSYCL's xe2.hpp dpas helpers). It replaces the IGC builtin +// intel_sub_group_i8_i4_matrix_mad_k32, which is not in the public extension list: drivers that don't provide it (e.g. Lunar +// Lake, 2026-10) failed to load the whole device image ("No device image found for external symbol"), crashing every model. +// SIMD16, systolic depth 8, repeat count 8: A = 8 rows of K32 s8 (one short = 2 s8 per lane per row), B = K32 x 16 s4 (one +// int4 = 16 bytes of nibbles per lane / column), zero accumulator. Same result as the builtin with acc = 0. +#pragma once + +#include + +namespace xe2dp { +typedef short s8x8 __attribute__((ext_vector_type(8))); +typedef int s4x4 __attribute__((ext_vector_type(4))); +typedef int i32x8 __attribute__((ext_vector_type(8))); + +inline i32x8 dpas_s4s8_r8(s8x8 a, s4x4 b) { + i32x8 d; +#ifdef __SYCL_DEVICE_ONLY__ + __asm__("{\n" + ".decl DB v_type=G type=ud num_elts=64 align=GRF alias=<%1,0>\n" + ".decl DA v_type=G type=ud num_elts=64 align=GRF alias=<%2,0>\n" + "dpas.s4.s8.8.8 (M1, 16) %0.0 %%null.0 DB.0 DA(0,0)\n" + "}\n" + : "=rw"(d) + : "rw"(b), "rw"(a)); +#else + d = 0; + (void) a; + (void) b; +#endif + return d; +} +// f16 x f16 -> f32, repeat count 8: A = 8 rows of K16 halves (one short per lane per row), B = K16 x 16 (VNNI: 8 dwords +// per lane), accumulate. Replaces intel_sub_group_f16_f16_matrix_mad_k16 (public, but one builtin fewer to resolve). +typedef float f32x8 __attribute__((ext_vector_type(8))); +inline f32x8 dpas_hf_r8(s8x8 a, i32x8 b, f32x8 acc) { +#ifdef __SYCL_DEVICE_ONLY__ + __asm__("{\n" + ".decl DB v_type=G type=ud num_elts=128 align=GRF alias=<%1,0>\n" + ".decl DA v_type=G type=ud num_elts=64 align=GRF alias=<%2,0>\n" + "dpas.hf.hf.8.8 (M1, 16) %0.0 %0.0 DB.0 DA(0,0)\n" + "}\n" + : "+rw"(acc) + : "rw"(b), "rw"(a)); +#else + (void) a; + (void) b; +#endif + return acc; +} +// ARC-LAB: transposed 2D block load (lsc_load_block2d, d32, 8 dwords x 16 rows): lane l receives row y + l, component c = +// dword x + c of that row. Surface: base (64-byte aligned), width / height / pitch minus one (bytes, rows, bytes; pitch a +// multiple of 16); rows or dwords outside the surface read as zero. One message replaces 8 per-lane dword gathers. +// The arguments must be uniform over the subgroup; lane 0's values are used (copied into scalars here, so IGC need not +// prove uniformity: "rw.u" operands failed its inline-asm validation after a spill recompile). +typedef unsigned u32x8 __attribute__((ext_vector_type(8))); +inline u32x8 ld2d_t32_8x16(long long base, int wm1, int hm1, int pm1, int x, int y) { + u32x8 v; +#ifdef __SYCL_DEVICE_ONLY__ + __asm__("{\n" + ".decl SB v_type=G type=q num_elts=1 align=qword\n" + ".decl SW v_type=G type=d num_elts=1 align=dword\n" + ".decl SH v_type=G type=d num_elts=1 align=dword\n" + ".decl SP v_type=G type=d num_elts=1 align=dword\n" + ".decl SX v_type=G type=d num_elts=1 align=dword\n" + ".decl SY v_type=G type=d num_elts=1 align=dword\n" + "mov (M1_NM, 1) SB(0,0)<1> %1(0,0)<0;1,0>\n" + "mov (M1_NM, 1) SW(0,0)<1> %2(0,0)<0;1,0>\n" + "mov (M1_NM, 1) SH(0,0)<1> %3(0,0)<0;1,0>\n" + "mov (M1_NM, 1) SP(0,0)<1> %4(0,0)<0;1,0>\n" + "mov (M1_NM, 1) SX(0,0)<1> %5(0,0)<0;1,0>\n" + "mov (M1_NM, 1) SY(0,0)<1> %6(0,0)<0;1,0>\n" + "lsc_load_block2d.ugm (M1, 1) %0:d32.8x16tn flat[SB,SW,SH,SP,SX,SY]\n" + "}\n" + : "=rw"(v) + : "rw"(base), "rw"(wm1), "rw"(hm1), "rw"(pm1), "rw"(x), "rw"(y)); +#else + v = 0; + (void) base; (void) wm1; (void) hm1; (void) pm1; (void) x; (void) y; +#endif + return v; +} +} // namespace xe2dp diff --git a/src/llama-context.cpp b/src/llama-context.cpp index cb6cf1f2854a..77eb959cae7e 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -678,6 +678,10 @@ void llama_context::resolve_fused_ops(const llama_memory_context_i * mctx, uint3 LLAMA_LOG_INFO("%s: resolving fused Gated Delta Net support:\n", func); resolve(llm_fused_op_gdn_ar_probe, cparams.fused_gdn_ar); resolve(llm_fused_op_gdn_ch_probe, cparams.fused_gdn_ch); + if (const char * e = getenv("LLAMA_FUSED_GDN_CH"); e && atoi(e) == 0) { + cparams.fused_gdn_ch = false; // A/B: use the graph-level chunked delta net for prompt batches + LLAMA_LOG_INFO("%s: fused GDN (chunked) disabled by LLAMA_FUSED_GDN_CH=0\n", func); + } cparams.auto_fgdn = false; } diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 6f6d7b4b5538..327a63de1ed2 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -4848,6 +4848,195 @@ struct test_fwht_signed : public test_case { } }; +// ARC-LAB fusion tests (arctune, 2026-10-05): each builds the exact node pattern one SYCL fusion matches, at several +// shapes / token counts, and compares the fused outputs against the CPU backend. The graph's final node only keeps every +// output alive; its operand order makes the graph come out in the model's node order. `seed` repeats a case with fresh +// random data. Run with GGML_SYCL_PTQ1_T2=all so the ternary weights take the T2 (XMX) path the fusions feed. +static void arclab_init_hadamard(ggml_tensor * t) { + const int64_t n = t->ne[0]; + std::vector data(n * n); + const float scale = 1.0f / sqrtf((float) n); + for (int64_t r = 0; r < n; r++) { + for (int64_t i = 0; i < n; i++) { + data[r * n + i] = (__builtin_popcountll((unsigned long long) (r & i)) % 2 == 0) ? scale : -scale; + } + } + ggml_backend_tensor_set(t, data.data(), 0, data.size() * sizeof(float)); +} +static void arclab_init_signs(ggml_tensor * t) { + std::vector data(ggml_nelements(t)); + for (size_t i = 0; i < data.size(); i++) { + data[i] = ((i * 2654435761u) >> 7) & 1 ? -1.0f : 1.0f; + } + ggml_backend_tensor_set(t, data.data(), 0, data.size() * sizeof(float)); +} +static void arclab_init(ggml_context * ctx) { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (t->op != GGML_OP_NONE) { + continue; + } + if (strcmp(t->name, "hadamard") == 0) { + arclab_init_hadamard(t); + } else if (strncmp(t->name, "signs", 5) == 0) { + arclab_init_signs(t); + } else { + init_tensor_uniform(t); + } + } +} +// rotated input -> T2 mat-vec, as llama_mul_mat_hadamard + build_lora_mm build it +static ggml_tensor * arclab_rot_matmul(ggml_context * ctx, ggml_tensor * cur, ggml_tensor * signs, ggml_tensor * had, + ggml_tensor * w, ggml_tensor ** mm_out) { + const int64_t K = cur->ne[0], M = cur->ne[1]; + ggml_tensor * ms = ggml_mul(ctx, cur, signs); + ggml_tensor * rs = ggml_reshape_2d(ctx, ms, had->ne[0], ggml_nelements(ms) / had->ne[0]); + ggml_tensor * mm = ggml_mul_mat(ctx, had, rs); + ggml_mul_mat_set_hint(mm, GGML_HINT_SRC0_IS_HADAMARD); + ggml_set_name(mm, "rotated"); + *mm_out = mm; + ggml_tensor * back = ggml_reshape_2d(ctx, mm, K, M); + ggml_tensor * out = ggml_mul_mat(ctx, w, back); + ggml_set_name(out, "t2_out"); + return out; +} + +// SWIGLU -> MUL(signs) -> RESHAPE -> Hadamard MUL_MAT -> T2 mat-vec (GGML_SYCL_GLU_ROT) +struct test_arclab_glu_rot : public test_case { + const int64_t K, M, N; const int seed; + std::vector outs; + test_arclab_glu_rot(int64_t K, int64_t M, int64_t N, int seed) : K(K), M(M), N(N), seed(seed) {} + std::string op_desc(ggml_tensor *) override { return "ARCLAB_GLU_ROT"; } + std::string vars() override { return VARS_TO_STR4(K, M, N, seed); } + bool run_whole_graph() override { return true; } + std::vector fusion_test_nodes() override { return outs; } + double max_nmse_err() override { return 5e-4; } + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * g = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, K, M); ggml_set_name(g, "gate"); + ggml_tensor * u = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, K, M); ggml_set_name(u, "up"); + ggml_tensor * sg = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, K); ggml_set_name(sg, "signs"); + ggml_tensor * had = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 1024, 1024); ggml_set_name(had, "hadamard"); + ggml_tensor * w = ggml_new_tensor_2d(ctx, GGML_TYPE_PTQ1_0, K, N); ggml_set_name(w, "w"); + ggml_tensor * x = ggml_swiglu_split(ctx, g, u); + ggml_tensor * mm; + ggml_tensor * out = arclab_rot_matmul(ctx, x, sg, had, w, &mm); + outs = { mm, out }; + return out; + } + void initialize_tensors(ggml_context * ctx) override { arclab_init(ctx); } +}; + +// ADD -> RMS_NORM -> MUL(w) -> MUL(signs) -> RESHAPE -> Hadamard MUL_MAT -> T2 mat-vec (GGML_SYCL_PTQ1_T2_NORMQ). The +// far variant (GGML_SYCL_NORM_ROT_FAR: cache ops between norm and rotation) cannot be built here: a single-output graph +// has no node to put in between; it is covered by the whole-model checks (byte-identical outputs). +struct test_arclab_add_norm_rot : public test_case { + const int64_t K, M, N; const bool far; const int seed; + std::vector outs; + test_arclab_add_norm_rot(int64_t K, int64_t M, int64_t N, bool far, int seed) : K(K), M(M), N(N), far(far), seed(seed) {} + std::string op_desc(ggml_tensor *) override { return "ARCLAB_ADD_NORM_ROT"; } + std::string vars() override { return VARS_TO_STR5(K, M, N, far, seed); } + bool run_whole_graph() override { return true; } + std::vector fusion_test_nodes() override { return outs; } + double max_nmse_err() override { return 5e-4; } + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, K, M); ggml_set_name(a, "a"); + ggml_tensor * b = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, K, M); ggml_set_name(b, "b"); + ggml_tensor * nw = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, K); ggml_set_name(nw, "norm_w"); + ggml_tensor * sg = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, K); ggml_set_name(sg, "signs"); + ggml_tensor * had = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 1024, 1024); ggml_set_name(had, "hadamard"); + ggml_tensor * w = ggml_new_tensor_2d(ctx, GGML_TYPE_PTQ1_0, K, N); ggml_set_name(w, "w"); + ggml_tensor * sum = ggml_add(ctx, a, b); + ggml_tensor * normed = ggml_mul(ctx, ggml_rms_norm(ctx, sum, 1e-6f), nw); + ggml_tensor * mm; + ggml_tensor * t2 = arclab_rot_matmul(ctx, normed, sg, had, w, &mm); + // keep the residual and the normed rows alive after the chain (the next layer and other mat-vecs read them) + ggml_tensor * out = ggml_add(ctx, ggml_sum(ctx, t2), ggml_add(ctx, ggml_sum(ctx, normed), ggml_sum(ctx, sum))); + outs = { sum, normed, mm, t2 }; + return out; + } + void initialize_tensors(ggml_context * ctx) override { arclab_init(ctx); } +}; + +// DeltaNet conv chain for one token: CONCAT(state, qkv^T) -> CPY(newest columns -> cache) -> SSM_CONV -> SILU -> L2_NORM +// (GGML_SYCL_CONV_CHAIN) +struct test_arclab_conv_chain : public test_case { + const int64_t C, S; const int seed; + std::vector outs; + test_arclab_conv_chain(int64_t C, int64_t S, int seed) : C(C), S(S), seed(seed) {} + std::string op_desc(ggml_tensor *) override { return "ARCLAB_CONV_CHAIN"; } + std::string vars() override { return VARS_TO_STR3(C, S, seed); } + bool run_whole_graph() override { return true; } + std::vector fusion_test_nodes() override { return outs; } + double max_nmse_err() override { return 1e-6; } + ggml_tensor * build_graph(ggml_context * ctx) override { + const int64_t dc = 4, H = 2 * C / (5 * S); // q and k heads: the first 2 C / 5 channels (Bonsai: 32 heads of 128) + ggml_tensor * cs = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, dc - 1, C); ggml_set_name(cs, "conv_state"); + ggml_tensor * x = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, C, 1); ggml_set_name(x, "qkv"); + ggml_tensor * w = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, dc, C); ggml_set_name(w, "conv_w"); + ggml_tensor * cache = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, (dc - 1) * C); ggml_set_name(cache, "cache"); + ggml_tensor * cat = ggml_concat(ctx, cs, ggml_transpose(ctx, x), 0); + ggml_tensor * last = ggml_view_2d(ctx, cat, dc - 1, C, cat->nb[1], sizeof(float)); + ggml_tensor * cpy = ggml_cpy(ctx, last, cache); + ggml_tensor * conv = ggml_ssm_conv(ctx, cat, w); // the concat itself, as in the model + ggml_tensor * silu = ggml_silu(ctx, conv); + ggml_tensor * qk = ggml_view_2d(ctx, silu, S, H, S * sizeof(float), 0); + ggml_tensor * l2 = ggml_l2_norm(ctx, qk, 1e-6f); + ggml_tensor * out = ggml_add(ctx, ggml_sum(ctx, cpy), ggml_add(ctx, ggml_sum(ctx, l2), ggml_sum(ctx, silu))); + outs = { cpy, silu, l2 }; + return out; + } + void initialize_tensors(ggml_context * ctx) override { arclab_init(ctx); } +}; + +// DeltaNet gate block: bf16 alpha / beta mat-vecs + bias add + softplus + ssm_a mul + sigmoid (GGML_SYCL_DN_GATES) +struct test_arclab_dn_gates : public test_case { + const int64_t K, R, M; const int seed; + test_arclab_dn_gates(int64_t K, int64_t R, int64_t M, int seed) : K(K), R(R), M(M), seed(seed) {} + std::string op_desc(ggml_tensor *) override { return "ARCLAB_DN_GATES"; } + std::string vars() override { return VARS_TO_STR4(K, R, M, seed); } + bool run_whole_graph() override { return true; } + double max_nmse_err() override { return 5e-4; } + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * x = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, K, M); ggml_set_name(x, "x"); + ggml_tensor * wa = ggml_new_tensor_2d(ctx, GGML_TYPE_BF16, K, R); ggml_set_name(wa, "w_alpha"); + ggml_tensor * wb = ggml_new_tensor_2d(ctx, GGML_TYPE_BF16, K, R); ggml_set_name(wb, "w_beta"); + ggml_tensor * bias = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, R); ggml_set_name(bias, "dt_bias"); + ggml_tensor * sa = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, R); ggml_set_name(sa, "ssm_a"); + ggml_tensor * alpha = ggml_reshape_2d(ctx, ggml_mul_mat(ctx, wa, x), R, M); + ggml_tensor * gate = ggml_mul(ctx, ggml_softplus(ctx, ggml_add(ctx, alpha, bias)), sa); + ggml_tensor * gate3 = ggml_reshape_3d(ctx, gate, 1, R, M); + ggml_tensor * beta = ggml_sigmoid(ctx, ggml_reshape_3d(ctx, ggml_mul_mat(ctx, wb, x), 1, R, M)); + ggml_tensor * out = ggml_concat(ctx, gate3, beta, 0); + return out; + } + void initialize_tensors(ggml_context * ctx) override { arclab_init(ctx); } +}; + +// same-input T2 mat-vecs merged into one launch (GGML_SYCL_T2_MULTI): 2 or 3 ternary weights on one activation +struct test_arclab_t2_multi : public test_case { + const int64_t K, M; const std::array N; const int seed; + std::vector outs; + test_arclab_t2_multi(int64_t K, int64_t M, std::array N, int seed) : K(K), M(M), N(N), seed(seed) {} + std::string op_desc(ggml_tensor *) override { return "ARCLAB_T2_MULTI"; } + std::string vars() override { return VARS_TO_STR4(K, M, N, seed); } + bool run_whole_graph() override { return true; } + std::vector fusion_test_nodes() override { return outs; } + double max_nmse_err() override { return 5e-4; } + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * x = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, K, M); ggml_set_name(x, "x"); + outs.clear(); + ggml_tensor * acc = nullptr; + for (int m = 0; m < 3; ++m) { + if (N[m] == 0) continue; + ggml_tensor * w = ggml_new_tensor_2d(ctx, GGML_TYPE_PTQ1_0, K, N[m]); + ggml_tensor * y = ggml_mul_mat(ctx, w, x); + outs.push_back(y); + acc = acc ? ggml_add(ctx, acc, ggml_sum(ctx, y)) : ggml_sum(ctx, y); + } + return acc; + } + void initialize_tensors(ggml_context * ctx) override { arclab_init(ctx); } +}; + static void init_mul_mat_id_tensors(ggml_context * ctx, int n_mats) { std::random_device rd; std::default_random_engine rng(rd()); @@ -9427,6 +9616,21 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_fwht_signed(128, 1024, 5, GGML_TYPE_F32, true)); test_cases.emplace_back(new test_fwht_signed(2048, 8192, 3, GGML_TYPE_F32, true)); test_cases.emplace_back(new test_fwht_signed(1024, 5120, 4, GGML_TYPE_F16, true)); + // ARC-LAB fusion tests (arctune): Bonsai 27B shapes and token counts, 3 seeds each + for (int seed = 0; seed < 3; ++seed) { + for (int64_t M : {1, 4, 5, 8}) { + test_cases.emplace_back(new test_arclab_glu_rot(17408, M, 512, seed)); + test_cases.emplace_back(new test_arclab_add_norm_rot(5120, M, 512, false, seed)); + test_cases.emplace_back(new test_arclab_dn_gates(5120, 48, M, seed)); + test_cases.emplace_back(new test_arclab_t2_multi(5120, M, {10240, 6144, 0}, seed)); + test_cases.emplace_back(new test_arclab_t2_multi(5120, M, {12288, 1024, 1024}, seed)); + } + test_cases.emplace_back(new test_arclab_glu_rot(4096, 16, 256, seed)); + test_cases.emplace_back(new test_arclab_add_norm_rot(2048, 2, 256, false, seed)); + test_cases.emplace_back(new test_arclab_dn_gates(2048, 32, 2, seed)); + test_cases.emplace_back(new test_arclab_conv_chain(10240, 128, seed)); + test_cases.emplace_back(new test_arclab_conv_chain(3072, 64, seed)); + } // Block widths above the register path's reach, plus a couple below it as controls. 4096 and // 8192 exercise the shared-memory kernel; before it existed the CUDA backend declined them and // the op fell back, which cost both speed and (measurably) a little accuracy. @@ -9476,6 +9680,14 @@ static std::vector> make_test_cases_eval() { } } + // ARC-LAB: ternary int8 coopmat GEMM path (Vulkan, GGML_VK_TERN_I8_MIN_N): rows % 128 == 0, several N incl. partial tiles + for (ggml_type t : {GGML_TYPE_PTQ1_0, GGML_TYPE_PQ2_0}) { + for (int n : {9, 64, 100}) { + for (int k : {1024, 5120}) { + test_cases.emplace_back(new test_mul_mat(t, GGML_TYPE_F32, 256, n, k, {1, 1}, {1, 1})); + } + } + } // PTQ1_0 / PQ2_0 integer-dot mat-vec: Bonsai-2 shapes, odd row counts (row tail), batches and multi-column B for (int64_t n : {1, 2, 3, 4, 5, 6, 7, 8}) { for (int64_t k : {1024, 5120, 6144, 17408}) { @@ -10324,6 +10536,57 @@ static std::vector> make_test_cases_eval() { } // mixed quant and Q1_0 test cases + // ARC-LAB: q4_0-cache decode kernel (SYCL fattn-dec): Bonsai 27B shape (D 256, 4 KV heads x GQA 6), 1-4 query tokens + for (int64_t kv : {256, 512, 4096, 16384}) { + for (int64_t nb : {1, 2, 3, 4, 5, 8}) { + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)); + } + } + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 1024, 1, false, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)); + // ARC-LAB: DPAS decode kernel (GGML_SYCL_FA_DEC_DPAS): longer verify batches (token chunks) and the real cache layout + for (int64_t kv : {512, 4096}) { + for (int64_t nb : {12, 16, 17, 32}) { + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)); + } + for (int64_t nb : {1, 3, 4, 7, 16}) { + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, {0, 2, 1, 3})); + } + } + // ARC-LAB: Gemma 4 global layers on the DPAS decode kernel (head 512, 1 KV head x 16 query heads, q4_0 cache) + for (int64_t kv : {512, 4096}) { + for (int64_t nb : {1, 2, 3, 4, 8}) { + test_cases.emplace_back(new test_flash_attn_ext(512, 512, 1, {16, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)); + test_cases.emplace_back(new test_flash_attn_ext(512, 512, 1, {16, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, {0, 2, 1, 3})); + } + } + // ARC-LAB: reordered q4_0 small-batch XMX GEMM (GGML_SYCL_Q4_0_DPAS): Gemma 4 shapes, 5..64 columns + for (auto mk : std::vector>{{4096, 3840}, {3840, 15360}}) { + for (int64_t n : {1, 5, 8, 9, 16, 33, 64, 65, 129, 256}) { + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, mk[0], n, mk[1], {1, 1}, {1, 1})); + } + } + // ARC-LAB: Gemma 4 sliding layers on the DPAS decode kernel (head 256, 8 KV heads x 2 query heads, q4_0 cache) + for (int64_t kv : {512, 1024}) { + for (int64_t nb : {1, 2, 5, 8, 12}) { + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 8, {2, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, {0, 2, 1, 3})); + } + } + // ARC-LAB: oneDNN / MKL prompt-FA correctness sweep in the real cache layout (a report showed kv=1025 / 4096 failing) + for (int64_t kv : {1024, 1025, 2048, 3072, 4095, 4096, 4097, 6144, 8192}) { + for (int64_t nb : {33, 64, 512}) { + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, {0, 2, 1, 3})); + } + } + // ARC-LAB: Intel prompt FA (Vulkan fa_pf_gqa): prompt batches at the Bonsai shape, f16 and q4_0 (f16 scratch), both layouts + for (ggml_type t : {GGML_TYPE_F16, GGML_TYPE_Q4_0}) { + for (int64_t nb : {16, 37, 64}) { + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 512, nb, true, false, 0, 0, GGML_PREC_F32, t, t, {0, 2, 1, 3})); + } + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 1024, 64, true, false, 0, 0, GGML_PREC_F32, t, t)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 2}, 512, 20, true, false, 0, 0, GGML_PREC_F32, t, t, {0, 2, 1, 3})); + } + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 512, 64, false, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 2}, 512, 4, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)); test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0)); test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_F16)); test_cases.emplace_back(new test_flash_attn_ext(72, 72, 4, {1, 1}, 96, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0)); @@ -10459,6 +10722,17 @@ static std::vector> make_test_cases_eval() { false, 1, 1, false, false, true, false, {1, 1})); } + // ARC-LAB: the MTP draft head's Q6_K weights at their shapes, 1..6 tokens (draft steps, chain step 0): the mat-vecs + // (GGML_SYCL_Q6K_GEMV / _U / _SG) and the FFN gate+up+SWIGLU (GGML_SYCL_Q6K_GLU=1 takes the fused kernel) + for (int n : {1, 2, 6}) { + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K, GGML_TYPE_F32, 12288, n, 5120, {1, 1}, {1, 1})); // q + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K, GGML_TYPE_F32, 5120, n, 10240, {1, 1}, {1, 1})); // eh_proj + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K, GGML_TYPE_F32, 5120, n, 6144, {1, 1}, {1, 1})); // wo + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K, GGML_TYPE_F32, 5120, n, 17408, {1, 1}, {1, 1})); // down + test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_Q6_K, GGML_GLU_OP_SWIGLU, n, 17408, 5120, + false, 1, 1, false, false, true, false, {1, 1})); // gate+up + } + for (auto gate : {GATING_FUNC_SOFTMAX, GATING_FUNC_SIGMOID, GATING_FUNC_SOFTMAX_WEIGHT, GATING_FUNC_SQRT_SOFTPLUS}) { for (bool with_norm : {false, true}) { for (bool bias_probs : {false, true}) { @@ -10479,6 +10753,19 @@ static std::vector> make_test_cases_eval() { } test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 1, 1)); + // ARC-LAB: chunked prefill path (SYCL, S = 128, one sequence, >= 64 tokens): full / partial chunks, GQA x3, + // snapshot hand-off (K > 1: last K tokens via the sequential kernel), permuted q/k + for (int nt : {64, 100, 1024}) { + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, nt, 1, 3)); + } + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 100, 1, 3, false, false, 4)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 1024, 1, 3, false, false, 17)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 100, 1, 3, true)); + // ARC-LAB: token-blocked sequential kernel (SYCL, S = 128, >= 16 tokens): several sequences, snapshots, short batches + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 64, 2, 3)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 64, 2, 3, false, false, 4)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 37, 3, 3, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 16, 2, 1, false, false, 3)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 16, 1, 1)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 16, 1, 1, 1, true, true)); // raw gates (sigmoid / softplus folded into the op): decode, prefill, rows mode @@ -10574,12 +10861,28 @@ static std::vector> make_test_cases_eval() { // Test cases for performance evaluation: should be representative of real-world use cases static std::vector> make_test_cases_perf() { std::vector> test_cases; + + // ARC-LAB: Bonsai 2 GDN layer shape (16 k-heads x 128, 48 v-heads): prompt batch and decode + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 1024, 1, 3)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 1, 1, 3)); + // ARC-LAB: MTP draft head (Q6_K) mat-vecs at their shapes, 1..5 tokens: q, eh_proj, wo, ffn_down, ffn_gate / up + for (int n : {1, 2, 3, 4, 5}) { + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K, GGML_TYPE_F32, 12288, n, 5120, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K, GGML_TYPE_F32, 5120, n, 10240, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K, GGML_TYPE_F32, 5120, n, 6144, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K, GGML_TYPE_F32, 5120, n, 17408, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K, GGML_TYPE_F32, 17408, n, 5120, {1, 1}, {1, 1})); + } // bandwidth comparison at Bonsai-2 shapes for (ggml_type t : {GGML_TYPE_PTQ1_0, GGML_TYPE_PQ2_0, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q2_K, GGML_TYPE_TQ2_0}) { test_cases.emplace_back(new test_mul_mat(t, GGML_TYPE_F32, 17408, 1, 5120, {1, 1}, {1, 1})); test_cases.emplace_back(new test_mul_mat(t, GGML_TYPE_F32, 5120, 1, 17408, {1, 1}, {1, 1})); test_cases.emplace_back(new test_mul_mat(t, GGML_TYPE_F32, 10240, 1, 5120, {1, 1}, {1, 1})); } + // ARC-LAB: prompt-sized GEMM ceilings at the Bonsai FFN shape - ternary vs f16 and tuned quants + for (ggml_type t : {GGML_TYPE_PTQ1_0, GGML_TYPE_PQ2_0, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0}) { + test_cases.emplace_back(new test_mul_mat(t, GGML_TYPE_F32, 17408, 512, 5120, {1, 1}, {1, 1})); + } // batched decode (several sequences per step) through the mat-vec path for (ggml_type t : {GGML_TYPE_PTQ1_0, GGML_TYPE_PQ2_0, GGML_TYPE_Q4_0}) { for (int n : {2, 3, 4, 8}) { @@ -10796,10 +11099,15 @@ static std::vector> make_test_cases_perf() { } // Ternary Bonsai 2 27B (qwen35): the bf16 gated-delta-net gate projections - for (int bs : {1, 2, 3, 4, 8}) { + for (int bs : {1, 2, 3, 4, 8, 96, 512, 2048}) { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 48, bs, 5120, {1, 1}, {1, 1})); // ssm_alpha, ssm_beta } + // ARC-LAB: Bonsai 2 27B fused FFN gate/up + SwiGLU (PTQ1_0) at decode / MTP-verify batch sizes + for (int bs : {1, 4}) { + test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_PTQ1_0, GGML_GLU_OP_SWIGLU, 17408, bs, 5120, + false, 1, 1, false, false, true, false, {1, 1})); + } // Ternary Bonsai 2 27B (qwen35, PTQ1_0) projections at speculative-decoding batch sizes for (int bs : {1, 2, 3, 4, 8}) { for (ggml_type type_a : {GGML_TYPE_PTQ1_0, GGML_TYPE_Q4_0}) { @@ -10811,6 +11119,9 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 12288, bs, 5120, {1, 1}, {1, 1})); // attn_q } } + // ARC-LAB: the smallest and largest Bonsai 27B mat-vecs (attn k / v, output head) + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_PTQ1_0, GGML_TYPE_F32, 1024, 1, 5120, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_PTQ1_0, GGML_TYPE_F32, 248320, 1, 5120, {1, 1}, {1, 1})); // qwen3-30b-a3b for (int bs : {1, 4, 8, 32, 64, 128, 256, 512}) { @@ -10857,6 +11168,42 @@ static std::vector> make_test_cases_perf() { // Qwen3-VL-8B https://github.com/ggml-org/llama.cpp/issues/17012 test_cases.emplace_back(new test_flash_attn_ext(72, 72, 16, {1, 1}, 5776, 5776, false, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + // ARC-LAB: Gemma 4 12B QAT q4_0 weight shapes at decode / verify batch sizes (q4_0 weight-kernel work) + for (auto mk : std::vector>{{15360, 3840}, {3840, 15360}, {4096, 3840}, {2048, 3840}, {3840, 4096}}) { + for (int64_t n : {1, 2, 4, 8, 16}) { + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, mk[0], n, mk[1], {1, 1}, {1, 1})); + } + } + for (int64_t kv : {16384, 49152}) { // ARC-LAB: Bonsai 27B decode / MTP verify on a q4_0 cache + for (int64_t nb : {1, 4, 5, 8}) { + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)); + } + for (int64_t nb : {1, 4}) { // ARC-LAB: Gemma 4 global layers, q4_0 vs f16 cache + for (ggml_type t : {GGML_TYPE_Q4_0, GGML_TYPE_F16}) { + test_cases.emplace_back(new test_flash_attn_ext(512, 512, 1, {16, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, t, t, {0, 2, 1, 3})); + } + } + for (int64_t nb : {1, 4, 5, 6, 8, 16, 32, 64, 128, 256}) { // real cache layout (DPAS kernel targets; 5 = MTP verify; 64+ = long n-gram drafts) + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, {0, 2, 1, 3})); + } + } + for (int64_t nkv : {1, 2}) { // ARC-LAB: fewer KV heads, same 6 q heads per KV head: per-head cost without the 4-head row interleave + test_cases.emplace_back(new test_flash_attn_ext(256, 256, nkv, {6, 1}, 49152, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, {0, 2, 1, 3})); + } + for (int64_t nb : {1, 4, 64, 256}) { // ARC-LAB: ~115K history (Bonsai at 128K context), real cache layout + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 114688, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, {0, 2, 1, 3})); + } + for (int64_t kv : {16384, 65536, 131072}) { // ARC-LAB: Bonsai verify shapes (1 token, MTP 5, n-gram 8-16), real cache layout + for (int64_t nb : {1, 5, 8, 12, 16}) { + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, {0, 2, 1, 3})); + } + } + for (ggml_type t : {GGML_TYPE_Q4_0, GGML_TYPE_F16}) { // ARC-LAB: Bonsai 27B prompt batch at 16K history + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 16384, 512, true, false, 0, 0, GGML_PREC_F32, t, t)); + for (int64_t kv : {512, 16384}) { // real KV-cache layout (heads interleaved per token) + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, kv, 512, true, false, 0, 0, GGML_PREC_F32, t, t, {0, 2, 1, 3})); + } + } test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 4, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)); @@ -10997,6 +11344,10 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 64, 1, 1)); // smaller model test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 1, 1, 1, false, true)); // KDA // PP: n_seq_tokens=64,256 (prompt processing) + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 1024, 1, 3, false, false, 4)); // ARC-LAB: live server keeps snapshots (K > 1) + for (int nt : {16, 32, 64, 128}) { // ARC-LAB: Bonsai shape (48 v-heads), draft-verify batch sizes + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, nt, 1, 3)); + } test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 64, 1)); // PP-64 test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 256, 1)); // PP-256 test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 512, 1)); // PP-512