From f5507194cb434973ebfa4e37462c9e26b371fe0f Mon Sep 17 00:00:00 2001 From: bri-prism <288398250+bri-prism@users.noreply.github.com> Date: Sun, 11 Oct 2026 17:31:14 -0700 Subject: [PATCH] cuda: keep the quantized K/V type in the short-query MMA FA fallback The Q length 3-4 quantized-V fallback in ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1 always launched the F16 instance of the ncols=64 kernel. When the call came from the in-place q4_0/q8_0 K/V dispatch (hs 128/256, K and V the same type), that F16 instance converts K and V to F16 into scratch after dst, but ggml_cuda_flash_attn_ext_get_alloc_size reserves no such scratch for the in-place kernels. The conversion then wrote past the dst allocation and FLASH_ATTN_EXT hit an illegal memory access (e.g. hsk=256, nr23=[6,1], kv=1024, nb=3, q4_0/q8_0). Pass type_KV through so the fallback uses the matching in-place instance. --- ggml/src/ggml-cuda/fattn.cu | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu index 03d925bb6025..929fd1d31480 100644 --- a/ggml/src/ggml-cuda/fattn.cu +++ b/ggml/src/ggml-cuda/fattn.cu @@ -157,8 +157,10 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ggml_backend_cuda_con const ggml_tensor * Q = dst->src[0]; const ggml_tensor * V = dst->src[2]; + // Keep type_KV: for the in-place quantized K/V kernels ggml_cuda_flash_attn_ext_get_alloc_size reserves no F16 + // K/V scratch after dst, so falling back to the F16 instance here would convert the cache past the allocation. if (turing_mma_available(cc) && ggml_is_quantized(V->type) && Q->ne[1] > 2 && Q->ne[1] <= 4) { - ggml_cuda_flash_attn_ext_mma_f16_case(ctx, dst); + ggml_cuda_flash_attn_ext_mma_f16_case(ctx, dst); return; }