From bebe162cf7c2f9099c879956740b52237381463d Mon Sep 17 00:00:00 2001 From: bri-prism <288398250+bri-prism@users.noreply.github.com> Date: Sun, 25 Oct 2026 09:47:40 +0800 Subject: [PATCH] cuda: turn the GDN decode conv-chain fusion and alpha/beta mat-vec pair on by default (GGML_CUDA_GDN_FUSION=0 disables) The exact fusion from #383 (one launch for the conv-state gather, concat, state write-back, ssm_conv + silu and q/k L2 norm of a one-token GDN step, and one paired launch for the ssm_alpha/ssm_beta mat-vecs) is now on by default for the one-token path it covers. GGML_CUDA_GDN_FUSION=0 restores the unfused kernels. NVIDIA only, as before. --- ggml/src/ggml-cuda/common.cuh | 9 +++++---- ggml/src/ggml-cuda/ggml-cuda.cu | 13 ++++++++----- tests/test-backend-ops.cpp | 5 +++-- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index df201922c86c..5aea9c96a828 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -1597,10 +1597,11 @@ struct ggml_cuda_gdn_gather_context { } }; -// GDN decode conv chain fused into one launch (GGML_CUDA_GDN_FUSION=1): the conv-state GET_ROWS is skipped and -// registered under the CONCAT it feeds; the CONCAT launches the fused kernel, which also computes the conv-state CPY, -// SSM_CONV + SILU and the q/k L2_NORM, and those nodes are marked done. Keyed by node identity, valid for one graph -// evaluation, owned by the evaluating context (reset with the other per-evaluation registries). +// GDN decode conv chain fused into one launch (on by default, GGML_CUDA_GDN_FUSION=0 disables it): the conv-state +// GET_ROWS is skipped and registered under the CONCAT it feeds; the CONCAT launches the fused kernel, which also +// computes the conv-state CPY, SSM_CONV + SILU and the q/k L2_NORM, and those nodes are marked done. Keyed by node +// identity, valid for one graph evaluation, owned by the evaluating context (reset with the other per-evaluation +// registries). struct ggml_cuda_gdn_conv_context { std::unordered_map gathers; // CONCAT -> the GET_ROWS it replaces std::unordered_set done; // nodes computed by a fused launch diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 534f6b4b8947..4c0128993a3a 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -3066,8 +3066,8 @@ static int ggml_cuda_try_fwht_q8(ggml_backend_cuda_context & ctx, const ggml_cgr return consumed; } -// GDN decode conv chain (GGML_CUDA_GDN_FUSION=1, off by default). For one token of one sequence, a hybrid-model -// gated delta net layer runs five launch-bound kernels between the qkv projection and the GDN op: +// GDN decode conv chain (on by default, GGML_CUDA_GDN_FUSION=0 disables it). For one token of one sequence, a +// hybrid-model gated delta net layer runs five launch-bound kernels between the qkv projection and the GDN op: // GET_ROWS(conv cache, s_copy) -> RESHAPE -> CONCAT(state, qkv^T) -> CPY(last window -> conv cache) // -> SSM_CONV + SILU -> L2_NORM(q/k heads of the SILU output) // ggml_cuda_ssm_conv_gdn_fused computes all of them in one launch with the same arithmetic per element (bit-identical @@ -3078,8 +3078,10 @@ static int ggml_cuda_try_fwht_q8(ggml_backend_cuda_context & ctx, const ggml_cgr // of it) must not touch any memory the fused kernel reads or writes. Megakernel-style launch removal in the spirit of // Hazy Research's "no bubbles" low-latency decode work, applied as a graph-level fusion rather than a persistent kernel. static bool ggml_cuda_gdn_fusion_enabled() { - static const bool enabled = getenv("GGML_CUDA_GDN_FUSION") != nullptr && - atoi(getenv("GGML_CUDA_GDN_FUSION")) != 0; + static const bool enabled = [] { + const char * env = getenv("GGML_CUDA_GDN_FUSION"); + return env == nullptr || atoi(env) != 0; + }(); return enabled; } @@ -4171,7 +4173,8 @@ static int ggml_cuda_try_batched_cpy(ggml_backend_cuda_context * cuda_ctx, ggml_ } // Two consecutive independent F32/F16/BF16 mat-vecs on the same activation (the GDN alpha and beta projections, which -// the fused GDN op consumes raw) as one launch, GGML_CUDA_GDN_FUSION=1. Returns the number of nodes consumed. +// the fused GDN op consumes raw) as one launch (on by default, GGML_CUDA_GDN_FUSION=0 disables it). Returns the +// number of nodes consumed. static int ggml_cuda_try_mmvf_pair(ggml_backend_cuda_context & ctx, ggml_cgraph * cgraph, int i) { ggml_tensor * mm0 = cgraph->nodes[i]; int j = i + 1; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 048c12f941d7..33ccfea22361 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -4139,7 +4139,8 @@ struct test_ssm_conv_bias_silu : public test_case { // The decode conv chain of a gated delta net layer (one token, one sequence), as the hybrid models build it: // GET_ROWS (conv state row) -> CONCAT with the new token -> CPY of the last window back into a cache row, // SSM_CONV -> SILU -> L2_NORM of the leading q/k heads. The CUDA backend can run it as one launch -// (GGML_CUDA_GDN_FUSION=1); the L2_NORM, the SILU output and the written cache row are all checked. +// (by default; GGML_CUDA_GDN_FUSION=0 disables it); the L2_NORM, the SILU output and the written cache row are all +// checked. struct test_gdn_conv_chain : public test_case { const int64_t d_conv; const int64_t head_dim; // L2 row length @@ -4217,7 +4218,7 @@ struct test_gdn_conv_chain : public test_case { }; // Two mat-vecs of one activation with same-shape float weights (the GDN alpha and beta projections); the CUDA backend -// can run them as one launch (GGML_CUDA_GDN_FUSION=1). Both products are checked. +// can run them as one launch (by default; GGML_CUDA_GDN_FUSION=0 disables it). Both products are checked. struct test_mul_mat_vec_pair : public test_case { const ggml_type type; const int64_t k;