diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index df201922c86c..5aea9c96a828 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -1597,10 +1597,11 @@ struct ggml_cuda_gdn_gather_context { } }; -// GDN decode conv chain fused into one launch (GGML_CUDA_GDN_FUSION=1): the conv-state GET_ROWS is skipped and -// registered under the CONCAT it feeds; the CONCAT launches the fused kernel, which also computes the conv-state CPY, -// SSM_CONV + SILU and the q/k L2_NORM, and those nodes are marked done. Keyed by node identity, valid for one graph -// evaluation, owned by the evaluating context (reset with the other per-evaluation registries). +// GDN decode conv chain fused into one launch (on by default, GGML_CUDA_GDN_FUSION=0 disables it): the conv-state +// GET_ROWS is skipped and registered under the CONCAT it feeds; the CONCAT launches the fused kernel, which also +// computes the conv-state CPY, SSM_CONV + SILU and the q/k L2_NORM, and those nodes are marked done. Keyed by node +// identity, valid for one graph evaluation, owned by the evaluating context (reset with the other per-evaluation +// registries). struct ggml_cuda_gdn_conv_context { std::unordered_map gathers; // CONCAT -> the GET_ROWS it replaces std::unordered_set done; // nodes computed by a fused launch diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 5e641b238942..9ce02706cb11 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -3066,7 +3066,7 @@ static int ggml_cuda_try_fwht_q8(ggml_backend_cuda_context & ctx, const ggml_cgr return consumed; } -// GDN conv chain (GGML_CUDA_GDN_FUSION=1, off by default). For n_t tokens of one sequence (a decode step, n_t = 1, or a +// GDN conv chain (on by default, GGML_CUDA_GDN_FUSION=0 disables it). For n_t tokens of one sequence (a decode step, n_t = 1, or a // speculative verify step, n_t = 2..GGML_CUDA_GDN_CONV_MAX_TOKENS), a hybrid-model gated delta net layer runs these // launch-bound kernels between the qkv projection and the GDN op: // GET_ROWS(conv cache, s_copy) -> RESHAPE -> CONCAT(state, qkv^T) @@ -3080,8 +3080,10 @@ static int ggml_cuda_try_fwht_q8(ggml_backend_cuda_context & ctx, const ggml_cgr // of it) must not touch any memory the fused kernel reads or writes. Megakernel-style launch removal in the spirit of // Hazy Research's "no bubbles" low-latency decode work, applied as a graph-level fusion rather than a persistent kernel. static bool ggml_cuda_gdn_fusion_enabled() { - static const bool enabled = getenv("GGML_CUDA_GDN_FUSION") != nullptr && - atoi(getenv("GGML_CUDA_GDN_FUSION")) != 0; + static const bool enabled = [] { + const char * env = getenv("GGML_CUDA_GDN_FUSION"); + return env == nullptr || atoi(env) != 0; + }(); return enabled; } @@ -4218,7 +4220,8 @@ static int ggml_cuda_try_batched_cpy(ggml_backend_cuda_context * cuda_ctx, ggml_ } // Two consecutive independent F32/F16/BF16 mat-vecs on the same activation (the GDN alpha and beta projections, which -// the fused GDN op consumes raw) as one launch, GGML_CUDA_GDN_FUSION=1. Returns the number of nodes consumed. +// the fused GDN op consumes raw) as one launch (on by default, GGML_CUDA_GDN_FUSION=0 disables it). Returns the +// number of nodes consumed. static int ggml_cuda_try_mmvf_pair(ggml_backend_cuda_context & ctx, ggml_cgraph * cgraph, int i) { ggml_tensor * mm0 = cgraph->nodes[i]; int j = i + 1; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 385689dad32f..c856f6229693 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -4140,7 +4140,7 @@ struct test_ssm_conv_bias_silu : public test_case { // GET_ROWS (conv state row) -> CONCAT with the new tokens -> CPY of the last window back into a cache row (n_rs == 0), // or one CPY per rollback snapshot slot (n_rs > 0: 1 + n_rs slots, as llm_build_delta_net_base::build_conv_state writes // them), SSM_CONV -> SILU -> L2_NORM of the leading q/k heads of every token. The CUDA backend can run it as one -// launch (GGML_CUDA_GDN_FUSION=1); the L2_NORM, the SILU output and the written cache rows are all checked. +// launch (by default; GGML_CUDA_GDN_FUSION=0 disables it); the L2_NORM, the SILU output and the written cache rows are all checked. struct test_gdn_conv_chain : public test_case { const int64_t d_conv; const int64_t head_dim; // L2 row length @@ -4230,7 +4230,7 @@ struct test_gdn_conv_chain : public test_case { }; // Two mat-vecs of one activation with same-shape float weights (the GDN alpha and beta projections); the CUDA backend -// can run them as one launch (GGML_CUDA_GDN_FUSION=1). Both products are checked. +// can run them as one launch (by default; GGML_CUDA_GDN_FUSION=0 disables it). Both products are checked. struct test_mul_mat_vec_pair : public test_case { const ggml_type type; const int64_t k;