From d6190b97a560e427c19c6d72bbc329383aa2cf76 Mon Sep 17 00:00:00 2001 From: Oliver Simons Date: Tue, 2 Jun 2026 15:08:52 +0200 Subject: [PATCH] Cleanup ggml-cuda/mmvq 1. Unrestrict post-scale fusion 2. Rename names accordingly 3. Remove env variable to disable fusion --- ggml/src/ggml-cuda/ggml-cuda.cu | 64 +++++++++++++++------------------ ggml/src/ggml-cuda/mmvq.cu | 2 -- 2 files changed, 28 insertions(+), 38 deletions(-) diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 187c689458..12f07dceef 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -3885,13 +3885,13 @@ struct ggml_cuda_mmid_lane { int n_nodes = 0; }; -static bool ggml_cuda_parse_nvfp4_mmid_lane(const ggml_cgraph * cgraph, int i, ggml_cuda_mmid_lane & lane) { +static bool ggml_cuda_parse_mmid_scale_lane(const ggml_cgraph * cgraph, int i, ggml_cuda_mmid_lane & lane) { if (i >= cgraph->n_nodes || cgraph->nodes[i]->op != GGML_OP_MUL_MAT_ID) { return false; } ggml_tensor * mm = cgraph->nodes[i]; - if (mm->src[0]->type != GGML_TYPE_NVFP4 || mm->src[1]->type != GGML_TYPE_F32 || mm->type != GGML_TYPE_F32 || mm->src[2] == nullptr) { + if (!ggml_is_quantized(mm->src[0]->type) || mm->src[1]->type != GGML_TYPE_F32 || mm->type != GGML_TYPE_F32 || mm->src[2] == nullptr) { return false; } @@ -3950,13 +3950,13 @@ static bool ggml_cuda_parse_nvfp4_mmid_lane(const ggml_cgraph * cgraph, int i, g return true; } -static bool ggml_cuda_parse_nvfp4_mm_lane(const ggml_cgraph * cgraph, int i, ggml_cuda_mmid_lane & lane) { +static bool ggml_cuda_parse_mm_scale_lane(const ggml_cgraph * cgraph, int i, ggml_cuda_mmid_lane & lane) { if (i >= cgraph->n_nodes || cgraph->nodes[i]->op != GGML_OP_MUL_MAT) { return false; } ggml_tensor * mm = cgraph->nodes[i]; - if (mm->src[0]->type != GGML_TYPE_NVFP4 || mm->src[1]->type != GGML_TYPE_F32 || mm->type != GGML_TYPE_F32) { + if (!ggml_is_quantized(mm->src[0]->type) || mm->src[1]->type != GGML_TYPE_F32 || mm->type != GGML_TYPE_F32) { return false; } @@ -4102,14 +4102,14 @@ static bool ggml_cuda_should_fuse_mm_lanes(const ggml_cuda_mmid_lane & up, const return !split; } -static int ggml_cuda_try_fuse_nvfp4_mmid_scale_glu(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, int i) { +static int ggml_cuda_try_fuse_mmid_scale_glu(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, int i) { ggml_cuda_mmid_lane lane0; - if (!ggml_cuda_parse_nvfp4_mmid_lane(cgraph, i, lane0)) { + if (!ggml_cuda_parse_mmid_scale_lane(cgraph, i, lane0)) { return 0; } ggml_cuda_mmid_lane lane1; - if (!ggml_cuda_parse_nvfp4_mmid_lane(cgraph, i + lane0.n_nodes, lane1)) { + if (!ggml_cuda_parse_mmid_scale_lane(cgraph, i + lane0.n_nodes, lane1)) { return 0; } @@ -4166,14 +4166,14 @@ static int ggml_cuda_try_fuse_nvfp4_mmid_scale_glu(ggml_backend_cuda_context * c return glu_idx - i; } -static int ggml_cuda_try_fuse_nvfp4_mm_scale_glu(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, int i) { +static int ggml_cuda_try_fuse_mm_scale_glu(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, int i) { ggml_cuda_mmid_lane lane0; - if (!ggml_cuda_parse_nvfp4_mm_lane(cgraph, i, lane0)) { + if (!ggml_cuda_parse_mm_scale_lane(cgraph, i, lane0)) { return 0; } ggml_cuda_mmid_lane lane1; - if (!ggml_cuda_parse_nvfp4_mm_lane(cgraph, i + lane0.n_nodes, lane1)) { + if (!ggml_cuda_parse_mm_scale_lane(cgraph, i + lane0.n_nodes, lane1)) { return 0; } @@ -4226,9 +4226,9 @@ static int ggml_cuda_try_fuse_nvfp4_mm_scale_glu(ggml_backend_cuda_context * cud return glu_idx - i; } -static int ggml_cuda_try_fuse_nvfp4_mmid_scale(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, int i) { +static int ggml_cuda_try_fuse_mmid_scale(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, int i) { ggml_cuda_mmid_lane lane; - if (!ggml_cuda_parse_nvfp4_mmid_lane(cgraph, i, lane) || lane.scale == nullptr) { + if (!ggml_cuda_parse_mmid_scale_lane(cgraph, i, lane) || lane.scale == nullptr) { return 0; } @@ -4256,9 +4256,9 @@ static int ggml_cuda_try_fuse_nvfp4_mmid_scale(ggml_backend_cuda_context * cuda_ return lane.n_nodes - 1; } -static int ggml_cuda_try_fuse_nvfp4_mm_scale(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, int i) { +static int ggml_cuda_try_fuse_mm_scale(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, int i) { ggml_cuda_mmid_lane lane; - if (!ggml_cuda_parse_nvfp4_mm_lane(cgraph, i, lane) || lane.scale == nullptr) { + if (!ggml_cuda_parse_mm_scale_lane(cgraph, i, lane) || lane.scale == nullptr) { return 0; } @@ -4285,9 +4285,6 @@ static int ggml_cuda_try_fuse_nvfp4_mm_scale(ggml_backend_cuda_context * cuda_ct static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, int i) { static bool disable_fusion = getenv("GGML_CUDA_DISABLE_FUSION") != nullptr && std::atoi(getenv("GGML_CUDA_DISABLE_FUSION")); - const char * disable_nvfp4_mmid_scale_fusion_env = getenv("GGML_CUDA_DISABLE_NVFP4_MMID_SCALE_FUSION"); - const bool disable_nvfp4_mmid_scale_fusion = - disable_nvfp4_mmid_scale_fusion_env != nullptr && std::atoi(disable_nvfp4_mmid_scale_fusion_env); if (disable_fusion) { return 0; } @@ -4453,16 +4450,13 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph } } - int fused_nvfp4_mmid_nodes = 0; - if (!disable_nvfp4_mmid_scale_fusion) { - fused_nvfp4_mmid_nodes = ggml_cuda_try_fuse_nvfp4_mmid_scale_glu(cuda_ctx, cgraph, i); - if (fused_nvfp4_mmid_nodes > 0) { - return fused_nvfp4_mmid_nodes; - } - fused_nvfp4_mmid_nodes = ggml_cuda_try_fuse_nvfp4_mm_scale_glu(cuda_ctx, cgraph, i); - if (fused_nvfp4_mmid_nodes > 0) { - return fused_nvfp4_mmid_nodes; - } + int fused_scale_nodes = ggml_cuda_try_fuse_mmid_scale_glu(cuda_ctx, cgraph, i); + if (fused_scale_nodes > 0) { + return fused_scale_nodes; + } + fused_scale_nodes = ggml_cuda_try_fuse_mm_scale_glu(cuda_ctx, cgraph, i); + if (fused_scale_nodes > 0) { + return fused_scale_nodes; } bool fused_mul_mat_vec = false; @@ -4595,15 +4589,13 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph fused_mul_mat_vec = false; fused_node_count = 0; - if (!disable_nvfp4_mmid_scale_fusion) { - fused_nvfp4_mmid_nodes = ggml_cuda_try_fuse_nvfp4_mmid_scale(cuda_ctx, cgraph, i); - if (fused_nvfp4_mmid_nodes > 0) { - return fused_nvfp4_mmid_nodes; - } - fused_nvfp4_mmid_nodes = ggml_cuda_try_fuse_nvfp4_mm_scale(cuda_ctx, cgraph, i); - if (fused_nvfp4_mmid_nodes > 0) { - return fused_nvfp4_mmid_nodes; - } + fused_scale_nodes = ggml_cuda_try_fuse_mmid_scale(cuda_ctx, cgraph, i); + if (fused_scale_nodes > 0) { + return fused_scale_nodes; + } + fused_scale_nodes = ggml_cuda_try_fuse_mm_scale(cuda_ctx, cgraph, i); + if (fused_scale_nodes > 0) { + return fused_scale_nodes; } // gate + add + glu + up + add diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index 6aad0b3059..5a4b2fdf42 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -1194,14 +1194,12 @@ void ggml_cuda_mul_mat_vec_q( fusion_local.gate_bias = fusion->gate_bias->data; } if (fusion->x_scale) { - GGML_ASSERT(src0->type == GGML_TYPE_NVFP4); GGML_ASSERT(fusion->x_scale->type == GGML_TYPE_F32); GGML_ASSERT(ggml_is_contiguous(fusion->x_scale)); GGML_ASSERT(ggml_nelements(fusion->x_scale) == (ids ? src0->ne[2] : 1)); fusion_local.x_scale = fusion->x_scale->data; } if (fusion->gate_scale) { - GGML_ASSERT(src0->type == GGML_TYPE_NVFP4); GGML_ASSERT(fusion->gate_scale->type == GGML_TYPE_F32); GGML_ASSERT(ggml_is_contiguous(fusion->gate_scale)); GGML_ASSERT(ggml_nelements(fusion->gate_scale) == (ids ? src0->ne[2] : 1));