From 2173d97dd41850e28d836aeae5378bff76bd951a Mon Sep 17 00:00:00 2001 From: Oliver Simons Date: Fri, 29 May 2026 14:09:01 +0200 Subject: [PATCH] Add tests for the added fusion ops --- tests/test-backend-ops.cpp | 60 ++++++++++++++++++++++++++++++++++---- 1 file changed, 54 insertions(+), 6 deletions(-) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index f561d09b5b..2de84bcae9 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -5788,19 +5788,21 @@ struct test_mul_mat_vec_fusion : public test_case { const bool b; // broadcast b matrix (only for use_id) const bool with_bias; const bool with_gate; + const bool with_scale; std::array batch_dims; test_mul_mat_vec_fusion(ggml_type type, ggml_glu_op op, int64_t m, int64_t n, int64_t k, bool use_id = false, int n_mats = 1, int n_used = 1, bool b = false, bool with_bias = false, bool with_gate = true, - std::array batch_dims = {4, 2}) - : type(type), glu_op(op), m(m), n(n), k(k), use_id(use_id), n_mats(n_mats), n_used(n_used), b(b), with_bias(with_bias), with_gate(with_gate), batch_dims(batch_dims) { + std::array batch_dims = {4, 2}, bool with_scale = false) + : type(type), glu_op(op), m(m), n(n), k(k), use_id(use_id), n_mats(n_mats), n_used(n_used), b(b), with_bias(with_bias), + with_gate(with_gate), with_scale(with_scale), batch_dims(batch_dims) { if (use_id) { GGML_ASSERT(n_used <= n_mats); } } std::string vars() override { - return VARS_TO_STR12(type, glu_op, m, n, k, use_id, n_mats, n_used, b, with_bias, with_gate, batch_dims); + return VARS_TO_STR13(type, glu_op, m, n, k, use_id, n_mats, n_used, b, with_bias, with_gate, with_scale, batch_dims); } std::string op_desc(ggml_tensor * t) override { @@ -5824,6 +5826,19 @@ struct test_mul_mat_vec_fusion : public test_case { return out; } + ggml_tensor * build_dense_scale(ggml_context * ctx, ggml_tensor * out) { + ggml_tensor * scale = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); + return ggml_mul(ctx, out, scale); + } + + ggml_tensor * build_id_scale(ggml_context * ctx, ggml_tensor * out, ggml_tensor * ids) { + ggml_tensor * scale = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_mats); + ggml_tensor * s = ggml_reshape_3d(ctx, scale, 1, n_mats, 1); + s = ggml_repeat_4d(ctx, s, 1, n_mats, m, 1); + s = ggml_get_rows(ctx, s, ids); + return ggml_mul(ctx, out, s); + } + ggml_tensor * build_graph(ggml_context * ctx) override { if (!use_id) { const int channels = batch_dims[0]; @@ -5841,6 +5856,9 @@ struct test_mul_mat_vec_fusion : public test_case { ggml_tensor * up_bias = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, bias_ne.data()); ffn_up = ggml_add(ctx, ffn_up, up_bias); } + if (with_scale) { + ffn_up = build_dense_scale(ctx, ffn_up); + } ggml_tensor * ffn_gate = with_gate ? ggml_mul_mat(ctx, gate, cur) : nullptr; if (with_bias && with_gate) { @@ -5848,6 +5866,9 @@ struct test_mul_mat_vec_fusion : public test_case { ggml_tensor * gate_bias = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, bias_ne.data()); ffn_gate = ggml_add(ctx, ffn_gate, gate_bias); } + if (with_scale && with_gate) { + ffn_gate = build_dense_scale(ctx, ffn_gate); + } ggml_tensor * out = with_gate ? build_gate(ctx, ffn_gate, ffn_up) : ffn_up; @@ -5874,18 +5895,26 @@ struct test_mul_mat_vec_fusion : public test_case { ggml_tensor * up_bias_param = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, ffn_up->ne[0], n_mats); ffn_up = ggml_add_id(ctx, ffn_up, up_bias_param, ids); } + if (with_scale) { + ffn_up = build_id_scale(ctx, ffn_up, ids); + } ggml_tensor * ffn_gate = with_gate? ggml_mul_mat_id(ctx, gates, cur, ids) : nullptr; if (with_bias && with_gate) { ggml_tensor * gate_bias_param = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, ffn_gate->ne[0], n_mats); ffn_gate = ggml_add_id(ctx, ffn_gate, gate_bias_param, ids); } + if (with_scale && with_gate) { + ffn_gate = build_id_scale(ctx, ffn_gate, ids); + } ggml_tensor * out = with_gate ? build_gate(ctx, ffn_gate, ffn_up) : ffn_up; - std::array scale_ne { 1, out->ne[1], out->ne[2], out->ne[3] }; - ggml_tensor * scale = ggml_new_tensor(ctx, out->type, 4, scale_ne.data()); - out = ggml_mul(ctx, out, scale); + if (!with_scale) { + std::array scale_ne { 1, out->ne[1], out->ne[2], out->ne[3] }; + ggml_tensor * scale = ggml_new_tensor(ctx, out->type, 4, scale_ne.data()); + out = ggml_mul(ctx, out, scale); + } ggml_set_name(out, "out"); return out; @@ -5905,6 +5934,13 @@ struct test_mul_mat_vec_fusion : public test_case { double max_nmse_err() override { return 5e-3; } + + double max_nmse_err(ggml_backend_t backend) override { + if (type == GGML_TYPE_NVFP4 && backend_has_feature(backend, "BLACKWELL_NATIVE_FP4")) { + return 2e-2; + } + return max_nmse_err(); + } }; // GGML_OP_SUM @@ -9094,6 +9130,18 @@ static std::vector> make_test_cases_eval() { } } + // NVFP4 scale2 fusion patterns: dense/MMID, GLU/two-lane, single-lane, with and without bias. + for (bool with_bias : {false, true}) { + test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 1, 32, 256, + false, 1, 1, false, with_bias, true, {1, 1}, true)); + test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 1, 32, 256, + true, 16, 8, false, with_bias, true, {1, 1}, true)); + test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 1, 32, 256, + false, 1, 1, false, with_bias, false, {1, 1}, true)); + test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 1, 32, 256, + true, 16, 8, false, with_bias, false, {1, 1}, true)); + } + for (auto gate : {GATING_FUNC_SOFTMAX, GATING_FUNC_SIGMOID, GATING_FUNC_SOFTMAX_WEIGHT}) { for (bool with_norm : {false, true}) { for (bool bias_probs : {false, true}) {