From 73ee84fffee5429724c9c3e1d2fec540578471d1 Mon Sep 17 00:00:00 2001 From: "Yu, Zijun" Date: Thu, 3 Jul 2025 11:03:40 +0800 Subject: [PATCH] Add SwiGLU --- ggml/src/ggml-openvino/.clang-format | 2 +- ggml/src/ggml-openvino/ggml-decoder.cpp | 87 +++++++++++-------- ggml/src/ggml-openvino/ggml-openvino.cpp | 36 +++++--- .../ggml-openvino/openvino/op/glu_swiglu.cpp | 29 +++++++ ggml/src/ggml-openvino/openvino/op_table.cpp | 37 ++++---- ggml/src/ggml-openvino/openvino/op_table.hpp | 2 +- 6 files changed, 123 insertions(+), 70 deletions(-) create mode 100644 ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp diff --git a/ggml/src/ggml-openvino/.clang-format b/ggml/src/ggml-openvino/.clang-format index 9382a117b8..6d77ecea3c 100644 --- a/ggml/src/ggml-openvino/.clang-format +++ b/ggml/src/ggml-openvino/.clang-format @@ -4,6 +4,7 @@ AlignConsecutiveAssignments: false AlignConsecutiveDeclarations: false ReferenceAlignment: Left PointerAlignment: Left +Cpp11BracedListStyle: true Language: Cpp AlignAfterOpenBracket: Align @@ -65,7 +66,6 @@ CommentPragmas: '^ IWYU pragma:' CompactNamespaces: false ConstructorInitializerIndentWidth: 4 ContinuationIndentWidth: 4 -Cpp11BracedListStyle: false DerivePointerAlignment: false DisableFormat: false EmptyLineBeforeAccessModifier: Leave diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index e30f026e36..61c0fe4833 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -563,43 +563,58 @@ void GgmlOvDecoder::visit_subgraph(std::function opTypeMap = { - {GGML_OP_ACC, "GGML_OP_ACC"}, {GGML_OP_ADD, "GGML_OP_ADD"}, - {GGML_OP_ADD1, "GGML_OP_ADD1"}, {GGML_OP_CONT, "GGML_OP_CONT"}, - {GGML_OP_CPY, "GGML_OP_CPY"}, {GGML_OP_DIV, "GGML_OP_DIV"}, - {GGML_OP_DUP, "GGML_OP_DUP"}, {GGML_OP_GET_ROWS, "GGML_OP_GET_ROWS"}, - {GGML_OP_MUL, "GGML_OP_MUL"}, {GGML_OP_MUL_MAT, "GGML_OP_MUL_MAT"}, - {GGML_OP_PERMUTE, "GGML_OP_PERMUTE"}, {GGML_OP_RESHAPE, "GGML_OP_RESHAPE"}, - {GGML_OP_RMS_NORM, "GGML_OP_RMS_NORM"}, {GGML_OP_ROPE, "GGML_OP_ROPE"}, - {GGML_OP_SCALE, "GGML_OP_SCALE"}, {GGML_OP_SOFT_MAX, "GGML_OP_SOFT_MAX"}, - {GGML_OP_SUB, "GGML_OP_SUB"}, {GGML_OP_TRANSPOSE, "GGML_OP_TRANSPOSE"}, - {GGML_OP_UNARY, "GGML_OP_UNARY"}, {GGML_OP_VIEW, "GGML_OP_VIEW"}}; - static const std::map unaryOpTypeMap = { - {GGML_UNARY_OP_ABS, "GGML_UNARY_OP_ABS"}, - {GGML_UNARY_OP_SGN, "GGML_UNARY_OP_SGN"}, - {GGML_UNARY_OP_NEG, "GGML_UNARY_OP_NEG"}, - {GGML_UNARY_OP_STEP, "GGML_UNARY_OP_STEP"}, - {GGML_UNARY_OP_TANH, "GGML_UNARY_OP_TANH"}, - {GGML_UNARY_OP_ELU, "GGML_UNARY_OP_ELU"}, - {GGML_UNARY_OP_RELU, "GGML_UNARY_OP_RELU"}, - {GGML_UNARY_OP_SIGMOID, "GGML_UNARY_OP_SIGMOID"}, - {GGML_UNARY_OP_GELU, "GGML_UNARY_OP_GELU"}, - {GGML_UNARY_OP_GELU_QUICK, "GGML_UNARY_OP_GELU_QUICK"}, - {GGML_UNARY_OP_SILU, "GGML_UNARY_OP_SILU"}, - {GGML_UNARY_OP_HARDSWISH, "GGML_UNARY_OP_HARDSWISH"}, + static const std::map ops = { + {GGML_OP_ACC, "GGML_OP_ACC" }, + {GGML_OP_ADD, "GGML_OP_ADD" }, + {GGML_OP_ADD1, "GGML_OP_ADD1" }, + {GGML_OP_CONT, "GGML_OP_CONT" }, + {GGML_OP_CPY, "GGML_OP_CPY" }, + {GGML_OP_DIV, "GGML_OP_DIV" }, + {GGML_OP_DUP, "GGML_OP_DUP" }, + {GGML_OP_GET_ROWS, "GGML_OP_GET_ROWS" }, + {GGML_OP_MUL, "GGML_OP_MUL" }, + {GGML_OP_MUL_MAT, "GGML_OP_MUL_MAT" }, + {GGML_OP_PERMUTE, "GGML_OP_PERMUTE" }, + {GGML_OP_RESHAPE, "GGML_OP_RESHAPE" }, + {GGML_OP_RMS_NORM, "GGML_OP_RMS_NORM" }, + {GGML_OP_ROPE, "GGML_OP_ROPE" }, + {GGML_OP_SCALE, "GGML_OP_SCALE" }, + {GGML_OP_SOFT_MAX, "GGML_OP_SOFT_MAX" }, + {GGML_OP_SUB, "GGML_OP_SUB" }, + {GGML_OP_TRANSPOSE, "GGML_OP_TRANSPOSE"}, + {GGML_OP_VIEW, "GGML_OP_VIEW" } + }; + static const std::map unary_ops = { + {GGML_UNARY_OP_ABS, "GGML_UNARY_OP_ABS" }, + {GGML_UNARY_OP_SGN, "GGML_UNARY_OP_SGN" }, + {GGML_UNARY_OP_NEG, "GGML_UNARY_OP_NEG" }, + {GGML_UNARY_OP_STEP, "GGML_UNARY_OP_STEP" }, + {GGML_UNARY_OP_TANH, "GGML_UNARY_OP_TANH" }, + {GGML_UNARY_OP_ELU, "GGML_UNARY_OP_ELU" }, + {GGML_UNARY_OP_RELU, "GGML_UNARY_OP_RELU" }, + {GGML_UNARY_OP_SIGMOID, "GGML_UNARY_OP_SIGMOID" }, + {GGML_UNARY_OP_GELU, "GGML_UNARY_OP_GELU" }, + {GGML_UNARY_OP_GELU_QUICK, "GGML_UNARY_OP_GELU_QUICK" }, + {GGML_UNARY_OP_SILU, "GGML_UNARY_OP_SILU" }, + {GGML_UNARY_OP_HARDSWISH, "GGML_UNARY_OP_HARDSWISH" }, {GGML_UNARY_OP_HARDSIGMOID, "GGML_UNARY_OP_HARDSIGMOID"}, - {GGML_UNARY_OP_EXP, "GGML_UNARY_OP_EXP"}, - {GGML_UNARY_OP_COUNT, "GGML_UNARY_OP_COUNT"}}; - auto it = opTypeMap.find(m_node->op); - if (it != opTypeMap.end()) { - if (it->first == GGML_OP_UNARY) { - auto unary_it = unaryOpTypeMap.find(ggml_get_unary_op(m_node)); - if (unary_it != unaryOpTypeMap.end()) { - return unary_it->second; - } - } - return it->second; + {GGML_UNARY_OP_EXP, "GGML_UNARY_OP_EXP" }, + {GGML_UNARY_OP_COUNT, "GGML_UNARY_OP_COUNT" } + }; + static const std::map glu_ops = { + {GGML_GLU_OP_SWIGLU, "GGML_GLU_OP_SWIGLU"}, + {GGML_GLU_OP_GEGLU, "GGML_GLU_OP_GEGLU" }, + {GGML_GLU_OP_REGLU, "GGML_GLU_OP_REGLU" } + }; + + switch (m_node->op) { + case GGML_OP_UNARY: + return unary_ops.at(ggml_get_unary_op(m_node)); + case GGML_OP_GLU: + return glu_ops.at(ggml_get_glu_op(m_node)); + default: + return ops.at(m_node->op); } - static const std::string unknown_op = "UNKNOWN_OP"; + static const std::string unknown_op = "UNKNOWN_GGML_OP"; return unknown_op; } diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index 19e4ed5b77..167453b215 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -237,21 +237,29 @@ static ggml_backend_buffer_t ggml_backend_openvino_device_buffer_from_host_ptr(g static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) { GGML_ASSERT(dev->reg != nullptr); - static const std::set supported_ops{ - GGML_OP_ADD, GGML_OP_MUL, GGML_OP_MUL_MAT, GGML_OP_VIEW, - GGML_OP_CONT, GGML_OP_CPY, GGML_OP_RESHAPE, GGML_OP_PERMUTE, - GGML_OP_TRANSPOSE, GGML_OP_GET_ROWS, GGML_OP_ROPE, GGML_OP_RMS_NORM, - GGML_OP_SCALE, GGML_OP_SOFT_MAX, - }; - static const std::set supported_unary_ops{ - GGML_UNARY_OP_SILU, - }; + static const std::set supported_ops{GGML_OP_NONE, GGML_OP_ADD, GGML_OP_MUL, GGML_OP_MUL_MAT, + GGML_OP_VIEW, GGML_OP_CONT, GGML_OP_CPY, GGML_OP_RESHAPE, + GGML_OP_PERMUTE, GGML_OP_TRANSPOSE, GGML_OP_GET_ROWS, GGML_OP_ROPE, + GGML_OP_RMS_NORM, GGML_OP_SCALE, GGML_OP_SOFT_MAX}; + static const std::set supported_unary_ops{ + GGML_UNARY_OP_SILU, + }; + static const std::set supported_glu_ops{ + GGML_GLU_OP_SWIGLU, + }; - if (op->op == GGML_OP_UNARY) { - return supported_unary_ops.find(ggml_get_unary_op(op)) != - supported_unary_ops.end(); - } - return supported_ops.find(op->op) != supported_ops.end(); + auto res = false; + switch (op->op) { + case GGML_OP_UNARY: + res = supported_unary_ops.find(ggml_get_unary_op(op)) != supported_unary_ops.end(); + break; + case GGML_OP_GLU: + res = supported_glu_ops.find(ggml_get_glu_op(op)) != supported_glu_ops.end(); + break; + default: + res = supported_ops.find(op->op) != supported_ops.end(); + } + return res; } static bool ggml_backend_openvino_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { diff --git a/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp b/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp new file mode 100644 index 0000000000..28013fbaa0 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp @@ -0,0 +1,29 @@ +#include +#include +#include + +#include "../node_context.hpp" +#include "../op_table.hpp" +#include "../utils.hpp" + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_glu_swiglu(const NodeContext& context) { + num_inputs_check(context, 2, 2); + + auto src1 = context.get_input(0); + auto src2 = context.get_input(1); + auto sigmoid = std::make_shared(src1); + auto silu = std::make_shared(src1, sigmoid); + auto res = std::make_shared(silu, src2); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op_table.cpp b/ggml/src/ggml-openvino/openvino/op_table.cpp index bf7d54d9a1..a99450ea95 100644 --- a/ggml/src/ggml-openvino/openvino/op_table.cpp +++ b/ggml/src/ggml-openvino/openvino/op_table.cpp @@ -16,24 +16,25 @@ namespace ggml { std::unordered_map get_supported_ops() { using namespace ov::op; return { - { "GGML_OP_ADD", op::translate_1to1_match_2_inputs }, - { "GGML_OP_ADD1", op::translate_1to1_match_2_inputs }, - { "GGML_OP_CONT", op::translate_cont }, - { "GGML_OP_CPY", op::translate_cpy }, - { "GGML_OP_DIV", op::translate_1to1_match_2_inputs }, - { "GGML_OP_GET_ROWS", op::translate_get_rows }, - { "GGML_OP_MUL", op::translate_1to1_match_2_inputs }, - { "GGML_OP_MUL_MAT", op::translate_mulmat }, - { "GGML_OP_PERMUTE", op::translate_permute }, - { "GGML_OP_RESHAPE", op::translate_reshape }, - { "GGML_OP_RMS_NORM", op::translate_rms_norm }, - { "GGML_OP_ROPE", op::translate_rope }, - { "GGML_OP_SCALE", op::translate_scale }, - { "GGML_OP_SOFT_MAX", op::translate_soft_max }, - { "GGML_OP_SUB", op::translate_1to1_match_2_inputs }, - { "GGML_OP_TRANSPOSE", op::translate_transpose }, - { "GGML_UNARY_OP_SILU", op::translate_unary_silu }, - { "GGML_OP_VIEW", op::translate_view } + {"GGML_OP_ADD", op::translate_1to1_match_2_inputs }, + {"GGML_OP_ADD1", op::translate_1to1_match_2_inputs }, + {"GGML_OP_CONT", op::translate_cont }, + {"GGML_OP_CPY", op::translate_cpy }, + {"GGML_OP_DIV", op::translate_1to1_match_2_inputs }, + {"GGML_OP_GET_ROWS", op::translate_get_rows }, + {"GGML_OP_MUL", op::translate_1to1_match_2_inputs}, + {"GGML_OP_MUL_MAT", op::translate_mulmat }, + {"GGML_OP_PERMUTE", op::translate_permute }, + {"GGML_OP_RESHAPE", op::translate_reshape }, + {"GGML_OP_RMS_NORM", op::translate_rms_norm }, + {"GGML_OP_ROPE", op::translate_rope }, + {"GGML_OP_SCALE", op::translate_scale }, + {"GGML_OP_SOFT_MAX", op::translate_soft_max }, + {"GGML_OP_SUB", op::translate_1to1_match_2_inputs}, + {"GGML_OP_TRANSPOSE", op::translate_transpose }, + {"GGML_UNARY_OP_SILU", op::translate_unary_silu }, + {"GGML_OP_VIEW", op::translate_view }, + {"GGML_GLU_OP_SWIGLU", op::translate_glu_swiglu }, }; } diff --git a/ggml/src/ggml-openvino/openvino/op_table.hpp b/ggml/src/ggml-openvino/openvino/op_table.hpp index d576c2a135..9b141d6d20 100644 --- a/ggml/src/ggml-openvino/openvino/op_table.hpp +++ b/ggml/src/ggml-openvino/openvino/op_table.hpp @@ -24,8 +24,8 @@ GGML_OP_CONVERTER(translate_scale); GGML_OP_CONVERTER(translate_unary_silu); GGML_OP_CONVERTER(translate_soft_max); GGML_OP_CONVERTER(translate_transpose); -GGML_OP_CONVERTER(translate_unary); GGML_OP_CONVERTER(translate_view); +GGML_OP_CONVERTER(translate_glu_swiglu); } // namespace op