diff --git a/ggml/src/ggml-openvino/openvino/op/add.cpp b/ggml/src/ggml-openvino/openvino/op/add.cpp index c218cf34de..18bc463fb9 100644 --- a/ggml/src/ggml-openvino/openvino/op/add.cpp +++ b/ggml/src/ggml-openvino/openvino/op/add.cpp @@ -1,4 +1,4 @@ -#include "openvino/op/add.hpp" +#include #include "../node_context.hpp" #include "../utils.hpp" @@ -15,7 +15,7 @@ OutputVector translate_add(const NodeContext& context) { auto rhs = context.get_input(1); auto add = std::make_shared(lhs, rhs); return {add}; -}; +} } // namespace op } // namespace ggml diff --git a/ggml/src/ggml-openvino/openvino/op/cont.cpp b/ggml/src/ggml-openvino/openvino/op/cont.cpp index 2ebc890fda..e8e9bf0a4e 100644 --- a/ggml/src/ggml-openvino/openvino/op/cont.cpp +++ b/ggml/src/ggml-openvino/openvino/op/cont.cpp @@ -2,12 +2,12 @@ #include #include #include +#include +#include #include #include "../node_context.hpp" #include "../utils.hpp" -#include "openvino/op/reshape.hpp" -#include "openvino/op/slice.hpp" namespace ov { namespace frontend { @@ -48,7 +48,7 @@ OutputVector translate_cont(const NodeContext& context) { return {slice}; } -}; +} } // namespace op } // namespace ggml diff --git a/ggml/src/ggml-openvino/openvino/op/cpy.cpp b/ggml/src/ggml-openvino/openvino/op/cpy.cpp index b4f4d59408..2808d3ee91 100644 --- a/ggml/src/ggml-openvino/openvino/op/cpy.cpp +++ b/ggml/src/ggml-openvino/openvino/op/cpy.cpp @@ -1,19 +1,19 @@ #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include "../node_context.hpp" #include "../utils.hpp" -#include "openvino/core/node.hpp" -#include "openvino/core/node_output.hpp" -#include "openvino/op/add.hpp" -#include "openvino/op/constant.hpp" -#include "openvino/op/convert_like.hpp" -#include "openvino/op/range.hpp" -#include "openvino/op/reshape.hpp" -#include "openvino/op/scatter_nd_update.hpp" -#include "openvino/op/transpose.hpp" -#include "openvino/op/unsqueeze.hpp" namespace ov { namespace frontend { @@ -98,7 +98,7 @@ OutputVector translate_cpy(const NodeContext& context) { false); return {reshaped_res}; } -}; +} } // namespace op } // namespace ggml diff --git a/ggml/src/ggml-openvino/openvino/op/get_rows.cpp b/ggml/src/ggml-openvino/openvino/op/get_rows.cpp index edb25d9124..64fc57bd88 100644 --- a/ggml/src/ggml-openvino/openvino/op/get_rows.cpp +++ b/ggml/src/ggml-openvino/openvino/op/get_rows.cpp @@ -1,14 +1,13 @@ -#include +#include +#include +#include +#include +#include +#include #include #include "../node_context.hpp" #include "../utils.hpp" -#include "openvino/core/node.hpp" -#include "openvino/core/node_output.hpp" -#include "openvino/op/constant.hpp" -#include "openvino/op/convert.hpp" -#include "openvino/op/gather.hpp" -#include "openvino/op/reshape.hpp" namespace ov { namespace frontend { @@ -32,7 +31,7 @@ OutputVector translate_get_rows(const NodeContext& context) { } return {res}; -}; +} } // namespace op } // namespace ggml diff --git a/ggml/src/ggml-openvino/openvino/op/mul.cpp b/ggml/src/ggml-openvino/openvino/op/mul.cpp index 1b1c69f7df..14473f4e27 100644 --- a/ggml/src/ggml-openvino/openvino/op/mul.cpp +++ b/ggml/src/ggml-openvino/openvino/op/mul.cpp @@ -1,14 +1,7 @@ -#include -#include +#include #include "../node_context.hpp" #include "../utils.hpp" -#include "openvino/core/node.hpp" -#include "openvino/core/node_output.hpp" -#include "openvino/op/broadcast.hpp" -#include "openvino/op/constant.hpp" -#include "openvino/op/multiply.hpp" -#include "openvino/op/reshape.hpp" namespace ov { namespace frontend { @@ -20,7 +13,7 @@ OutputVector translate_mul(const NodeContext& context) { auto res = std::make_shared(context.get_input(0), context.get_input(1)); return {res}; -}; +} } // namespace op } // namespace ggml diff --git a/ggml/src/ggml-openvino/openvino/op/permute.cpp b/ggml/src/ggml-openvino/openvino/op/permute.cpp index 42472f18cc..478c9430f0 100644 --- a/ggml/src/ggml-openvino/openvino/op/permute.cpp +++ b/ggml/src/ggml-openvino/openvino/op/permute.cpp @@ -1,21 +1,23 @@ +#include +#include + #include "../node_context.hpp" #include "../utils.hpp" -#include "openvino/op/constant.hpp" -#include "openvino/op/transpose.hpp" namespace ov { namespace frontend { namespace ggml { namespace op { + OutputVector translate_permute(const NodeContext& context) { num_inputs_check(context, 1, 1); - // TODO: make this more general + auto perm = argsort_descend(context.get_output_stride(0)); auto res = std::make_shared(context.get_input(0), - ov::op::v0::Constant::create(ov::element::i64, {3}, {1, 0, 2})); - + ov::op::v0::Constant::create(ov::element::i64, {3}, perm)); return {res}; -}; +} + } // namespace op } // namespace ggml } // namespace frontend diff --git a/ggml/src/ggml-openvino/openvino/op/reshape.cpp b/ggml/src/ggml-openvino/openvino/op/reshape.cpp index ca18b72c42..06b2bd339e 100644 --- a/ggml/src/ggml-openvino/openvino/op/reshape.cpp +++ b/ggml/src/ggml-openvino/openvino/op/reshape.cpp @@ -1,13 +1,12 @@ -#include "openvino/op/reshape.hpp" - #include +#include +#include +#include +#include #include #include "../node_context.hpp" #include "../utils.hpp" -#include "openvino/core/node.hpp" -#include "openvino/core/node_output.hpp" -#include "openvino/op/constant.hpp" namespace ov { namespace frontend { @@ -27,7 +26,7 @@ OutputVector translate_reshape(const NodeContext& context) { std::vector{-1, (int64_t)output_shape[1], (int64_t)output_shape[2]}); Output res = std::make_shared(context.get_input(0), new_shape_node, false); return {res}; -}; +} } // namespace op } // namespace ggml diff --git a/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp b/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp index 7b9783e8c9..a91fffb72d 100644 --- a/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp +++ b/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp @@ -1,11 +1,12 @@ +#include +#include +#include +#include +#include +#include + #include "../node_context.hpp" #include "../utils.hpp" -#include "openvino/op/add.hpp" -#include "openvino/op/constant.hpp" -#include "openvino/op/divide.hpp" -#include "openvino/op/multiply.hpp" -#include "openvino/op/reduce_sum.hpp" -#include "openvino/op/sqrt.hpp" namespace ov { namespace frontend { @@ -39,7 +40,7 @@ OutputVector translate_rms_norm(const NodeContext& context) { auto res = std::make_shared(input_node, scale); return {res}; -}; +} } // namespace op } // namespace ggml diff --git a/ggml/src/ggml-openvino/openvino/op/rope.cpp b/ggml/src/ggml-openvino/openvino/op/rope.cpp index d5083ae14b..aad156082e 100644 --- a/ggml/src/ggml-openvino/openvino/op/rope.cpp +++ b/ggml/src/ggml-openvino/openvino/op/rope.cpp @@ -1,27 +1,27 @@ #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include "../node_context.hpp" #include "../utils.hpp" -#include "openvino/core/node.hpp" -#include "openvino/core/node_output.hpp" -#include "openvino/op/add.hpp" -#include "openvino/op/broadcast.hpp" -#include "openvino/op/concat.hpp" -#include "openvino/op/constant.hpp" -#include "openvino/op/convert.hpp" -#include "openvino/op/cos.hpp" -#include "openvino/op/divide.hpp" -#include "openvino/op/multiply.hpp" -#include "openvino/op/reshape.hpp" -#include "openvino/op/shape_of.hpp" -#include "openvino/op/sin.hpp" -#include "openvino/op/slice.hpp" -#include "openvino/op/split.hpp" -#include "openvino/op/subtract.hpp" -#include "openvino/op/transpose.hpp" #define GGML_ROPE_TYPE_NEOX 2 @@ -163,7 +163,7 @@ OutputVector translate_rope(const NodeContext& context) { auto res_node = std::make_shared(ov::OutputVector{first_half_node, second_half_node}, 2); return {res_node}; } -}; +} } // namespace op } // namespace ggml diff --git a/ggml/src/ggml-openvino/openvino/op/scale.cpp b/ggml/src/ggml-openvino/openvino/op/scale.cpp index 392bfc1ed4..b393dd8aa2 100644 --- a/ggml/src/ggml-openvino/openvino/op/scale.cpp +++ b/ggml/src/ggml-openvino/openvino/op/scale.cpp @@ -1,12 +1,9 @@ -#include +#include +#include #include #include "../node_context.hpp" #include "../utils.hpp" -#include "openvino/core/node.hpp" -#include "openvino/core/node_output.hpp" -#include "openvino/op/constant.hpp" -#include "openvino/op/multiply.hpp" namespace ov { namespace frontend { @@ -23,7 +20,7 @@ OutputVector translate_scale(const NodeContext& context) { auto res = std::make_shared(context.get_input(0), scale_node); return {res}; -}; +} } // namespace op } // namespace ggml diff --git a/ggml/src/ggml-openvino/openvino/op/soft_max.cpp b/ggml/src/ggml-openvino/openvino/op/soft_max.cpp index cdb59f47d9..549c35a9b6 100644 --- a/ggml/src/ggml-openvino/openvino/op/soft_max.cpp +++ b/ggml/src/ggml-openvino/openvino/op/soft_max.cpp @@ -1,19 +1,19 @@ #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include "../node_context.hpp" #include "../utils.hpp" -#include "openvino/core/node.hpp" -#include "openvino/core/node_output.hpp" -#include "openvino/op/add.hpp" -#include "openvino/op/concat.hpp" -#include "openvino/op/constant.hpp" -#include "openvino/op/convert.hpp" -#include "openvino/op/multiply.hpp" -#include "openvino/op/slice.hpp" -#include "openvino/op/softmax.hpp" namespace ov { namespace frontend { diff --git a/ggml/src/ggml-openvino/openvino/op/transpose.cpp b/ggml/src/ggml-openvino/openvino/op/transpose.cpp index f7408f40d4..7d33ca9d61 100644 --- a/ggml/src/ggml-openvino/openvino/op/transpose.cpp +++ b/ggml/src/ggml-openvino/openvino/op/transpose.cpp @@ -1,4 +1,4 @@ -#include "openvino/op/transpose.hpp" +#include #include "../node_context.hpp" #include "../utils.hpp" @@ -15,7 +15,7 @@ OutputVector translate_transpose(const NodeContext& context) { auto res = std::make_shared(context.get_input(0), ov::op::v0::Constant::create(ov::element::i64, {3}, perm)); return {res}; -}; +} } // namespace op } // namespace ggml diff --git a/ggml/src/ggml-openvino/openvino/op/unary.cpp b/ggml/src/ggml-openvino/openvino/op/unary.cpp deleted file mode 100644 index 391e0a7599..0000000000 --- a/ggml/src/ggml-openvino/openvino/op/unary.cpp +++ /dev/null @@ -1,24 +0,0 @@ - -#include -#include - -#include "../node_context.hpp" -#include "../utils.hpp" -#include "openvino/core/node.hpp" -#include "openvino/core/node_output.hpp" - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_unary(const NodeContext& context) { - num_inputs_check(context, 1, 1); - - return {context.get_input(0)}; -}; - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp b/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp index 2a90a79475..1c396e6aaf 100644 --- a/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp +++ b/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp @@ -1,12 +1,9 @@ -#include -#include +#include +#include +#include #include "../node_context.hpp" #include "../utils.hpp" -#include "openvino/core/node.hpp" -#include "openvino/core/node_output.hpp" -#include "openvino/op/multiply.hpp" -#include "openvino/op/sigmoid.hpp" namespace ov { namespace frontend { @@ -21,7 +18,7 @@ OutputVector translate_unary_silu(const NodeContext& context) { auto res = std::make_shared(input, sigmoid); return {res}; -}; +} } // namespace op } // namespace ggml diff --git a/ggml/src/ggml-openvino/openvino/op/view.cpp b/ggml/src/ggml-openvino/openvino/op/view.cpp index aaf117b662..fcfb9f732c 100644 --- a/ggml/src/ggml-openvino/openvino/op/view.cpp +++ b/ggml/src/ggml-openvino/openvino/op/view.cpp @@ -1,13 +1,4 @@ -#include -#include - #include "../utils.hpp" -#include "openvino/core/node.hpp" -#include "openvino/core/node_output.hpp" -#include "openvino/op/constant.hpp" -#include "openvino/op/convert.hpp" -#include "openvino/op/reshape.hpp" -#include "openvino/op/strided_slice.hpp" namespace ov { namespace frontend { @@ -18,7 +9,7 @@ OutputVector translate_view(const NodeContext& context) { num_inputs_check(context, 1, 1); return {context.get_input(0)}; -}; +} } // namespace op } // namespace ggml diff --git a/ggml/src/ggml-openvino/openvino/op_table.cpp b/ggml/src/ggml-openvino/openvino/op_table.cpp index af51bb157e..d588b2bff0 100644 --- a/ggml/src/ggml-openvino/openvino/op_table.cpp +++ b/ggml/src/ggml-openvino/openvino/op_table.cpp @@ -37,7 +37,7 @@ GGML_OP_CONVERTER(translate_view); } // namespace op -const std::unordered_map get_supported_ops() { +std::unordered_map get_supported_ops() { return {{"GGML_OP_ADD", op::translate_1to1_match_2_inputs}, {"GGML_OP_ADD1", op::translate_1to1_match_2_inputs}, {"GGML_OP_CONT", op::translate_cont}, diff --git a/ggml/src/ggml-openvino/openvino/op_table.hpp b/ggml/src/ggml-openvino/openvino/op_table.hpp index c83aaa199f..1a71a06c18 100644 --- a/ggml/src/ggml-openvino/openvino/op_table.hpp +++ b/ggml/src/ggml-openvino/openvino/op_table.hpp @@ -6,7 +6,7 @@ namespace ov { namespace frontend { namespace ggml { -const std::unordered_map get_supported_ops(); +std::unordered_map get_supported_ops(); } // namespace ggml } // namespace frontend diff --git a/ggml/src/ggml-openvino/openvino/translate_session.cpp b/ggml/src/ggml-openvino/openvino/translate_session.cpp index f5b14d3a0f..012e9178c6 100644 --- a/ggml/src/ggml-openvino/openvino/translate_session.cpp +++ b/ggml/src/ggml-openvino/openvino/translate_session.cpp @@ -1,8 +1,5 @@ #include "translate_session.hpp" -#include -#include - #include "input_model.hpp" namespace ov { @@ -22,39 +19,9 @@ std::shared_ptr TranslateSession::get_converted_model() { return m_ov_model; } m_ov_model = translate_graph(m_input_model); - // print_model_topology(); return m_ov_model; } -void TranslateSession::print_model_topology() { - try { - std::ofstream outfile("model_topology.txt", std::ios::out | std::ios::app); - if (!outfile.is_open()) { - throw std::runtime_error("Failed to open file for writing model topology."); - } - - outfile << "============ Model ============" << std::endl; - for (const auto& op : m_ov_model->get_ordered_ops()) { - outfile << "Operation: " << op->get_friendly_name() << std::endl; - outfile << " Inputs:" << std::endl; - for (const auto& input : op->inputs()) { - outfile << " " << input.get_node()->get_friendly_name() << " -> " << input.get_element_type() << " " - << input.get_shape() << std::endl; - } - outfile << " Outputs:" << std::endl; - for (const auto& output : op->outputs()) { - outfile << " " << output.get_node()->get_friendly_name() << " -> " << output.get_element_type() - << " " << output.get_shape() << std::endl; - } - outfile << std::endl; - } - outfile << "===============================" << std::endl; - outfile.close(); - } catch (const std::exception& ex) { - std::cout << ex.what() << std::endl; - } -} - std::shared_ptr TranslateSession::translate_graph(const frontend::InputModel::Ptr& input_model) { ov::ParameterVector params; ov::ResultVector results; @@ -86,16 +53,12 @@ std::shared_ptr TranslateSession::translate_graph(const frontend::InputMo auto operation_type = node->get_op_type(); ov::OutputVector converted_outputs; auto it = m_translator_map.find(operation_type); - if (it != m_translator_map.end()) { - try { - NodeContext node_context(node, tensor_map, this); - converted_outputs = it->second(node_context); - } catch (const std::exception& ex) { - std::cout << ex.what() << std::endl; - } - } else { - // TODO - } + FRONT_END_OP_CONVERSION_CHECK(it != m_translator_map.end(), + "Translation for operation type ", + operation_type, + " is not implemented."); + NodeContext node_context(node, tensor_map, this); + converted_outputs = it->second(node_context); const auto& node_output_names = node->get_output_names(); FRONT_END_OP_CONVERSION_CHECK(node_output_names.size() == converted_outputs.size(), @@ -122,7 +85,7 @@ std::shared_ptr TranslateSession::translate_graph(const frontend::InputMo "Output name not found in tensor map: ", name); auto result = std::make_shared(tensor_map->at(name)); - // result->set_friendly_name(it); + result->set_friendly_name(name); results.push_back(result); } diff --git a/ggml/src/ggml-openvino/utils.cpp b/ggml/src/ggml-openvino/utils.cpp index 34bcfc54a7..09bf0d0ac5 100644 --- a/ggml/src/ggml-openvino/utils.cpp +++ b/ggml/src/ggml-openvino/utils.cpp @@ -69,10 +69,13 @@ enum ggml_status openvino_frontend_compute(ggml_backend_t backend, struct ggml_c std::shared_ptr model; ov::CompiledModel compiled_model; + int64_t decoder_end_time; int64_t conversion_end_time; int64_t compile_end_time; auto ggml_decoder = get_ggml_decoder(cgraph); + decoder_end_time = ggml_time_us(); + auto it = compiled_cache.find(cgraph); if (it != compiled_cache.end()) { model = it->second.first; @@ -147,7 +150,8 @@ enum ggml_status openvino_frontend_compute(ggml_backend_t backend, struct ggml_c if (getenv("GGML_OPENVINO_PROFILING")) { GGML_LOG_INFO("GGML OpenVINO Backend: \n"); - GGML_LOG_INFO(" - Graph conversion Time: %ld ms \n", (conversion_end_time - start_time) / 1000); + GGML_LOG_INFO(" - Graph decoder Time: %ld ms \n", (decoder_end_time - start_time) / 1000); + GGML_LOG_INFO(" - Graph conversion Time: %ld ms \n", (conversion_end_time - decoder_end_time) / 1000); GGML_LOG_INFO(" - Graph compile Time: %ld ms \n", (compile_end_time - conversion_end_time) / 1000); GGML_LOG_INFO(" - Graph Input Time: %ld ms \n", (input_end_time - compile_end_time) / 1000); GGML_LOG_INFO(" - Graph Inference Time: %ld ms \n", (infer_end_time - input_end_time) / 1000);