diff --git a/ggml/src/ggml-openvino/decoder.h b/ggml/src/ggml-openvino/decoder.h index be943716f2..c7f1bbd725 100644 --- a/ggml/src/ggml-openvino/decoder.h +++ b/ggml/src/ggml-openvino/decoder.h @@ -12,9 +12,9 @@ class GgmlDecoder : public DecoderBase { public: virtual ov::Any get_attribute(const std::string& name) const = 0; - virtual PartialShape get_input_shape(size_t index) const = 0; + virtual PartialShape get_input_shape(const std::string& name) const = 0; - virtual element::Type get_input_type(size_t index) const = 0; + virtual element::Type get_input_type(const std::string& name) const = 0; virtual size_t get_input_size() const = 0; @@ -23,19 +23,15 @@ public: std::string& producer_output_port_name, size_t& producer_output_port_index) const = 0; - virtual bool is_graph_input(size_t index) const = 0; - virtual std::string& get_input_name(size_t index) const = 0; - virtual PartialShape get_output_shape(size_t index) const = 0; + virtual std::vector get_input_names() const = 0; - virtual element::Type get_output_type(size_t index) const = 0; + virtual PartialShape get_output_shape(const std::string& name) const = 0; - virtual size_t get_output_size() const = 0; + virtual element::Type get_output_type(const std::string& name) const = 0; - virtual bool is_graph_output(size_t index) const = 0; - - virtual int32_t* get_output_op_params(size_t index) const = 0; + virtual int32_t* get_output_op_params(const std::string& name) const = 0; virtual std::string& get_output_name(size_t index) const = 0; @@ -49,6 +45,8 @@ public: virtual const std::string& get_op_name() const = 0; + virtual void visit_subgraph(std::function)> node_visitor) const = 0; + // virtual const std::vector& outputs() const = 0; // virtual size_t output(size_t index) const = 0; diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index b367987372..ab4b0995a5 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -2,11 +2,8 @@ #include #include -GgmlOvDecoder::GgmlOvDecoder(struct ggml_tensor * node, struct ggml_cgraph * cgraph) - :m_cgraph(cgraph), - m_node(node), - m_op_name(std::string(m_node->name)) { - switch (m_node->op) { +void GgmlOvDecoder::set_input_output(ggml_tensor* node, std::map& inputs, std::map& outputs) { + switch (node->op) { // Unary OPs case GGML_OP_UNARY: case GGML_OP_RESHAPE: @@ -16,22 +13,26 @@ GgmlOvDecoder::GgmlOvDecoder(struct ggml_tensor * node, struct ggml_cgraph * cgr case GGML_OP_CPY: case GGML_OP_RMS_NORM: { - m_inputs.push_back(m_node->src[0]); - m_outputs.push_back(m_node); + inputs[node->src[0]->name] = node->src[0]; + outputs[node->name] = node; + m_input_names.push_back(node->src[0]->name); + m_output_names.push_back(node->name); break; } - // For view, input is m_node itself + // For view, input is node itself case GGML_OP_VIEW: { - m_inputs.push_back(m_node); - m_outputs.push_back(m_node); + inputs[node->src[0]->name] = node; + outputs[node->name] = node; break; } // SCALE case GGML_OP_SCALE: { - m_inputs.push_back(m_node->src[0]); - m_outputs.push_back(m_node); + inputs[node->src[0]->name] = node->src[0]; + outputs[node->name] = node; + m_input_names.push_back(node->name); + m_output_names.push_back(node->name); break; } // OPs with 2 inputs @@ -43,18 +44,25 @@ GgmlOvDecoder::GgmlOvDecoder(struct ggml_tensor * node, struct ggml_cgraph * cgr case GGML_OP_GET_ROWS: case GGML_OP_SOFT_MAX: { - m_inputs.push_back(m_node->src[0]); - m_inputs.push_back(m_node->src[1]); - m_outputs.push_back(m_node); + inputs[node->src[0]->name] = node->src[0]; + inputs[node->src[1]->name] = node->src[1]; + outputs[node->name] = node; + m_input_names.push_back(node->src[0]->name); + m_input_names.push_back(node->src[1]->name); + m_output_names.push_back(node->name); break; } // OPs with 3 inputs: case GGML_OP_ROPE: { - m_inputs.push_back(m_node->src[0]); - m_inputs.push_back(m_node->src[1]); - m_inputs.push_back(m_node->src[2]); // ??? - m_outputs.push_back(m_node); + inputs[node->src[0]->name] = node->src[0]; + inputs[node->src[1]->name] = node->src[1]; + inputs[node->src[2]->name] = node->src[2]; + outputs[node->name] = node; + m_input_names.push_back(node->src[0]->name); + m_input_names.push_back(node->src[1]->name); + m_input_names.push_back(node->src[2]->name); + m_output_names.push_back(node->name); break; } default: @@ -62,13 +70,33 @@ GgmlOvDecoder::GgmlOvDecoder(struct ggml_tensor * node, struct ggml_cgraph * cgr } } -ov::PartialShape GgmlOvDecoder::get_input_shape(size_t index) const { +GgmlOvDecoder::GgmlOvDecoder(struct ggml_tensor * node, struct ggml_cgraph * cgraph) + :m_cgraph(cgraph), + m_node(node), + m_op_name(m_node ? std::string(m_node->name) : "NONE_OP") { + m_inputs.clear(); + m_outputs.clear(); + m_input_names.clear(); + m_output_names.clear(); + // If first init + if (m_node) { + set_input_output(m_node, m_inputs, m_outputs); + } else { + for (int node_n = 0; node_n < m_cgraph->n_nodes; node_n++) { + auto cur_node = m_cgraph->nodes[node_n]; + m_nodes.push_back(cur_node); + // Init model input and output + set_input_output(cur_node, m_inputs, m_outputs); + } + } +} + +ov::PartialShape GgmlOvDecoder::get_input_shape(const std::string& name) const { ov::PartialShape input_shape; // Use input_node->ne - ggml_tensor * node = m_inputs[index]; + ggml_tensor * node = m_inputs.at(name); std::vector shape; - // GGML_MAX_DIMS - // for (int i = 0; i < GGML_MAX_DIMS; ++i) { + for (int i = GGML_MAX_DIMS - 2; i >= 0 ; --i) { if (node->ne[i] == 0) { return input_shape; @@ -79,10 +107,9 @@ ov::PartialShape GgmlOvDecoder::get_input_shape(size_t index) const { return input_shape; } -ov::element::Type GgmlOvDecoder::get_input_type(size_t index) const { +ov::element::Type GgmlOvDecoder::get_input_type(const std::string& name) const { ov::element::Type type = ov::element::dynamic; - // GGML_LOG_DEBUG("%d\n", m_inputs[index]->type); - switch (m_inputs[index]->type) { + switch (m_inputs.at(name)->type) { case GGML_TYPE_F32: type = ov::element::f32; break; @@ -102,28 +129,24 @@ ov::element::Type GgmlOvDecoder::get_input_type(size_t index) const { } size_t GgmlOvDecoder::get_input_size() const { - return m_inputs.size(); -} - -bool GgmlOvDecoder::is_graph_input(size_t index) const { - if (m_inputs[index]->flags & GGML_TENSOR_FLAG_INPUT ) { - return true; - } - return false; + return m_input_names.size(); } std::string& GgmlOvDecoder::get_input_name(size_t index) const { - m_name = std::string(m_inputs[index]->name); + m_name = m_input_names[index]; return m_name; } -ov::PartialShape GgmlOvDecoder::get_output_shape(size_t index) const { +std::vector GgmlOvDecoder::get_input_names() const { + return m_input_names; +} + +ov::PartialShape GgmlOvDecoder::get_output_shape(const std::string& name) const { ov::PartialShape output_shape; // Use input_node->ne - ggml_tensor * node = m_outputs[index]; + ggml_tensor * node = m_outputs.at(name); std::vector shape; - // GGML_MAX_DIMS - // for (int i = 0; i < GGML_MAX_DIMS; ++i) { + for (int i = GGML_MAX_DIMS - 2; i >= 0 ; --i) { if (node->ne[i] == 0 ) { // empty if any dimension has no elements @@ -135,10 +158,10 @@ ov::PartialShape GgmlOvDecoder::get_output_shape(size_t index) const { return output_shape; } -ov::element::Type GgmlOvDecoder::get_output_type(size_t index) const { +ov::element::Type GgmlOvDecoder::get_output_type(const std::string& name) const { // TODO: Change to Output ov::element::Type type = ov::element::dynamic; - switch (m_outputs[index]->type) { + switch (m_outputs.at(name)->type) { case GGML_TYPE_F32: type = ov::element::f32; break; @@ -157,30 +180,31 @@ ov::element::Type GgmlOvDecoder::get_output_type(size_t index) const { return type; } -bool GgmlOvDecoder::is_graph_output(size_t index) const { - if (m_outputs[index]->flags & GGML_TENSOR_FLAG_OUTPUT) { - return true; - } - return false; -} - -int32_t* GgmlOvDecoder::get_output_op_params(size_t index) const{ - return m_outputs[index]->op_params; -} - -size_t GgmlOvDecoder::get_output_size() const { - return m_outputs.size(); +int32_t* GgmlOvDecoder::get_output_op_params(const std::string& name) const{ + return m_outputs.at(name)->op_params; } std::string& GgmlOvDecoder::get_output_name(size_t index) const { - m_name = std::string(m_outputs[index]->name); + m_name = std::string(m_output_names[index]); return m_name; } +std::vector GgmlOvDecoder::get_output_names() const { + return m_output_names; +} + const std::string& GgmlOvDecoder::get_op_name() const { return m_op_name; } +void GgmlOvDecoder::visit_subgraph(std::function)> node_visitor) const { + for (const auto& node : m_nodes) { + auto decoder = std::make_shared(node, m_cgraph); + // m_decoders.push_back(decoder); + node_visitor(decoder); + } +} + const std::string& GgmlOvDecoder::get_op_type() const { static const std::map opTypeMap = { {GGML_OP_ACC, "GGML_OP_ACC"}, diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h index ceae589ed4..56bb3f889f 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.h +++ b/ggml/src/ggml-openvino/ggml-decoder.h @@ -6,6 +6,7 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { public: using ov::frontend::ggml::GgmlDecoder::GgmlDecoder; + GgmlOvDecoder(struct ggml_tensor * node, struct ggml_cgraph * cgraph); virtual ov::Any get_attribute(const std::string& name) const override { @@ -13,9 +14,9 @@ public: GGML_UNUSED(name); } - virtual ov::PartialShape get_input_shape(size_t index) const override; + virtual ov::PartialShape get_input_shape(const std::string& name) const override; - virtual ov::element::Type get_input_type(size_t index) const override; + virtual ov::element::Type get_input_type(const std::string& name) const override; virtual size_t get_input_size() const override; @@ -29,19 +30,15 @@ public: GGML_UNUSED(producer_output_port_index); } - virtual bool is_graph_input(size_t index) const override; - virtual std::string& get_input_name(size_t index) const override; - virtual ov::PartialShape get_output_shape(size_t index) const override; + virtual std::vector get_input_names() const override; - virtual ov::element::Type get_output_type(size_t index) const override; + virtual ov::PartialShape get_output_shape(const std::string& name) const override; - virtual size_t get_output_size() const override; + virtual ov::element::Type get_output_type(const std::string& name) const override; - virtual bool is_graph_output(size_t index) const override; - - virtual int32_t* get_output_op_params(size_t index) const override; + virtual int32_t* get_output_op_params(const std::string& name) const override; virtual std::string& get_output_name(size_t index) const override; @@ -55,24 +52,27 @@ public: virtual const std::string& get_op_name() const override; - const ggml_tensor* get_input_ggml_tensor(size_t index) const { - return m_inputs[index]; + virtual void visit_subgraph(std::function)> node_visitor) const override; + + const ggml_tensor* get_input_ggml_tensor(std::string& name) const { + return m_inputs.at(name); } - const ggml_tensor* get_output_ggml_tensor(size_t index) const { - return m_outputs[index]; + const ggml_tensor* get_output_ggml_tensor(std::string& name) const { + return m_outputs.at(name); } - // virtual const std::vector& outputs() const override; - - // virtual size_t output(size_t index) const override; - private: - size_t m_index; + void set_input_output(ggml_tensor* node, std::map& inputs, std::map& outputs); + struct ggml_cgraph * m_cgraph; - std::vector m_inputs; - std::vector m_outputs; - ggml_tensor * m_node; + std::map m_inputs; + std::vector m_input_names; + std::map m_outputs; + std::vector m_output_names; + ggml_tensor* m_node; + std::vector m_nodes; + std::vector> m_decoders; const std::string m_op_name; mutable std::string m_name; }; diff --git a/ggml/src/ggml-openvino/ggml-graph-iterator.cpp b/ggml/src/ggml-openvino/ggml-graph-iterator.cpp deleted file mode 100644 index 5c06179023..0000000000 --- a/ggml/src/ggml-openvino/ggml-graph-iterator.cpp +++ /dev/null @@ -1,95 +0,0 @@ -#include "ggml-graph-iterator.h" -#include -#include - -namespace ov { -namespace frontend { -namespace tensorflow { -namespace ggml { - -GgmlOvGraphIterator::GgmlOvGraphIterator(struct ggml_cgraph * cgraph) - :m_cgraph(cgraph) { - initialize_decoders(); - #ifdef GGML_OPENVINO_DEBUG - dump_graph_iterator(); - #endif -} - -void GgmlOvGraphIterator::initialize_decoders() { - auto nodes_size = m_cgraph->n_nodes; - // Initialize decoder for each node - // m_decoders.resize(static_cast(nodes_size)); - - for (int i = 0; i < nodes_size; ++i) { - auto decoder = std::make_shared(m_cgraph->nodes[i], m_cgraph); - m_decoders.push_back(decoder); - for (size_t inp = 0; inp < decoder->get_input_size(); ++inp) { - // Skip duplicate input name - if (std::find(m_input_names.begin(), m_input_names.end(), decoder->get_input_name(inp)) == m_input_names.end()) { - m_input_names.push_back(decoder->get_input_name(inp)); - } - } - for (size_t inp = 0; inp < decoder->get_output_size(); ++inp) { - // Skip duplicate output name - auto output_name = decoder->get_output_name(inp); - if (std::find(m_output_names.begin(), m_output_names.end(), output_name) == m_output_names.end()) { - m_output_names.push_back(decoder->get_output_name(inp)); - } - } - } - -} - -void GgmlOvGraphIterator::reset() { - node_index = 0; - } - -size_t GgmlOvGraphIterator::size() const { - return m_decoders.size(); -} - -void GgmlOvGraphIterator::next() { - node_index++; -} - -bool GgmlOvGraphIterator::is_end() const { - return node_index >= m_decoders.size(); -} - -std::shared_ptr GgmlOvGraphIterator::get_decoder() const { - return m_decoders[node_index]; -} - -std::vector GgmlOvGraphIterator::get_input_names() const { - return m_input_names; -} - -std::vector GgmlOvGraphIterator::get_output_names() const { - return m_output_names; -} - -void GgmlOvGraphIterator::dump_graph_iterator() const { - for (size_t i = 0; i < m_decoders.size(); ++i) { - GGML_LOG_INFO("\nOP %zu: %s\n", i, m_decoders[i]->get_op_name().c_str()); - for (size_t inp = 0; inp < m_decoders[i]->get_input_size(); ++inp) { - ov::PartialShape pshape = std::dynamic_pointer_cast(m_decoders[i])->get_input_shape(inp); - ov::element::Type ptype = std::dynamic_pointer_cast(m_decoders[i])->get_input_type(inp); - GGML_LOG_INFO("- Input name: %s\n", std::dynamic_pointer_cast(m_decoders[i])->get_input_name(inp).c_str()); - GGML_LOG_INFO(" Input shape: %s\n", pshape.to_string().c_str()); - GGML_LOG_INFO(" Input type: %s\n", ptype.to_string().c_str()); - } - for (size_t outp = 0; outp < std::dynamic_pointer_cast(m_decoders[i])->get_output_size(); ++outp) { - ov::PartialShape pshape = std::dynamic_pointer_cast(m_decoders[i])->get_output_shape(outp); - ov::element::Type ptype = std::dynamic_pointer_cast(m_decoders[i])->get_output_type(outp); - GGML_LOG_INFO("- Output name: %s\n", std::dynamic_pointer_cast(m_decoders[i])->get_output_name(outp).c_str()); - GGML_LOG_INFO(" Output shape: %s\n", pshape.to_string().c_str()); - GGML_LOG_INFO(" Output type: %s\n", ptype.to_string().c_str()); - - } - } -} - -} -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/ggml-graph-iterator.h b/ggml/src/ggml-openvino/ggml-graph-iterator.h deleted file mode 100644 index 305afb5c98..0000000000 --- a/ggml/src/ggml-openvino/ggml-graph-iterator.h +++ /dev/null @@ -1,61 +0,0 @@ -#pragma once - -#include "graph_iterator.h" -#include "ggml-decoder.h" -#include - -// To remove tensorflow -namespace ov { -namespace frontend { -namespace tensorflow { -namespace ggml { - -class GgmlOvGraphIterator : public GgmlGraphIterator { - -protected: - void initialize_decoders(); - -public: - using Ptr = std::shared_ptr; - GgmlOvGraphIterator(struct ggml_cgraph * cgraph); - - /// \brief Get a number of operation nodes in the sgraph - virtual size_t size() const override; - - /// \brief Set iterator to the start position - virtual void reset() override; - - /// \brief Move to the next node in the graph - virtual void next() override; - - /// \brief Returns true if iterator goes out of the range of available nodes - virtual bool is_end() const override; - - /// \brief Return a pointer to a decoder of the current node - virtual std::shared_ptr get_decoder() const override; - - virtual std::shared_ptr get_body_graph_iterator(const std::string& func_name) const override { - return nullptr; - GGML_UNUSED(func_name); - } - - /// \brief Returns a vector of input names in the original order - virtual std::vector get_input_names() const override; - - /// \brief Returns a vector of output names in the original order - virtual std::vector get_output_names() const override; - - virtual void dump_graph_iterator() const; - -private: - struct ggml_cgraph * m_cgraph; - size_t node_index = 0; - std::vector> m_decoders; - std::vector m_input_names; - std::vector m_output_names; -}; - -} -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/graph_iterator.h b/ggml/src/ggml-openvino/graph_iterator.h deleted file mode 100644 index e0b475e445..0000000000 --- a/ggml/src/ggml-openvino/graph_iterator.h +++ /dev/null @@ -1,43 +0,0 @@ -#pragma once - -#include "openvino/frontend/graph_iterator.hpp" - -namespace ov { -namespace frontend { -namespace tensorflow { // To be Removed -namespace ggml { - -// TODO: Directly include from openvino -class GgmlGraphIterator : public GraphIterator { -public: - - virtual size_t size() const = 0; - - virtual void reset() = 0; - - virtual void next() = 0; - - virtual bool is_end() const = 0; - - virtual std::shared_ptr get_decoder() const = 0; - - virtual std::vector get_input_names() const = 0; - - virtual std::vector get_output_names() const = 0; - - virtual std::shared_ptr get_body_graph_iterator(const std::string& func_name) const = 0; - - virtual std::map get_input_names_map() const { - return {}; - } - - virtual std::map get_output_names_map() const { - return {}; - } - -}; - -} -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/utils.cpp b/ggml/src/ggml-openvino/utils.cpp index 2dfe837cbd..2436f86feb 100644 --- a/ggml/src/ggml-openvino/utils.cpp +++ b/ggml/src/ggml-openvino/utils.cpp @@ -1,49 +1,40 @@ #include "utils.h" +#include "ggml-impl.h" #include "ggml-backend-impl.h" #include #include -using ov::frontend::tensorflow::ggml::GgmlOvGraphIterator; +using ov::frontend::ggml::GgmlDecoder; -std::shared_ptr get_ggml_graph_iterator(struct ggml_cgraph * cgraph) { - return std::make_shared(cgraph); +std::shared_ptr get_ggml_decoder(struct ggml_cgraph * cgraph) { + return std::make_shared(nullptr, cgraph); } -std::map get_ggml_graph_input_tensors(std::shared_ptr ggml_graph_iterator) { +std::map get_ggml_graph_input_tensors(std::shared_ptr ggml_decoder) { std::map input_tensors; - auto input_names = ggml_graph_iterator->get_input_names(); - ggml_graph_iterator->reset(); - for (; !ggml_graph_iterator->is_end(); ggml_graph_iterator->next()) { - auto decoder = std::dynamic_pointer_cast(ggml_graph_iterator->get_decoder()); - for (size_t inp = 0; inp < decoder->get_input_size(); ++inp) { - if (std::find(input_names.begin(), input_names.end(), decoder->get_input_name(inp)) != input_names.end()) { - auto input_data = decoder->get_input_ggml_tensor(inp)->data; - #ifdef GGML_OPENVINO_DEBUG - printf("Subgraph input %d: %g\n", inp, *(double*)(input_data)); - #endif - ov::Tensor input_tensor = ov::Tensor(decoder->get_input_type(inp), decoder->get_input_shape(inp).to_shape(), input_data); - input_tensors[decoder->get_input_name(inp)] = input_tensor; - } - } + auto input_names = ggml_decoder->get_input_names(); + for (size_t inp = 0; inp < input_names.size(); ++inp) { + auto name = input_names[inp]; + auto input_data = ggml_decoder->get_input_ggml_tensor(name)->data; + #ifdef GGML_OPENVINO_DEBUG + printf("Subgraph input %d: %g\n", inp, *(double*)(input_data)); + #endif + ov::Tensor input_tensor = ov::Tensor(ggml_decoder->get_input_type(name), ggml_decoder->get_input_shape(name).to_shape(), input_data); + input_tensors[name] = input_tensor; } return input_tensors; } -std::map get_ggml_graph_output_dst(std::shared_ptr ggml_graph_iterator) { +std::map get_ggml_graph_output_dst(std::shared_ptr ggml_decoder) { std::map output_tensors; - auto output_names = ggml_graph_iterator->get_output_names(); - ggml_graph_iterator->reset(); - for (; !ggml_graph_iterator->is_end(); ggml_graph_iterator->next()) { - auto decoder = std::dynamic_pointer_cast(ggml_graph_iterator->get_decoder()); - for (size_t inp = 0; inp < decoder->get_output_size(); ++inp) { - if (std::find(output_names.begin(), output_names.end(), decoder->get_output_name(inp)) != output_names.end()) { - auto output_data = decoder->get_output_ggml_tensor(inp)->data; - #ifdef GGML_OPENVINO_DEBUG - printf("Output %d: %g\n", inp, *(double*)(output_data)); - #endif - output_tensors[decoder->get_output_name(inp)] = output_data; - } - } + auto output_names = ggml_decoder->get_output_names(); + for (size_t inp = 0; inp < output_names.size(); ++inp) { + auto name = output_names[inp]; + auto output_data = ggml_decoder->get_output_ggml_tensor(name)->data; + #ifdef GGML_OPENVINO_DEBUG + printf("Output %d: %g\n", inp, *(double*)(output_data)); + #endif + output_tensors[name] = output_data; } return output_tensors; } @@ -74,12 +65,10 @@ enum ggml_status openvino_frontend_compute(ggml_backend_t backend, struct ggml_c GGML_LOG_INFO("GGML FrontEnd is initialized \n"); #endif } - - auto ggml_graph_iterator = get_ggml_graph_iterator(cgraph); - std::shared_ptr graph_iterator = ggml_graph_iterator; - + auto ggml_decoder = get_ggml_decoder(cgraph); + std::shared_ptr graph_decoder = ggml_decoder; // Load GraphIterator -> InputModel - ov::frontend::InputModel::Ptr input_model = front_end->load(graph_iterator); + ov::frontend::InputModel::Ptr input_model = front_end->load(graph_decoder); if (!input_model) { GGML_LOG_ERROR("Input Model is not loaded \n"); return GGML_STATUS_FAILED; @@ -106,8 +95,8 @@ enum ggml_status openvino_frontend_compute(ggml_backend_t backend, struct ggml_c ov::InferRequest infer_request = compiled_model.create_infer_request(); // Get input tensor - auto input_names = ggml_graph_iterator->get_input_names(); - auto input_tensors = get_ggml_graph_input_tensors(ggml_graph_iterator); + auto input_names = ggml_decoder->get_input_names(); + auto input_tensors = get_ggml_graph_input_tensors(ggml_decoder); // Set input tensor for (size_t i = 0; i < input_names.size(); i++) { @@ -117,11 +106,14 @@ enum ggml_status openvino_frontend_compute(ggml_backend_t backend, struct ggml_c infer_request.infer(); // Set dst data for outputs - auto output_names = ggml_graph_iterator->get_output_names(); - auto output_tensors = get_ggml_graph_output_dst(ggml_graph_iterator); + auto output_names = ggml_decoder->get_output_names(); + auto output_tensors = get_ggml_graph_output_dst(ggml_decoder); for (size_t i = 0; i < output_names.size(); i++) { auto output_tensor = infer_request.get_output_tensor(i); std::memcpy(output_tensors[output_names[i]], output_tensor.data(), output_tensor.get_byte_size()); + #ifdef GGML_OPENVINO_DEBUG + printf("Output %s after: %g\n", output_names[i], *(double*)(output_tensor.data())); + #endif } return GGML_STATUS_SUCCESS; diff --git a/ggml/src/ggml-openvino/utils.h b/ggml/src/ggml-openvino/utils.h index 15dd46ed4e..7ec633beda 100644 --- a/ggml/src/ggml-openvino/utils.h +++ b/ggml/src/ggml-openvino/utils.h @@ -1,6 +1,4 @@ -#include "ggml-graph-iterator.h" +#include "ggml-decoder.h" #include "ggml-backend-impl.h" -std::shared_ptr get_ggml_graph_iterator(struct ggml_cgraph * cgraph); - enum ggml_status openvino_frontend_compute (ggml_backend_t backend, struct ggml_cgraph * cgraph);