happyz
/
llama.cpp
mirror of https://github.com/ggerganov/llama.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Change to implementation following pytorch frontend

This commit is contained in:
yumengbo 2024-12-13 07:28:28 +08:00 committed by Mustafa Cavus
parent e95f29cbc0
commit b100f89bad
8 changed files with 143 additions and 330 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
ggml/src/ggml-openvino/decoder.h
View File

@ -12,9 +12,9 @@ class GgmlDecoder : public DecoderBase {
public: public:
virtual ov::Any get_attribute(const std::string& name) const = 0; virtual ov::Any get_attribute(const std::string& name) const = 0;
virtual PartialShape get_input_shape(size_t index) const = 0; virtual PartialShape get_input_shape(const std::string& name) const = 0;
virtual element::Type get_input_type(size_t index) const = 0; virtual element::Type get_input_type(const std::string& name) const = 0;
virtual size_t get_input_size() const = 0; virtual size_t get_input_size() const = 0;
@ -23,19 +23,15 @@ public:
std::string& producer_output_port_name, std::string& producer_output_port_name,
size_t& producer_output_port_index) const = 0; size_t& producer_output_port_index) const = 0;
virtual bool is_graph_input(size_t index) const = 0;
virtual std::string& get_input_name(size_t index) const = 0; virtual std::string& get_input_name(size_t index) const = 0;
virtual PartialShape get_output_shape(size_t index) const = 0; virtual std::vector<std::string> get_input_names() const = 0;
virtual element::Type get_output_type(size_t index) const = 0; virtual PartialShape get_output_shape(const std::string& name) const = 0;
virtual size_t get_output_size() const = 0; virtual element::Type get_output_type(const std::string& name) const = 0;
virtual bool is_graph_output(size_t index) const = 0; virtual int32_t* get_output_op_params(const std::string& name) const = 0;
virtual int32_t* get_output_op_params(size_t index) const = 0;
virtual std::string& get_output_name(size_t index) const = 0; virtual std::string& get_output_name(size_t index) const = 0;
@ -49,6 +45,8 @@ public:
virtual const std::string& get_op_name() const = 0; virtual const std::string& get_op_name() const = 0;
virtual void visit_subgraph(std::function<void(std::shared_ptr<GgmlDecoder>)> node_visitor) const = 0;
// virtual const std::vector<size_t>& outputs() const = 0; // virtual const std::vector<size_t>& outputs() const = 0;
// virtual size_t output(size_t index) const = 0; // virtual size_t output(size_t index) const = 0;

134
ggml/src/ggml-openvino/ggml-decoder.cpp
View File

@ -2,11 +2,8 @@
#include <ggml.h> #include <ggml.h>
#include <ggml-impl.h> #include <ggml-impl.h>
GgmlOvDecoder::GgmlOvDecoder(struct ggml_tensor * node, struct ggml_cgraph * cgraph) void GgmlOvDecoder::set_input_output(ggml_tensor* node, std::map<std::string, ggml_tensor *>& inputs, std::map<std::string, ggml_tensor *>& outputs) {
:m_cgraph(cgraph), switch (node->op) {
m_node(node),
m_op_name(std::string(m_node->name)) {
switch (m_node->op) {
// Unary OPs // Unary OPs
case GGML_OP_UNARY: case GGML_OP_UNARY:
case GGML_OP_RESHAPE: case GGML_OP_RESHAPE:
@ -16,22 +13,26 @@ GgmlOvDecoder::GgmlOvDecoder(struct ggml_tensor * node, struct ggml_cgraph * cgr
case GGML_OP_CPY: case GGML_OP_CPY:
case GGML_OP_RMS_NORM: case GGML_OP_RMS_NORM:
{ {
m_inputs.push_back(m_node->src[0]); inputs[node->src[0]->name] = node->src[0];
m_outputs.push_back(m_node); outputs[node->name] = node;
m_input_names.push_back(node->src[0]->name);
m_output_names.push_back(node->name);
break; break;
} }
// For view, input is m_node itself // For view, input is node itself
case GGML_OP_VIEW: case GGML_OP_VIEW:
{ {
m_inputs.push_back(m_node); inputs[node->src[0]->name] = node;
m_outputs.push_back(m_node); outputs[node->name] = node;
break; break;
} }
// SCALE // SCALE
case GGML_OP_SCALE: case GGML_OP_SCALE:
{ {
m_inputs.push_back(m_node->src[0]); inputs[node->src[0]->name] = node->src[0];
m_outputs.push_back(m_node); outputs[node->name] = node;
m_input_names.push_back(node->name);
m_output_names.push_back(node->name);
break; break;
} }
// OPs with 2 inputs // OPs with 2 inputs
@ -43,18 +44,25 @@ GgmlOvDecoder::GgmlOvDecoder(struct ggml_tensor * node, struct ggml_cgraph * cgr
case GGML_OP_GET_ROWS: case GGML_OP_GET_ROWS:
case GGML_OP_SOFT_MAX: case GGML_OP_SOFT_MAX:
{ {
m_inputs.push_back(m_node->src[0]); inputs[node->src[0]->name] = node->src[0];
m_inputs.push_back(m_node->src[1]); inputs[node->src[1]->name] = node->src[1];
m_outputs.push_back(m_node); outputs[node->name] = node;
m_input_names.push_back(node->src[0]->name);
m_input_names.push_back(node->src[1]->name);
m_output_names.push_back(node->name);
break; break;
} }
// OPs with 3 inputs: // OPs with 3 inputs:
case GGML_OP_ROPE: case GGML_OP_ROPE:
{ {
m_inputs.push_back(m_node->src[0]); inputs[node->src[0]->name] = node->src[0];
m_inputs.push_back(m_node->src[1]); inputs[node->src[1]->name] = node->src[1];
m_inputs.push_back(m_node->src[2]); // ??? inputs[node->src[2]->name] = node->src[2];
m_outputs.push_back(m_node); outputs[node->name] = node;
m_input_names.push_back(node->src[0]->name);
m_input_names.push_back(node->src[1]->name);
m_input_names.push_back(node->src[2]->name);
m_output_names.push_back(node->name);
break; break;
} }
default: default:
@ -62,13 +70,33 @@ GgmlOvDecoder::GgmlOvDecoder(struct ggml_tensor * node, struct ggml_cgraph * cgr
} }
} }
ov::PartialShape GgmlOvDecoder::get_input_shape(size_t index) const { GgmlOvDecoder::GgmlOvDecoder(struct ggml_tensor * node, struct ggml_cgraph * cgraph)
:m_cgraph(cgraph),
m_node(node),
m_op_name(m_node ? std::string(m_node->name) : "NONE_OP") {
m_inputs.clear();
m_outputs.clear();
m_input_names.clear();
m_output_names.clear();
// If first init
if (m_node) {
set_input_output(m_node, m_inputs, m_outputs);
} else {
for (int node_n = 0; node_n < m_cgraph->n_nodes; node_n++) {
auto cur_node = m_cgraph->nodes[node_n];
m_nodes.push_back(cur_node);
// Init model input and output
set_input_output(cur_node, m_inputs, m_outputs);
}
}
}
ov::PartialShape GgmlOvDecoder::get_input_shape(const std::string& name) const {
ov::PartialShape input_shape; ov::PartialShape input_shape;
// Use input_node->ne // Use input_node->ne
ggml_tensor * node = m_inputs[index]; ggml_tensor * node = m_inputs.at(name);
std::vector<size_t> shape; std::vector<size_t> shape;
// GGML_MAX_DIMS
// for (int i = 0; i < GGML_MAX_DIMS; ++i) {
for (int i = GGML_MAX_DIMS - 2; i >= 0 ; --i) { for (int i = GGML_MAX_DIMS - 2; i >= 0 ; --i) {
if (node->ne[i] == 0) { if (node->ne[i] == 0) {
return input_shape; return input_shape;
@ -79,10 +107,9 @@ ov::PartialShape GgmlOvDecoder::get_input_shape(size_t index) const {
return input_shape; return input_shape;
} }
ov::element::Type GgmlOvDecoder::get_input_type(size_t index) const { ov::element::Type GgmlOvDecoder::get_input_type(const std::string& name) const {
ov::element::Type type = ov::element::dynamic; ov::element::Type type = ov::element::dynamic;
// GGML_LOG_DEBUG("%d\n", m_inputs[index]->type); switch (m_inputs.at(name)->type) {
switch (m_inputs[index]->type) {
case GGML_TYPE_F32: case GGML_TYPE_F32:
type = ov::element::f32; type = ov::element::f32;
break; break;
@ -102,28 +129,24 @@ ov::element::Type GgmlOvDecoder::get_input_type(size_t index) const {
} }
size_t GgmlOvDecoder::get_input_size() const { size_t GgmlOvDecoder::get_input_size() const {
return m_inputs.size(); return m_input_names.size();
}
bool GgmlOvDecoder::is_graph_input(size_t index) const {
if (m_inputs[index]->flags & GGML_TENSOR_FLAG_INPUT ) {
return true;
}
return false;
} }
std::string& GgmlOvDecoder::get_input_name(size_t index) const { std::string& GgmlOvDecoder::get_input_name(size_t index) const {
m_name = std::string(m_inputs[index]->name); m_name = m_input_names[index];
return m_name; return m_name;
} }
ov::PartialShape GgmlOvDecoder::get_output_shape(size_t index) const { std::vector<std::string> GgmlOvDecoder::get_input_names() const {
return m_input_names;
}
ov::PartialShape GgmlOvDecoder::get_output_shape(const std::string& name) const {
ov::PartialShape output_shape; ov::PartialShape output_shape;
// Use input_node->ne // Use input_node->ne
ggml_tensor * node = m_outputs[index]; ggml_tensor * node = m_outputs.at(name);
std::vector<size_t> shape; std::vector<size_t> shape;
// GGML_MAX_DIMS
// for (int i = 0; i < GGML_MAX_DIMS; ++i) {
for (int i = GGML_MAX_DIMS - 2; i >= 0 ; --i) { for (int i = GGML_MAX_DIMS - 2; i >= 0 ; --i) {
if (node->ne[i] == 0 ) { if (node->ne[i] == 0 ) {
// empty if any dimension has no elements // empty if any dimension has no elements
@ -135,10 +158,10 @@ ov::PartialShape GgmlOvDecoder::get_output_shape(size_t index) const {
return output_shape; return output_shape;
} }
ov::element::Type GgmlOvDecoder::get_output_type(size_t index) const { ov::element::Type GgmlOvDecoder::get_output_type(const std::string& name) const {
// TODO: Change to Output // TODO: Change to Output
ov::element::Type type = ov::element::dynamic; ov::element::Type type = ov::element::dynamic;
switch (m_outputs[index]->type) { switch (m_outputs.at(name)->type) {
case GGML_TYPE_F32: case GGML_TYPE_F32:
type = ov::element::f32; type = ov::element::f32;
break; break;
@ -157,30 +180,31 @@ ov::element::Type GgmlOvDecoder::get_output_type(size_t index) const {
return type; return type;
} }
bool GgmlOvDecoder::is_graph_output(size_t index) const { int32_t* GgmlOvDecoder::get_output_op_params(const std::string& name) const{
if (m_outputs[index]->flags & GGML_TENSOR_FLAG_OUTPUT) { return m_outputs.at(name)->op_params;
return true;
}
return false;
}
int32_t* GgmlOvDecoder::get_output_op_params(size_t index) const{
return m_outputs[index]->op_params;
}
size_t GgmlOvDecoder::get_output_size() const {
return m_outputs.size();
} }
std::string& GgmlOvDecoder::get_output_name(size_t index) const { std::string& GgmlOvDecoder::get_output_name(size_t index) const {
m_name = std::string(m_outputs[index]->name); m_name = std::string(m_output_names[index]);
return m_name; return m_name;
} }
std::vector<std::string> GgmlOvDecoder::get_output_names() const {
return m_output_names;
}
const std::string& GgmlOvDecoder::get_op_name() const { const std::string& GgmlOvDecoder::get_op_name() const {
return m_op_name; return m_op_name;
} }
void GgmlOvDecoder::visit_subgraph(std::function<void(std::shared_ptr<GgmlDecoder>)> node_visitor) const {
for (const auto& node : m_nodes) {
auto decoder = std::make_shared<GgmlOvDecoder>(node, m_cgraph);
// m_decoders.push_back(decoder);
node_visitor(decoder);
}
}
const std::string& GgmlOvDecoder::get_op_type() const { const std::string& GgmlOvDecoder::get_op_type() const {
static const std::map<ggml_op, std::string> opTypeMap = { static const std::map<ggml_op, std::string> opTypeMap = {
{GGML_OP_ACC, "GGML_OP_ACC"}, {GGML_OP_ACC, "GGML_OP_ACC"},

44
ggml/src/ggml-openvino/ggml-decoder.h
View File

@ -6,6 +6,7 @@
class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
public: public:
using ov::frontend::ggml::GgmlDecoder::GgmlDecoder; using ov::frontend::ggml::GgmlDecoder::GgmlDecoder;
GgmlOvDecoder(struct ggml_tensor * node, struct ggml_cgraph * cgraph); GgmlOvDecoder(struct ggml_tensor * node, struct ggml_cgraph * cgraph);
virtual ov::Any get_attribute(const std::string& name) const override { virtual ov::Any get_attribute(const std::string& name) const override {
@ -13,9 +14,9 @@ public:
GGML_UNUSED(name); GGML_UNUSED(name);
} }
virtual ov::PartialShape get_input_shape(size_t index) const override; virtual ov::PartialShape get_input_shape(const std::string& name) const override;
virtual ov::element::Type get_input_type(size_t index) const override; virtual ov::element::Type get_input_type(const std::string& name) const override;
virtual size_t get_input_size() const override; virtual size_t get_input_size() const override;
@ -29,19 +30,15 @@ public:
GGML_UNUSED(producer_output_port_index); GGML_UNUSED(producer_output_port_index);
} }
virtual bool is_graph_input(size_t index) const override;
virtual std::string& get_input_name(size_t index) const override; virtual std::string& get_input_name(size_t index) const override;
virtual ov::PartialShape get_output_shape(size_t index) const override; virtual std::vector<std::string> get_input_names() const override;
virtual ov::element::Type get_output_type(size_t index) const override; virtual ov::PartialShape get_output_shape(const std::string& name) const override;
virtual size_t get_output_size() const override; virtual ov::element::Type get_output_type(const std::string& name) const override;
virtual bool is_graph_output(size_t index) const override; virtual int32_t* get_output_op_params(const std::string& name) const override;
virtual int32_t* get_output_op_params(size_t index) const override;
virtual std::string& get_output_name(size_t index) const override; virtual std::string& get_output_name(size_t index) const override;
@ -55,24 +52,27 @@ public:
virtual const std::string& get_op_name() const override; virtual const std::string& get_op_name() const override;
const ggml_tensor* get_input_ggml_tensor(size_t index) const { virtual void visit_subgraph(std::function<void(std::shared_ptr<GgmlDecoder>)> node_visitor) const override;
return m_inputs[index];
const ggml_tensor* get_input_ggml_tensor(std::string& name) const {
return m_inputs.at(name);
} }
const ggml_tensor* get_output_ggml_tensor(size_t index) const { const ggml_tensor* get_output_ggml_tensor(std::string& name) const {
return m_outputs[index]; return m_outputs.at(name);
} }
// virtual const std::vector<size_t>& outputs() const override;
// virtual size_t output(size_t index) const override;
private: private:
size_t m_index; void set_input_output(ggml_tensor* node, std::map<std::string, ggml_tensor *>& inputs, std::map<std::string, ggml_tensor *>& outputs);
struct ggml_cgraph * m_cgraph; struct ggml_cgraph * m_cgraph;
std::vector<ggml_tensor *> m_inputs; std::map<std::string, ggml_tensor *> m_inputs;
std::vector<ggml_tensor *> m_outputs; std::vector<std::string> m_input_names;
ggml_tensor * m_node; std::map<std::string, ggml_tensor *> m_outputs;
std::vector<std::string> m_output_names;
ggml_tensor* m_node;
std::vector<ggml_tensor *> m_nodes;
std::vector<std::shared_ptr<GgmlOvDecoder>> m_decoders;
const std::string m_op_name; const std::string m_op_name;
mutable std::string m_name; mutable std::string m_name;
}; };

95
ggml/src/ggml-openvino/ggml-graph-iterator.cpp
View File

@ -1,95 +0,0 @@
#include "ggml-graph-iterator.h"
#include <ggml.h>
#include <ggml-impl.h>
namespace ov {
namespace frontend {
namespace tensorflow {
namespace ggml {
GgmlOvGraphIterator::GgmlOvGraphIterator(struct ggml_cgraph * cgraph)
:m_cgraph(cgraph) {
initialize_decoders();
#ifdef GGML_OPENVINO_DEBUG
dump_graph_iterator();
#endif
}
void GgmlOvGraphIterator::initialize_decoders() {
auto nodes_size = m_cgraph->n_nodes;
// Initialize decoder for each node
// m_decoders.resize(static_cast<size_t>(nodes_size));
for (int i = 0; i < nodes_size; ++i) {
auto decoder = std::make_shared<GgmlOvDecoder>(m_cgraph->nodes[i], m_cgraph);
m_decoders.push_back(decoder);
for (size_t inp = 0; inp < decoder->get_input_size(); ++inp) {
// Skip duplicate input name
if (std::find(m_input_names.begin(), m_input_names.end(), decoder->get_input_name(inp)) == m_input_names.end()) {
m_input_names.push_back(decoder->get_input_name(inp));
}
}
for (size_t inp = 0; inp < decoder->get_output_size(); ++inp) {
// Skip duplicate output name
auto output_name = decoder->get_output_name(inp);
if (std::find(m_output_names.begin(), m_output_names.end(), output_name) == m_output_names.end()) {
m_output_names.push_back(decoder->get_output_name(inp));
}
}
}
}
void GgmlOvGraphIterator::reset() {
node_index = 0;
}
size_t GgmlOvGraphIterator::size() const {
return m_decoders.size();
}
void GgmlOvGraphIterator::next() {
node_index++;
}
bool GgmlOvGraphIterator::is_end() const {
return node_index >= m_decoders.size();
}
std::shared_ptr<DecoderBase> GgmlOvGraphIterator::get_decoder() const {
return m_decoders[node_index];
}
std::vector<std::string> GgmlOvGraphIterator::get_input_names() const {
return m_input_names;
}
std::vector<std::string> GgmlOvGraphIterator::get_output_names() const {
return m_output_names;
}
void GgmlOvGraphIterator::dump_graph_iterator() const {
for (size_t i = 0; i < m_decoders.size(); ++i) {
GGML_LOG_INFO("\nOP %zu: %s\n", i, m_decoders[i]->get_op_name().c_str());
for (size_t inp = 0; inp < m_decoders[i]->get_input_size(); ++inp) {
ov::PartialShape pshape = std::dynamic_pointer_cast<GgmlOvDecoder>(m_decoders[i])->get_input_shape(inp);
ov::element::Type ptype = std::dynamic_pointer_cast<GgmlOvDecoder>(m_decoders[i])->get_input_type(inp);
GGML_LOG_INFO("- Input name: %s\n", std::dynamic_pointer_cast<GgmlOvDecoder>(m_decoders[i])->get_input_name(inp).c_str());
GGML_LOG_INFO(" Input shape: %s\n", pshape.to_string().c_str());
GGML_LOG_INFO(" Input type: %s\n", ptype.to_string().c_str());
}
for (size_t outp = 0; outp < std::dynamic_pointer_cast<GgmlOvDecoder>(m_decoders[i])->get_output_size(); ++outp) {
ov::PartialShape pshape = std::dynamic_pointer_cast<GgmlOvDecoder>(m_decoders[i])->get_output_shape(outp);
ov::element::Type ptype = std::dynamic_pointer_cast<GgmlOvDecoder>(m_decoders[i])->get_output_type(outp);
GGML_LOG_INFO("- Output name: %s\n", std::dynamic_pointer_cast<GgmlOvDecoder>(m_decoders[i])->get_output_name(outp).c_str());
GGML_LOG_INFO(" Output shape: %s\n", pshape.to_string().c_str());
GGML_LOG_INFO(" Output type: %s\n", ptype.to_string().c_str());
}
}
}
}
} // namespace ggml
} // namespace frontend
} // namespace ov

61
ggml/src/ggml-openvino/ggml-graph-iterator.h
View File

@ -1,61 +0,0 @@
#pragma once
#include "graph_iterator.h"
#include "ggml-decoder.h"
#include <ggml-impl.h>
// To remove tensorflow
namespace ov {
namespace frontend {
namespace tensorflow {
namespace ggml {
class GgmlOvGraphIterator : public GgmlGraphIterator {
protected:
void initialize_decoders();
public:
using Ptr = std::shared_ptr<GgmlOvGraphIterator>;
GgmlOvGraphIterator(struct ggml_cgraph * cgraph);
/// \brief Get a number of operation nodes in the sgraph
virtual size_t size() const override;
/// \brief Set iterator to the start position
virtual void reset() override;
/// \brief Move to the next node in the graph
virtual void next() override;
/// \brief Returns true if iterator goes out of the range of available nodes
virtual bool is_end() const override;
/// \brief Return a pointer to a decoder of the current node
virtual std::shared_ptr<DecoderBase> get_decoder() const override;
virtual std::shared_ptr<GraphIterator> get_body_graph_iterator(const std::string& func_name) const override {
return nullptr;
GGML_UNUSED(func_name);
}
/// \brief Returns a vector of input names in the original order
virtual std::vector<std::string> get_input_names() const override;
/// \brief Returns a vector of output names in the original order
virtual std::vector<std::string> get_output_names() const override;
virtual void dump_graph_iterator() const;
private:
struct ggml_cgraph * m_cgraph;
size_t node_index = 0;
std::vector<std::shared_ptr<DecoderBase>> m_decoders;
std::vector<std::string> m_input_names;
std::vector<std::string> m_output_names;
};
}
} // namespace ggml
} // namespace frontend
} // namespace ov

43
ggml/src/ggml-openvino/graph_iterator.h
View File

@ -1,43 +0,0 @@
#pragma once
#include "openvino/frontend/graph_iterator.hpp"
namespace ov {
namespace frontend {
namespace tensorflow { // To be Removed
namespace ggml {
// TODO: Directly include from openvino
class GgmlGraphIterator : public GraphIterator {
public:
virtual size_t size() const = 0;
virtual void reset() = 0;
virtual void next() = 0;
virtual bool is_end() const = 0;
virtual std::shared_ptr<DecoderBase> get_decoder() const = 0;
virtual std::vector<std::string> get_input_names() const = 0;
virtual std::vector<std::string> get_output_names() const = 0;
virtual std::shared_ptr<GraphIterator> get_body_graph_iterator(const std::string& func_name) const = 0;
virtual std::map<std::string, std::string> get_input_names_map() const {
return {};
}
virtual std::map<std::string, std::string> get_output_names_map() const {
return {};
}
};
}
} // namespace ggml
} // namespace frontend
} // namespace ov

74
ggml/src/ggml-openvino/utils.cpp
View File

@ -1,49 +1,40 @@
#include "utils.h" #include "utils.h"
#include "ggml-impl.h"
#include "ggml-backend-impl.h" #include "ggml-backend-impl.h"
#include <openvino/frontend/manager.hpp> #include <openvino/frontend/manager.hpp>
#include <openvino/openvino.hpp> #include <openvino/openvino.hpp>
using ov::frontend::tensorflow::ggml::GgmlOvGraphIterator; using ov::frontend::ggml::GgmlDecoder;
std::shared_ptr<GgmlOvGraphIterator> get_ggml_graph_iterator(struct ggml_cgraph * cgraph) { std::shared_ptr<GgmlOvDecoder> get_ggml_decoder(struct ggml_cgraph * cgraph) {
return std::make_shared<GgmlOvGraphIterator>(cgraph); return std::make_shared<GgmlOvDecoder>(nullptr, cgraph);
} }
std::map<std::string, ov::Tensor> get_ggml_graph_input_tensors(std::shared_ptr<GgmlOvGraphIterator> ggml_graph_iterator) { std::map<std::string, ov::Tensor> get_ggml_graph_input_tensors(std::shared_ptr<GgmlOvDecoder> ggml_decoder) {
std::map<std::string, ov::Tensor> input_tensors; std::map<std::string, ov::Tensor> input_tensors;
auto input_names = ggml_graph_iterator->get_input_names(); auto input_names = ggml_decoder->get_input_names();
ggml_graph_iterator->reset(); for (size_t inp = 0; inp < input_names.size(); ++inp) {
for (; !ggml_graph_iterator->is_end(); ggml_graph_iterator->next()) { auto name = input_names[inp];
auto decoder = std::dynamic_pointer_cast<GgmlOvDecoder>(ggml_graph_iterator->get_decoder()); auto input_data = ggml_decoder->get_input_ggml_tensor(name)->data;
for (size_t inp = 0; inp < decoder->get_input_size(); ++inp) { #ifdef GGML_OPENVINO_DEBUG
if (std::find(input_names.begin(), input_names.end(), decoder->get_input_name(inp)) != input_names.end()) { printf("Subgraph input %d: %g\n", inp, *(double*)(input_data));
auto input_data = decoder->get_input_ggml_tensor(inp)->data; #endif
#ifdef GGML_OPENVINO_DEBUG ov::Tensor input_tensor = ov::Tensor(ggml_decoder->get_input_type(name), ggml_decoder->get_input_shape(name).to_shape(), input_data);
printf("Subgraph input %d: %g\n", inp, *(double*)(input_data)); input_tensors[name] = input_tensor;
#endif
ov::Tensor input_tensor = ov::Tensor(decoder->get_input_type(inp), decoder->get_input_shape(inp).to_shape(), input_data);
input_tensors[decoder->get_input_name(inp)] = input_tensor;
}
}
} }
return input_tensors; return input_tensors;
} }
std::map<std::string, void*> get_ggml_graph_output_dst(std::shared_ptr<GgmlOvGraphIterator> ggml_graph_iterator) { std::map<std::string, void*> get_ggml_graph_output_dst(std::shared_ptr<GgmlOvDecoder> ggml_decoder) {
std::map<std::string, void*> output_tensors; std::map<std::string, void*> output_tensors;
auto output_names = ggml_graph_iterator->get_output_names(); auto output_names = ggml_decoder->get_output_names();
ggml_graph_iterator->reset(); for (size_t inp = 0; inp < output_names.size(); ++inp) {
for (; !ggml_graph_iterator->is_end(); ggml_graph_iterator->next()) { auto name = output_names[inp];
auto decoder = std::dynamic_pointer_cast<GgmlOvDecoder>(ggml_graph_iterator->get_decoder()); auto output_data = ggml_decoder->get_output_ggml_tensor(name)->data;
for (size_t inp = 0; inp < decoder->get_output_size(); ++inp) { #ifdef GGML_OPENVINO_DEBUG
if (std::find(output_names.begin(), output_names.end(), decoder->get_output_name(inp)) != output_names.end()) { printf("Output %d: %g\n", inp, *(double*)(output_data));
auto output_data = decoder->get_output_ggml_tensor(inp)->data; #endif
#ifdef GGML_OPENVINO_DEBUG output_tensors[name] = output_data;
printf("Output %d: %g\n", inp, *(double*)(output_data));
#endif
output_tensors[decoder->get_output_name(inp)] = output_data;
}
}
} }
return output_tensors; return output_tensors;
} }
@ -74,12 +65,10 @@ enum ggml_status openvino_frontend_compute(ggml_backend_t backend, struct ggml_c
GGML_LOG_INFO("GGML FrontEnd is initialized \n"); GGML_LOG_INFO("GGML FrontEnd is initialized \n");
#endif #endif
} }
auto ggml_decoder = get_ggml_decoder(cgraph);
auto ggml_graph_iterator = get_ggml_graph_iterator(cgraph); std::shared_ptr<ov::frontend::DecoderBase> graph_decoder = ggml_decoder;
std::shared_ptr<ov::frontend::tensorflow::GraphIterator> graph_iterator = ggml_graph_iterator;
// Load GraphIterator -> InputModel // Load GraphIterator -> InputModel
ov::frontend::InputModel::Ptr input_model = front_end->load(graph_iterator); ov::frontend::InputModel::Ptr input_model = front_end->load(graph_decoder);
if (!input_model) { if (!input_model) {
GGML_LOG_ERROR("Input Model is not loaded \n"); GGML_LOG_ERROR("Input Model is not loaded \n");
return GGML_STATUS_FAILED; return GGML_STATUS_FAILED;
@ -106,8 +95,8 @@ enum ggml_status openvino_frontend_compute(ggml_backend_t backend, struct ggml_c
ov::InferRequest infer_request = compiled_model.create_infer_request(); ov::InferRequest infer_request = compiled_model.create_infer_request();
// Get input tensor // Get input tensor
auto input_names = ggml_graph_iterator->get_input_names(); auto input_names = ggml_decoder->get_input_names();
auto input_tensors = get_ggml_graph_input_tensors(ggml_graph_iterator); auto input_tensors = get_ggml_graph_input_tensors(ggml_decoder);
// Set input tensor // Set input tensor
for (size_t i = 0; i < input_names.size(); i++) { for (size_t i = 0; i < input_names.size(); i++) {
@ -117,11 +106,14 @@ enum ggml_status openvino_frontend_compute(ggml_backend_t backend, struct ggml_c
infer_request.infer(); infer_request.infer();
// Set dst data for outputs // Set dst data for outputs
auto output_names = ggml_graph_iterator->get_output_names(); auto output_names = ggml_decoder->get_output_names();
auto output_tensors = get_ggml_graph_output_dst(ggml_graph_iterator); auto output_tensors = get_ggml_graph_output_dst(ggml_decoder);
for (size_t i = 0; i < output_names.size(); i++) { for (size_t i = 0; i < output_names.size(); i++) {
auto output_tensor = infer_request.get_output_tensor(i); auto output_tensor = infer_request.get_output_tensor(i);
std::memcpy(output_tensors[output_names[i]], output_tensor.data(), output_tensor.get_byte_size()); std::memcpy(output_tensors[output_names[i]], output_tensor.data(), output_tensor.get_byte_size());
#ifdef GGML_OPENVINO_DEBUG
printf("Output %s after: %g\n", output_names[i], *(double*)(output_tensor.data()));
#endif
} }
return GGML_STATUS_SUCCESS; return GGML_STATUS_SUCCESS;

4
ggml/src/ggml-openvino/utils.h
View File

@ -1,6 +1,4 @@
#include "ggml-graph-iterator.h" #include "ggml-decoder.h"
#include "ggml-backend-impl.h" #include "ggml-backend-impl.h"
std::shared_ptr<ov::frontend::tensorflow::ggml::GgmlOvGraphIterator> get_ggml_graph_iterator(struct ggml_cgraph * cgraph);
enum ggml_status openvino_frontend_compute (ggml_backend_t backend, struct ggml_cgraph * cgraph); enum ggml_status openvino_frontend_compute (ggml_backend_t backend, struct ggml_cgraph * cgraph);