happyz
/
llama.cpp
mirror of https://github.com/ggerganov/llama.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Change to implementation following pytorch frontend

This commit is contained in:
yumengbo 2024-12-13 07:28:28 +08:00 committed by Mustafa Cavus
parent e95f29cbc0
commit b100f89bad
8 changed files with 143 additions and 330 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
ggml/src/ggml-openvino/decoder.h
View File

@ -12,9 +12,9 @@ class GgmlDecoder : public DecoderBase {
public:
virtual ov::Any get_attribute(const std::string& name) const = 0;
virtual PartialShape get_input_shape(size_t index) const = 0;
virtual PartialShape get_input_shape(const std::string& name) const = 0;
virtual element::Type get_input_type(size_t index) const = 0;
virtual element::Type get_input_type(const std::string& name) const = 0;
virtual size_t get_input_size() const = 0;
@ -23,19 +23,15 @@ public:
std::string& producer_output_port_name,
size_t& producer_output_port_index) const = 0;
virtual bool is_graph_input(size_t index) const = 0;
virtual std::string& get_input_name(size_t index) const = 0;
virtual PartialShape get_output_shape(size_t index) const = 0;
virtual std::vector<std::string> get_input_names() const = 0;
virtual element::Type get_output_type(size_t index) const = 0;
virtual PartialShape get_output_shape(const std::string& name) const = 0;
virtual size_t get_output_size() const = 0;
virtual element::Type get_output_type(const std::string& name) const = 0;
virtual bool is_graph_output(size_t index) const = 0;
virtual int32_t* get_output_op_params(size_t index) const = 0;
virtual int32_t* get_output_op_params(const std::string& name) const = 0;
virtual std::string& get_output_name(size_t index) const = 0;
@ -49,6 +45,8 @@ public:
virtual const std::string& get_op_name() const = 0;
virtual void visit_subgraph(std::function<void(std::shared_ptr<GgmlDecoder>)> node_visitor) const = 0;
// virtual const std::vector<size_t>& outputs() const = 0;
// virtual size_t output(size_t index) const = 0;

134
ggml/src/ggml-openvino/ggml-decoder.cpp
View File

@ -2,11 +2,8 @@
#include <ggml.h>
#include <ggml-impl.h>
GgmlOvDecoder::GgmlOvDecoder(struct ggml_tensor * node, struct ggml_cgraph * cgraph)
:m_cgraph(cgraph),
m_node(node),
m_op_name(std::string(m_node->name)) {
switch (m_node->op) {
void GgmlOvDecoder::set_input_output(ggml_tensor* node, std::map<std::string, ggml_tensor *>& inputs, std::map<std::string, ggml_tensor *>& outputs) {
switch (node->op) {
// Unary OPs
case GGML_OP_UNARY:
case GGML_OP_RESHAPE:
@ -16,22 +13,26 @@ GgmlOvDecoder::GgmlOvDecoder(struct ggml_tensor * node, struct ggml_cgraph * cgr
case GGML_OP_CPY:
case GGML_OP_RMS_NORM:
{
m_inputs.push_back(m_node->src[0]);
m_outputs.push_back(m_node);
inputs[node->src[0]->name] = node->src[0];
outputs[node->name] = node;
m_input_names.push_back(node->src[0]->name);
m_output_names.push_back(node->name);
break;
}
// For view, input is m_node itself
// For view, input is node itself
case GGML_OP_VIEW:
{
m_inputs.push_back(m_node);
m_outputs.push_back(m_node);
inputs[node->src[0]->name] = node;
outputs[node->name] = node;
break;
}
// SCALE
case GGML_OP_SCALE:
{
m_inputs.push_back(m_node->src[0]);
m_outputs.push_back(m_node);
inputs[node->src[0]->name] = node->src[0];
outputs[node->name] = node;
m_input_names.push_back(node->name);
m_output_names.push_back(node->name);
break;
}
// OPs with 2 inputs
@ -43,18 +44,25 @@ GgmlOvDecoder::GgmlOvDecoder(struct ggml_tensor * node, struct ggml_cgraph * cgr
case GGML_OP_GET_ROWS:
case GGML_OP_SOFT_MAX:
{
m_inputs.push_back(m_node->src[0]);
m_inputs.push_back(m_node->src[1]);
m_outputs.push_back(m_node);
inputs[node->src[0]->name] = node->src[0];
inputs[node->src[1]->name] = node->src[1];
outputs[node->name] = node;
m_input_names.push_back(node->src[0]->name);
m_input_names.push_back(node->src[1]->name);
m_output_names.push_back(node->name);
break;
}
// OPs with 3 inputs:
case GGML_OP_ROPE:
{
m_inputs.push_back(m_node->src[0]);
m_inputs.push_back(m_node->src[1]);
m_inputs.push_back(m_node->src[2]); // ???
m_outputs.push_back(m_node);
inputs[node->src[0]->name] = node->src[0];
inputs[node->src[1]->name] = node->src[1];
inputs[node->src[2]->name] = node->src[2];
outputs[node->name] = node;
m_input_names.push_back(node->src[0]->name);
m_input_names.push_back(node->src[1]->name);
m_input_names.push_back(node->src[2]->name);
m_output_names.push_back(node->name);
break;
}
default:
@ -62,13 +70,33 @@ GgmlOvDecoder::GgmlOvDecoder(struct ggml_tensor * node, struct ggml_cgraph * cgr
}
}
ov::PartialShape GgmlOvDecoder::get_input_shape(size_t index) const {
GgmlOvDecoder::GgmlOvDecoder(struct ggml_tensor * node, struct ggml_cgraph * cgraph)
:m_cgraph(cgraph),
m_node(node),
m_op_name(m_node ? std::string(m_node->name) : "NONE_OP") {
m_inputs.clear();
m_outputs.clear();
m_input_names.clear();
m_output_names.clear();
// If first init
if (m_node) {
set_input_output(m_node, m_inputs, m_outputs);
} else {
for (int node_n = 0; node_n < m_cgraph->n_nodes; node_n++) {
auto cur_node = m_cgraph->nodes[node_n];
m_nodes.push_back(cur_node);
// Init model input and output
set_input_output(cur_node, m_inputs, m_outputs);
}
}
}
ov::PartialShape GgmlOvDecoder::get_input_shape(const std::string& name) const {
ov::PartialShape input_shape;
// Use input_node->ne
ggml_tensor * node = m_inputs[index];
ggml_tensor * node = m_inputs.at(name);
std::vector<size_t> shape;
// GGML_MAX_DIMS
// for (int i = 0; i < GGML_MAX_DIMS; ++i) {
for (int i = GGML_MAX_DIMS - 2; i >= 0 ; --i) {
if (node->ne[i] == 0) {
return input_shape;
@ -79,10 +107,9 @@ ov::PartialShape GgmlOvDecoder::get_input_shape(size_t index) const {
return input_shape;
}
ov::element::Type GgmlOvDecoder::get_input_type(size_t index) const {
ov::element::Type GgmlOvDecoder::get_input_type(const std::string& name) const {
ov::element::Type type = ov::element::dynamic;
// GGML_LOG_DEBUG("%d\n", m_inputs[index]->type);
switch (m_inputs[index]->type) {
switch (m_inputs.at(name)->type) {
case GGML_TYPE_F32:
type = ov::element::f32;
break;
@ -102,28 +129,24 @@ ov::element::Type GgmlOvDecoder::get_input_type(size_t index) const {
}
size_t GgmlOvDecoder::get_input_size() const {
return m_inputs.size();
}
bool GgmlOvDecoder::is_graph_input(size_t index) const {
if (m_inputs[index]->flags & GGML_TENSOR_FLAG_INPUT ) {
return true;
}
return false;
return m_input_names.size();
}
std::string& GgmlOvDecoder::get_input_name(size_t index) const {
m_name = std::string(m_inputs[index]->name);
m_name = m_input_names[index];
return m_name;
}
ov::PartialShape GgmlOvDecoder::get_output_shape(size_t index) const {
std::vector<std::string> GgmlOvDecoder::get_input_names() const {
return m_input_names;
}
ov::PartialShape GgmlOvDecoder::get_output_shape(const std::string& name) const {
ov::PartialShape output_shape;
// Use input_node->ne
ggml_tensor * node = m_outputs[index];
ggml_tensor * node = m_outputs.at(name);
std::vector<size_t> shape;
// GGML_MAX_DIMS
// for (int i = 0; i < GGML_MAX_DIMS; ++i) {
for (int i = GGML_MAX_DIMS - 2; i >= 0 ; --i) {
if (node->ne[i] == 0 ) {
// empty if any dimension has no elements
@ -135,10 +158,10 @@ ov::PartialShape GgmlOvDecoder::get_output_shape(size_t index) const {
return output_shape;
}
ov::element::Type GgmlOvDecoder::get_output_type(size_t index) const {
ov::element::Type GgmlOvDecoder::get_output_type(const std::string& name) const {
// TODO: Change to Output
ov::element::Type type = ov::element::dynamic;
switch (m_outputs[index]->type) {
switch (m_outputs.at(name)->type) {
case GGML_TYPE_F32:
type = ov::element::f32;
break;
@ -157,30 +180,31 @@ ov::element::Type GgmlOvDecoder::get_output_type(size_t index) const {
return type;
}
bool GgmlOvDecoder::is_graph_output(size_t index) const {
if (m_outputs[index]->flags & GGML_TENSOR_FLAG_OUTPUT) {
return true;
}
return false;
}
int32_t* GgmlOvDecoder::get_output_op_params(size_t index) const{
return m_outputs[index]->op_params;
}
size_t GgmlOvDecoder::get_output_size() const {
return m_outputs.size();
int32_t* GgmlOvDecoder::get_output_op_params(const std::string& name) const{
return m_outputs.at(name)->op_params;
}
std::string& GgmlOvDecoder::get_output_name(size_t index) const {
m_name = std::string(m_outputs[index]->name);
m_name = std::string(m_output_names[index]);
return m_name;
}
std::vector<std::string> GgmlOvDecoder::get_output_names() const {
return m_output_names;
}
const std::string& GgmlOvDecoder::get_op_name() const {
return m_op_name;
}
void GgmlOvDecoder::visit_subgraph(std::function<void(std::shared_ptr<GgmlDecoder>)> node_visitor) const {
for (const auto& node : m_nodes) {
auto decoder = std::make_shared<GgmlOvDecoder>(node, m_cgraph);
// m_decoders.push_back(decoder);
node_visitor(decoder);
}
}
const std::string& GgmlOvDecoder::get_op_type() const {
static const std::map<ggml_op, std::string> opTypeMap = {
{GGML_OP_ACC, "GGML_OP_ACC"},

44
ggml/src/ggml-openvino/ggml-decoder.h
View File

@ -6,6 +6,7 @@
class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
public:
using ov::frontend::ggml::GgmlDecoder::GgmlDecoder;
GgmlOvDecoder(struct ggml_tensor * node, struct ggml_cgraph * cgraph);
virtual ov::Any get_attribute(const std::string& name) const override {
@ -13,9 +14,9 @@ public:
GGML_UNUSED(name);
}
virtual ov::PartialShape get_input_shape(size_t index) const override;
virtual ov::PartialShape get_input_shape(const std::string& name) const override;
virtual ov::element::Type get_input_type(size_t index) const override;
virtual ov::element::Type get_input_type(const std::string& name) const override;
virtual size_t get_input_size() const override;
@ -29,19 +30,15 @@ public:
GGML_UNUSED(producer_output_port_index);
}
virtual bool is_graph_input(size_t index) const override;
virtual std::string& get_input_name(size_t index) const override;
virtual ov::PartialShape get_output_shape(size_t index) const override;
virtual std::vector<std::string> get_input_names() const override;
virtual ov::element::Type get_output_type(size_t index) const override;
virtual ov::PartialShape get_output_shape(const std::string& name) const override;
virtual size_t get_output_size() const override;
virtual ov::element::Type get_output_type(const std::string& name) const override;
virtual bool is_graph_output(size_t index) const override;
virtual int32_t* get_output_op_params(size_t index) const override;
virtual int32_t* get_output_op_params(const std::string& name) const override;
virtual std::string& get_output_name(size_t index) const override;
@ -55,24 +52,27 @@ public:
virtual const std::string& get_op_name() const override;
const ggml_tensor* get_input_ggml_tensor(size_t index) const {
return m_inputs[index];
virtual void visit_subgraph(std::function<void(std::shared_ptr<GgmlDecoder>)> node_visitor) const override;
const ggml_tensor* get_input_ggml_tensor(std::string& name) const {
return m_inputs.at(name);
}
const ggml_tensor* get_output_ggml_tensor(size_t index) const {
return m_outputs[index];
const ggml_tensor* get_output_ggml_tensor(std::string& name) const {
return m_outputs.at(name);
}
// virtual const std::vector<size_t>& outputs() const override;
// virtual size_t output(size_t index) const override;
private:
size_t m_index;
void set_input_output(ggml_tensor* node, std::map<std::string, ggml_tensor *>& inputs, std::map<std::string, ggml_tensor *>& outputs);
struct ggml_cgraph * m_cgraph;
std::vector<ggml_tensor *> m_inputs;
std::vector<ggml_tensor *> m_outputs;
ggml_tensor * m_node;
std::map<std::string, ggml_tensor *> m_inputs;
std::vector<std::string> m_input_names;
std::map<std::string, ggml_tensor *> m_outputs;
std::vector<std::string> m_output_names;
ggml_tensor* m_node;
std::vector<ggml_tensor *> m_nodes;
std::vector<std::shared_ptr<GgmlOvDecoder>> m_decoders;
const std::string m_op_name;
mutable std::string m_name;
};

95
ggml/src/ggml-openvino/ggml-graph-iterator.cpp
View File

@ -1,95 +0,0 @@
#include "ggml-graph-iterator.h"
#include <ggml.h>
#include <ggml-impl.h>
namespace ov {
namespace frontend {
namespace tensorflow {
namespace ggml {
GgmlOvGraphIterator::GgmlOvGraphIterator(struct ggml_cgraph * cgraph)
:m_cgraph(cgraph) {
initialize_decoders();
#ifdef GGML_OPENVINO_DEBUG
dump_graph_iterator();
#endif
}
void GgmlOvGraphIterator::initialize_decoders() {
auto nodes_size = m_cgraph->n_nodes;
// Initialize decoder for each node
// m_decoders.resize(static_cast<size_t>(nodes_size));
for (int i = 0; i < nodes_size; ++i) {
auto decoder = std::make_shared<GgmlOvDecoder>(m_cgraph->nodes[i], m_cgraph);
m_decoders.push_back(decoder);
for (size_t inp = 0; inp < decoder->get_input_size(); ++inp) {
// Skip duplicate input name
if (std::find(m_input_names.begin(), m_input_names.end(), decoder->get_input_name(inp)) == m_input_names.end()) {
m_input_names.push_back(decoder->get_input_name(inp));
}
}
for (size_t inp = 0; inp < decoder->get_output_size(); ++inp) {
// Skip duplicate output name
auto output_name = decoder->get_output_name(inp);
if (std::find(m_output_names.begin(), m_output_names.end(), output_name) == m_output_names.end()) {
m_output_names.push_back(decoder->get_output_name(inp));
}
}
}
}
void GgmlOvGraphIterator::reset() {
node_index = 0;
}
size_t GgmlOvGraphIterator::size() const {
return m_decoders.size();
}
void GgmlOvGraphIterator::next() {
node_index++;
}
bool GgmlOvGraphIterator::is_end() const {
return node_index >= m_decoders.size();
}
std::shared_ptr<DecoderBase> GgmlOvGraphIterator::get_decoder() const {
return m_decoders[node_index];
}
std::vector<std::string> GgmlOvGraphIterator::get_input_names() const {
return m_input_names;
}
std::vector<std::string> GgmlOvGraphIterator::get_output_names() const {
return m_output_names;
}
void GgmlOvGraphIterator::dump_graph_iterator() const {
for (size_t i = 0; i < m_decoders.size(); ++i) {
GGML_LOG_INFO("\nOP %zu: %s\n", i, m_decoders[i]->get_op_name().c_str());
for (size_t inp = 0; inp < m_decoders[i]->get_input_size(); ++inp) {
ov::PartialShape pshape = std::dynamic_pointer_cast<GgmlOvDecoder>(m_decoders[i])->get_input_shape(inp);
ov::element::Type ptype = std::dynamic_pointer_cast<GgmlOvDecoder>(m_decoders[i])->get_input_type(inp);
GGML_LOG_INFO("- Input name: %s\n", std::dynamic_pointer_cast<GgmlOvDecoder>(m_decoders[i])->get_input_name(inp).c_str());
GGML_LOG_INFO(" Input shape: %s\n", pshape.to_string().c_str());
GGML_LOG_INFO(" Input type: %s\n", ptype.to_string().c_str());
}
for (size_t outp = 0; outp < std::dynamic_pointer_cast<GgmlOvDecoder>(m_decoders[i])->get_output_size(); ++outp) {
ov::PartialShape pshape = std::dynamic_pointer_cast<GgmlOvDecoder>(m_decoders[i])->get_output_shape(outp);
ov::element::Type ptype = std::dynamic_pointer_cast<GgmlOvDecoder>(m_decoders[i])->get_output_type(outp);
GGML_LOG_INFO("- Output name: %s\n", std::dynamic_pointer_cast<GgmlOvDecoder>(m_decoders[i])->get_output_name(outp).c_str());
GGML_LOG_INFO(" Output shape: %s\n", pshape.to_string().c_str());
GGML_LOG_INFO(" Output type: %s\n", ptype.to_string().c_str());
}
}
}
}
} // namespace ggml
} // namespace frontend
} // namespace ov

61
ggml/src/ggml-openvino/ggml-graph-iterator.h
View File

@ -1,61 +0,0 @@
#pragma once
#include "graph_iterator.h"
#include "ggml-decoder.h"
#include <ggml-impl.h>
// To remove tensorflow
namespace ov {
namespace frontend {
namespace tensorflow {
namespace ggml {
class GgmlOvGraphIterator : public GgmlGraphIterator {
protected:
void initialize_decoders();
public:
using Ptr = std::shared_ptr<GgmlOvGraphIterator>;
GgmlOvGraphIterator(struct ggml_cgraph * cgraph);
/// \brief Get a number of operation nodes in the sgraph
virtual size_t size() const override;
/// \brief Set iterator to the start position
virtual void reset() override;
/// \brief Move to the next node in the graph
virtual void next() override;
/// \brief Returns true if iterator goes out of the range of available nodes
virtual bool is_end() const override;
/// \brief Return a pointer to a decoder of the current node
virtual std::shared_ptr<DecoderBase> get_decoder() const override;
virtual std::shared_ptr<GraphIterator> get_body_graph_iterator(const std::string& func_name) const override {
return nullptr;
GGML_UNUSED(func_name);
}
/// \brief Returns a vector of input names in the original order
virtual std::vector<std::string> get_input_names() const override;
/// \brief Returns a vector of output names in the original order
virtual std::vector<std::string> get_output_names() const override;
virtual void dump_graph_iterator() const;
private:
struct ggml_cgraph * m_cgraph;
size_t node_index = 0;
std::vector<std::shared_ptr<DecoderBase>> m_decoders;
std::vector<std::string> m_input_names;
std::vector<std::string> m_output_names;
};
}
} // namespace ggml
} // namespace frontend
} // namespace ov

43
ggml/src/ggml-openvino/graph_iterator.h
View File

@ -1,43 +0,0 @@
#pragma once
#include "openvino/frontend/graph_iterator.hpp"
namespace ov {
namespace frontend {
namespace tensorflow { // To be Removed
namespace ggml {
// TODO: Directly include from openvino
class GgmlGraphIterator : public GraphIterator {
public:
virtual size_t size() const = 0;
virtual void reset() = 0;
virtual void next() = 0;
virtual bool is_end() const = 0;
virtual std::shared_ptr<DecoderBase> get_decoder() const = 0;
virtual std::vector<std::string> get_input_names() const = 0;
virtual std::vector<std::string> get_output_names() const = 0;
virtual std::shared_ptr<GraphIterator> get_body_graph_iterator(const std::string& func_name) const = 0;
virtual std::map<std::string, std::string> get_input_names_map() const {
return {};
}
virtual std::map<std::string, std::string> get_output_names_map() const {
return {};
}
};
}
} // namespace ggml
} // namespace frontend
} // namespace ov

74
ggml/src/ggml-openvino/utils.cpp
View File

@ -1,49 +1,40 @@
#include "utils.h"
#include "ggml-impl.h"
#include "ggml-backend-impl.h"
#include <openvino/frontend/manager.hpp>
#include <openvino/openvino.hpp>
using ov::frontend::tensorflow::ggml::GgmlOvGraphIterator;
using ov::frontend::ggml::GgmlDecoder;
std::shared_ptr<GgmlOvGraphIterator> get_ggml_graph_iterator(struct ggml_cgraph * cgraph) {
return std::make_shared<GgmlOvGraphIterator>(cgraph);
std::shared_ptr<GgmlOvDecoder> get_ggml_decoder(struct ggml_cgraph * cgraph) {
return std::make_shared<GgmlOvDecoder>(nullptr, cgraph);
}
std::map<std::string, ov::Tensor> get_ggml_graph_input_tensors(std::shared_ptr<GgmlOvGraphIterator> ggml_graph_iterator) {
std::map<std::string, ov::Tensor> get_ggml_graph_input_tensors(std::shared_ptr<GgmlOvDecoder> ggml_decoder) {
std::map<std::string, ov::Tensor> input_tensors;
auto input_names = ggml_graph_iterator->get_input_names();
ggml_graph_iterator->reset();
for (; !ggml_graph_iterator->is_end(); ggml_graph_iterator->next()) {
auto decoder = std::dynamic_pointer_cast<GgmlOvDecoder>(ggml_graph_iterator->get_decoder());
for (size_t inp = 0; inp < decoder->get_input_size(); ++inp) {
if (std::find(input_names.begin(), input_names.end(), decoder->get_input_name(inp)) != input_names.end()) {
auto input_data = decoder->get_input_ggml_tensor(inp)->data;
#ifdef GGML_OPENVINO_DEBUG
printf("Subgraph input %d: %g\n", inp, *(double*)(input_data));
#endif
ov::Tensor input_tensor = ov::Tensor(decoder->get_input_type(inp), decoder->get_input_shape(inp).to_shape(), input_data);
input_tensors[decoder->get_input_name(inp)] = input_tensor;
}
}
auto input_names = ggml_decoder->get_input_names();
for (size_t inp = 0; inp < input_names.size(); ++inp) {
auto name = input_names[inp];
auto input_data = ggml_decoder->get_input_ggml_tensor(name)->data;
#ifdef GGML_OPENVINO_DEBUG
printf("Subgraph input %d: %g\n", inp, *(double*)(input_data));
#endif
ov::Tensor input_tensor = ov::Tensor(ggml_decoder->get_input_type(name), ggml_decoder->get_input_shape(name).to_shape(), input_data);
input_tensors[name] = input_tensor;
}
return input_tensors;
}
std::map<std::string, void*> get_ggml_graph_output_dst(std::shared_ptr<GgmlOvGraphIterator> ggml_graph_iterator) {
std::map<std::string, void*> get_ggml_graph_output_dst(std::shared_ptr<GgmlOvDecoder> ggml_decoder) {
std::map<std::string, void*> output_tensors;
auto output_names = ggml_graph_iterator->get_output_names();
ggml_graph_iterator->reset();
for (; !ggml_graph_iterator->is_end(); ggml_graph_iterator->next()) {
auto decoder = std::dynamic_pointer_cast<GgmlOvDecoder>(ggml_graph_iterator->get_decoder());
for (size_t inp = 0; inp < decoder->get_output_size(); ++inp) {
if (std::find(output_names.begin(), output_names.end(), decoder->get_output_name(inp)) != output_names.end()) {
auto output_data = decoder->get_output_ggml_tensor(inp)->data;
#ifdef GGML_OPENVINO_DEBUG
printf("Output %d: %g\n", inp, *(double*)(output_data));
#endif
output_tensors[decoder->get_output_name(inp)] = output_data;
}
}
auto output_names = ggml_decoder->get_output_names();
for (size_t inp = 0; inp < output_names.size(); ++inp) {
auto name = output_names[inp];
auto output_data = ggml_decoder->get_output_ggml_tensor(name)->data;
#ifdef GGML_OPENVINO_DEBUG
printf("Output %d: %g\n", inp, *(double*)(output_data));
#endif
output_tensors[name] = output_data;
}
return output_tensors;
}
@ -74,12 +65,10 @@ enum ggml_status openvino_frontend_compute(ggml_backend_t backend, struct ggml_c
GGML_LOG_INFO("GGML FrontEnd is initialized \n");
#endif
}
auto ggml_graph_iterator = get_ggml_graph_iterator(cgraph);
std::shared_ptr<ov::frontend::tensorflow::GraphIterator> graph_iterator = ggml_graph_iterator;
auto ggml_decoder = get_ggml_decoder(cgraph);
std::shared_ptr<ov::frontend::DecoderBase> graph_decoder = ggml_decoder;
// Load GraphIterator -> InputModel
ov::frontend::InputModel::Ptr input_model = front_end->load(graph_iterator);
ov::frontend::InputModel::Ptr input_model = front_end->load(graph_decoder);
if (!input_model) {
GGML_LOG_ERROR("Input Model is not loaded \n");
return GGML_STATUS_FAILED;
@ -106,8 +95,8 @@ enum ggml_status openvino_frontend_compute(ggml_backend_t backend, struct ggml_c
ov::InferRequest infer_request = compiled_model.create_infer_request();
// Get input tensor
auto input_names = ggml_graph_iterator->get_input_names();
auto input_tensors = get_ggml_graph_input_tensors(ggml_graph_iterator);
auto input_names = ggml_decoder->get_input_names();
auto input_tensors = get_ggml_graph_input_tensors(ggml_decoder);
// Set input tensor
for (size_t i = 0; i < input_names.size(); i++) {
@ -117,11 +106,14 @@ enum ggml_status openvino_frontend_compute(ggml_backend_t backend, struct ggml_c
infer_request.infer();
// Set dst data for outputs
auto output_names = ggml_graph_iterator->get_output_names();
auto output_tensors = get_ggml_graph_output_dst(ggml_graph_iterator);
auto output_names = ggml_decoder->get_output_names();
auto output_tensors = get_ggml_graph_output_dst(ggml_decoder);
for (size_t i = 0; i < output_names.size(); i++) {
auto output_tensor = infer_request.get_output_tensor(i);
std::memcpy(output_tensors[output_names[i]], output_tensor.data(), output_tensor.get_byte_size());
#ifdef GGML_OPENVINO_DEBUG
printf("Output %s after: %g\n", output_names[i], *(double*)(output_tensor.data()));
#endif
}
return GGML_STATUS_SUCCESS;

4
ggml/src/ggml-openvino/utils.h
View File

@ -1,6 +1,4 @@
#include "ggml-graph-iterator.h"
#include "ggml-decoder.h"
#include "ggml-backend-impl.h"
std::shared_ptr<ov::frontend::tensorflow::ggml::GgmlOvGraphIterator> get_ggml_graph_iterator(struct ggml_cgraph * cgraph);
enum ggml_status openvino_frontend_compute (ggml_backend_t backend, struct ggml_cgraph * cgraph);