happyz
/
llama.cpp
mirror of https://github.com/ggerganov/llama.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

All adjacent ops can conversion but calculation result is wrong and need debugging

This commit is contained in:
zhanmyz 2025-03-06 13:51:34 +08:00 committed by Mustafa Cavus
parent d05c458421
commit e08a7fda33
3 changed files with 114 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
ggml/src/ggml-openvino.cpp
View File

@ -480,12 +480,12 @@ void ggml_backend_openvino_mul_mat(struct ggml_tensor * dst) {
size_t total_src0 = indices_src0.size(); // = 96 * 32 * 32 size_t total_src0 = indices_src0.size(); // = 96 * 32 * 32
size_t total_src1 = indices_src1.size(); // = 96 * 7 * 32 size_t total_src1 = indices_src1.size(); // = 96 * 7 * 32
ov::Shape orig_shape_src0 = { static_cast<size_t>(src0->ne[0]), ov::Shape orig_shape_src0 = { static_cast<size_t>(src0->ne[2]),
static_cast<size_t>(src0->ne[1]), static_cast<size_t>(src0->ne[1]),
static_cast<size_t>(src0->ne[2])}; static_cast<size_t>(src0->ne[0])};
ov::Shape orig_shape_src1 = { static_cast<size_t>(src1->ne[0]), ov::Shape orig_shape_src1 = { static_cast<size_t>(src1->ne[2]),
static_cast<size_t>(src1->ne[1]), static_cast<size_t>(src1->ne[1]),
static_cast<size_t>(src1->ne[2])}; static_cast<size_t>(src1->ne[0])};
auto param_src0 = std::make_shared<ov::op::v0::Parameter>(ov::element::f16, orig_shape_src0); auto param_src0 = std::make_shared<ov::op::v0::Parameter>(ov::element::f16, orig_shape_src0);
auto param_src1 = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, orig_shape_src1); auto param_src1 = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, orig_shape_src1);
@ -573,12 +573,12 @@ void ggml_backend_openvino_mul_mat(struct ggml_tensor * dst) {
std::vector<int64_t> eff_shape_src1 = get_effective_shape(src1); std::vector<int64_t> eff_shape_src1 = get_effective_shape(src1);
std::vector<int64_t> eff_shape_dst = get_effective_shape(dst); std::vector<int64_t> eff_shape_dst = get_effective_shape(dst);
ov::Shape orig_shape_src0 = { static_cast<size_t>(src0->ne[0]), ov::Shape orig_shape_src0 = { static_cast<size_t>(src0->ne[2]),
static_cast<size_t>(src0->ne[1]), static_cast<size_t>(src0->ne[1]),
static_cast<size_t>(src0->ne[2])}; static_cast<size_t>(src0->ne[0])};
ov::Shape orig_shape_src1 = { static_cast<size_t>(src1->ne[0]), ov::Shape orig_shape_src1 = { static_cast<size_t>(src1->ne[2]),
static_cast<size_t>(src1->ne[1]), static_cast<size_t>(src1->ne[1]),
static_cast<size_t>(src1->ne[2])}; static_cast<size_t>(src1->ne[0])};
auto param_src0 = std::make_shared<ov::op::v0::Parameter>(ov::element::f16, orig_shape_src0); auto param_src0 = std::make_shared<ov::op::v0::Parameter>(ov::element::f16, orig_shape_src0);
auto param_src1 = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, orig_shape_src1); auto param_src1 = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, orig_shape_src1);
@ -999,40 +999,40 @@ static enum ggml_status ggml_backend_openvino_graph_compute(ggml_backend_t backe
} }
int end_node = cgraph->n_nodes - 1; int end_node = cgraph->n_nodes - 1;
// openvino_frontend_compute(backend, cgraph, 0, end_node); openvino_frontend_compute(backend, cgraph, 0, end_node);
// openvino_frontend_compute(backend, cgraph); // openvino_frontend_compute(backend, cgraph);
// Process nodes in order // Process nodes in order
for (int i = 0; i < cgraph->n_nodes; i++) { // for (int i = 0; i < cgraph->n_nodes; i++) {
if (std::find(permute_indices.begin(), permute_indices.end(), i) != permute_indices.end()) { // if (std::find(permute_indices.begin(), permute_indices.end(), i) != permute_indices.end()) {
ggml_backend_openvino_permute(cgraph->nodes[i]); // ggml_backend_openvino_permute(cgraph->nodes[i]);
} else if (std::find(cont_indices.begin(), cont_indices.end(), i) != cont_indices.end()) { // } else if (std::find(cont_indices.begin(), cont_indices.end(), i) != cont_indices.end()) {
ggml_backend_openvino_dup_bytes(cgraph->nodes[i]); // ggml_backend_openvino_dup_bytes(cgraph->nodes[i]);
} else if (std::find(view_indices.begin(), view_indices.end(), i) != view_indices.end()) { // } else if (std::find(view_indices.begin(), view_indices.end(), i) != view_indices.end()) {
ggml_backend_openvino_view(cgraph->nodes[i]); // ggml_backend_openvino_view(cgraph->nodes[i]);
} else if (std::find(cpy_indices.begin(), cpy_indices.end(), i) != cpy_indices.end()) { // } else if (std::find(cpy_indices.begin(), cpy_indices.end(), i) != cpy_indices.end()) {
ggml_backend_openvino_cpy(cgraph->nodes[i]); // ggml_backend_openvino_cpy(cgraph->nodes[i]);
} else if (std::find(transpose_indices.begin(), transpose_indices.end(), i) != transpose_indices.end()) { // } else if (std::find(transpose_indices.begin(), transpose_indices.end(), i) != transpose_indices.end()) {
ggml_backend_openvino_transpose(cgraph->nodes[i]); // ggml_backend_openvino_transpose(cgraph->nodes[i]);
} else if (std::find(reshape_indices.begin(), reshape_indices.end(), i) != reshape_indices.end()) { // } else if (std::find(reshape_indices.begin(), reshape_indices.end(), i) != reshape_indices.end()) {
ggml_backend_openvino_reshape(cgraph->nodes[i]); // ggml_backend_openvino_reshape(cgraph->nodes[i]);
} else if (std::find(mul_mat_indices.begin(), mul_mat_indices.end(), i) != mul_mat_indices.end()) { // } else if (std::find(mul_mat_indices.begin(), mul_mat_indices.end(), i) != mul_mat_indices.end()) {
ggml_backend_openvino_mul_mat(cgraph->nodes[i]); // ggml_backend_openvino_mul_mat(cgraph->nodes[i]);
} else { // } else {
// Process a range of nodes with openvino_frontend_compute // // Process a range of nodes with openvino_frontend_compute
int start_index = i; // int start_index = i;
while (i < cgraph->n_nodes // while (i < cgraph->n_nodes
&& std::find(view_indices.begin(), view_indices.end(), i) == view_indices.end() // && std::find(view_indices.begin(), view_indices.end(), i) == view_indices.end()
&& std::find(cpy_indices.begin(), cpy_indices.end(), i) == cpy_indices.end() // && std::find(cpy_indices.begin(), cpy_indices.end(), i) == cpy_indices.end()
&& std::find(cont_indices.begin(), cont_indices.end(), i) == cont_indices.end() // && std::find(cont_indices.begin(), cont_indices.end(), i) == cont_indices.end()
&& std::find(mul_mat_indices.begin(), mul_mat_indices.end(), i) == mul_mat_indices.end() // && std::find(mul_mat_indices.begin(), mul_mat_indices.end(), i) == mul_mat_indices.end()
) { // ) {
i++; // i++;
} // }
if (start_index < i) { // if (start_index < i) {
openvino_frontend_compute(backend, cgraph, start_index, --i); // openvino_frontend_compute(backend, cgraph, start_index, --i);
} // }
} // }
} // }
return GGML_STATUS_SUCCESS; return GGML_STATUS_SUCCESS;
@ -1257,14 +1257,13 @@ static const std::set<std::string>& openvino_ops = []() -> const std::set<std::s
case GGML_OP_ADD: case GGML_OP_ADD:
return true; return true;
case GGML_OP_MUL: case GGML_OP_MUL:
return false;
case GGML_OP_MUL_MAT: case GGML_OP_MUL_MAT:
return false; return false;
case GGML_OP_UNARY: case GGML_OP_UNARY:
switch (ggml_get_unary_op(op)) switch (ggml_get_unary_op(op))
{ {
case GGML_UNARY_OP_SILU: case GGML_UNARY_OP_SILU:
return false; return true;
case GGML_UNARY_OP_ABS: case GGML_UNARY_OP_ABS:
case GGML_UNARY_OP_SGN: case GGML_UNARY_OP_SGN:
case GGML_UNARY_OP_NEG: case GGML_UNARY_OP_NEG:

50
ggml/src/ggml-openvino/ggml-decoder.cpp
View File

@ -48,9 +48,9 @@ void GgmlOvDecoder::set_input_output(ggml_tensor* node, std::map<std::string, gg
m_output_names.push_back(node_name); m_output_names.push_back(node_name);
m_continuous = true; m_continuous = true;
ov::Shape flat_shape = { static_cast<size_t>(ggml_nelements(node)) }; // ov::Shape flat_shape = { static_cast<size_t>(ggml_nelements(node)) };
auto input_param = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, flat_shape); // auto input_param = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, flat_shape);
m_params.push_back(input_param); // m_params.push_back(input_param);
break; break;
} }
@ -72,15 +72,15 @@ void GgmlOvDecoder::set_input_output(ggml_tensor* node, std::map<std::string, gg
m_op_node_name.emplace_back(src0_name, ggml_op_name(node->op)); m_op_node_name.emplace_back(src0_name, ggml_op_name(node->op));
m_output_names.push_back(node_name); m_output_names.push_back(node_name);
const size_t element_size = ggml_type_size(node->src[0]->type); // const size_t element_size = ggml_type_size(node->src[0]->type);
size_t valid_elems = static_cast<size_t>(node->src[0]->ne[0]); // 3072 // size_t valid_elems = static_cast<size_t>(node->src[0]->ne[0]); // 3072
size_t num_rows = static_cast<size_t>(node->src[0]->ne[1]); // 7 // size_t num_rows = static_cast<size_t>(node->src[0]->ne[1]); // 7
size_t phys_stride = static_cast<size_t>(node->src[0]->nb[1]) / element_size; // 9216 // size_t phys_stride = static_cast<size_t>(node->src[0]->nb[1]) / element_size; // 9216
// size_t total_phys = (num_rows - 1) * phys_stride + valid_elems; // 6*9216 + 3072 = 58368 // // size_t total_phys = (num_rows - 1) * phys_stride + valid_elems; // 6*9216 + 3072 = 58368
size_t total_phys = num_rows * phys_stride; // 7 * 9216 = 64512 // size_t total_phys = num_rows * phys_stride; // 7 * 9216 = 64512
ov::Shape flat_input_shape = { total_phys }; // ov::Shape flat_input_shape = { total_phys };
auto flat_input_param = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, flat_input_shape); // auto flat_input_param = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, flat_input_shape);
m_params.push_back(flat_input_param); // m_params.push_back(flat_input_param);
m_continuous = false; m_continuous = false;
break; break;
@ -94,13 +94,13 @@ void GgmlOvDecoder::set_input_output(ggml_tensor* node, std::map<std::string, gg
m_op_node_name.emplace_back(src0_name, ggml_op_name(node->op)); m_op_node_name.emplace_back(src0_name, ggml_op_name(node->op));
m_output_names.push_back(node_name); m_output_names.push_back(node_name);
size_t valid_i = static_cast<size_t>(node->src[0]->ne[0]); // 96 // size_t valid_i = static_cast<size_t>(node->src[0]->ne[0]); // 96
size_t valid_j = static_cast<size_t>(node->src[0]->ne[1]); // 32 // size_t valid_j = static_cast<size_t>(node->src[0]->ne[1]); // 32
size_t valid_k = static_cast<size_t>(node->src[0]->ne[2]); // 7 // size_t valid_k = static_cast<size_t>(node->src[0]->ne[2]); // 7
size_t total_valid = valid_i * valid_j * valid_k; // 96 * 32 * 7 = 21504 // size_t total_valid = valid_i * valid_j * valid_k; // 96 * 32 * 7 = 21504
ov::Shape flat_input_shape = { total_valid }; // ov::Shape flat_input_shape = { total_valid };
auto input_param = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, flat_input_shape); // auto input_param = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, flat_input_shape);
m_params.push_back(input_param); // m_params.push_back(input_param);
m_continuous = false; m_continuous = false;
break; break;
@ -190,12 +190,12 @@ void GgmlOvDecoder::set_input_output(ggml_tensor* node, std::map<std::string, gg
} }
case GGML_OP_MUL_MAT: case GGML_OP_MUL_MAT:
{ {
ov::Shape flat_shape_src0 = { node->src[0]->ne[0]*node->src[0]->ne[1]*node->src[0]->ne[2] }; // ov::Shape flat_shape_src0 = { node->src[0]->ne[0]*node->src[0]->ne[1]*node->src[0]->ne[2] };
ov::Shape flat_shape_src1 = { node->src[1]->ne[0]*node->src[1]->ne[1]*node->src[1]->ne[2] }; // ov::Shape flat_shape_src1 = { node->src[1]->ne[0]*node->src[1]->ne[1]*node->src[1]->ne[2] };
auto param_src0 = std::make_shared<ov::op::v0::Parameter>(ov::element::f16, flat_shape_src0); // auto param_src0 = std::make_shared<ov::op::v0::Parameter>(ov::element::f16, flat_shape_src0);
auto param_src1 = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, flat_shape_src1); // auto param_src1 = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, flat_shape_src1);
m_params.push_back(param_src0); // m_params.push_back(param_src0);
m_params.push_back(param_src1); // m_params.push_back(param_src1);
if (!ggml_is_contiguous(node->src[1]) || node->src[1]->ne[0] * node->src[1]->nb[0] != node->src[1]->nb[1]) { if (!ggml_is_contiguous(node->src[1]) || node->src[1]->ne[0] * node->src[1]->nb[0] != node->src[1]->nb[1]) {
m_continuous = false; m_continuous = false;
} else { } else {

74
ggml/src/ggml-openvino/utils.cpp
View File

@ -14,12 +14,15 @@ std::map<std::string, ov::Tensor> get_ggml_graph_input_tensors(std::shared_ptr<G
std::map<std::string, ov::Tensor> input_tensors; std::map<std::string, ov::Tensor> input_tensors;
auto input_names = ggml_decoder->get_input_names(); auto input_names = ggml_decoder->get_input_names();
// auto node_name = ggml_decoder->get_op_name(); // auto node_name = ggml_decoder->get_op_name();
size_t iter = 0; size_t op_iter = 0;
for (size_t inp = 0; inp < input_names.size(); ++inp) { for (size_t inp = 0; inp < input_names.size(); ++inp) {
auto name = input_names[inp]; auto name = input_names[inp];
std::string op_node_name = ggml_decoder->get_op_node_name(name, iter++); std::string op_node_name = ggml_decoder->get_op_node_name(name, op_iter++);
// auto node_op_name = ggml_decoder->get_node_op_name(name); // auto node_op_name = ggml_decoder->get_node_op_name(name);
ov::element::Type input_type = ggml_decoder->get_input_type(name);
size_t element_size = input_type.size();
auto input_data = ggml_decoder->get_input_ggml_tensor(name)->data; auto input_data = ggml_decoder->get_input_ggml_tensor(name)->data;
std::vector<size_t> input_stride = ggml_decoder->get_input_stride(name);
#ifdef GGML_OPENVINO_DEBUG #ifdef GGML_OPENVINO_DEBUG
printf("Subgraph input %d: %g\n", inp, *(double*)(input_data)); printf("Subgraph input %d: %g\n", inp, *(double*)(input_data));
#endif #endif
@ -28,36 +31,51 @@ std::map<std::string, ov::Tensor> get_ggml_graph_input_tensors(std::shared_ptr<G
// if (node_op_name == "CPY" && (input_shape[0] != 7)) { // if (node_op_name == "CPY" && (input_shape[0] != 7)) {
// input_tensor = ov::Tensor(ggml_decoder->get_input_type(name), {80000}, input_data); // input_tensor = ov::Tensor(ggml_decoder->get_input_type(name), {80000}, input_data);
if (op_node_name == "CONT" && ggml_decoder->check_if_continuous()) { if (op_node_name == "CONT" && !ggml_decoder->check_if_continuous() && input_shape[0] == 1) {
ov::Shape flat_shape = { ggml_decoder->get_input_shape(name).to_shape()[0] * const size_t valid_elems = static_cast<size_t>(ggml_decoder->get_input_shape(name).to_shape()[2]);
ggml_decoder->get_input_shape(name).to_shape()[1] * const size_t num_rows = static_cast<size_t>(ggml_decoder->get_input_shape(name).to_shape()[1]);
ggml_decoder->get_input_shape(name).to_shape()[2] }; const size_t dim2 = static_cast<size_t>(ggml_decoder->get_input_shape(name).to_shape()[0]);
input_tensor = ov::Tensor(ggml_decoder->get_input_type(name), flat_shape, input_data); size_t phys_stride = static_cast<size_t>(input_stride[1]) / element_size;
} else if ( op_node_name == "CONT" && size_t total_logical = valid_elems * num_rows * dim2;
!ggml_decoder->check_if_continuous() &&
input_shape[0] == 1) { std::vector<float> contiguous_data(total_logical);
size_t valid_elems = static_cast<size_t>(ggml_decoder->get_input_shape(name).to_shape()[2]); // 3072
size_t num_rows = static_cast<size_t>(ggml_decoder->get_input_shape(name).to_shape()[1]); // 7 for (size_t j = 0; j < num_rows; j++) {
ov::element::Type input_type = ggml_decoder->get_input_type(name); const float *src_row = reinterpret_cast<const float*>(input_data) + j * phys_stride;
size_t element_size = input_type.size(); float *dst_row = contiguous_data.data() + j * valid_elems;
std::vector<size_t> strides = ggml_decoder->get_input_stride(name); std::copy(src_row, src_row + valid_elems, dst_row);
size_t phys_stride = static_cast<size_t>(strides[1]) / element_size; }
// size_t total_phys = (num_rows - 1) * phys_stride + valid_elems; input_tensor = ov::Tensor(ggml_decoder->get_input_type(name),
size_t total_phys = num_rows* phys_stride; ggml_decoder->get_input_shape(name).to_shape(),
ov::Shape flat_input_shape = { total_phys }; contiguous_data.data());
input_tensor = ov::Tensor(ggml_decoder->get_input_type(name), flat_input_shape, input_data); } else if (op_node_name == "CONT" && !ggml_decoder->check_if_continuous()){
} else if (op_node_name == "CONT") {
size_t valid_i = static_cast<size_t>(ggml_decoder->get_input_shape(name).to_shape()[2]); // 96 size_t valid_i = static_cast<size_t>(ggml_decoder->get_input_shape(name).to_shape()[2]); // 96
size_t valid_j = static_cast<size_t>(ggml_decoder->get_input_shape(name).to_shape()[1]); // 32 size_t valid_j = static_cast<size_t>(ggml_decoder->get_input_shape(name).to_shape()[1]); // 32
size_t valid_k = static_cast<size_t>(ggml_decoder->get_input_shape(name).to_shape()[0]); // 7 size_t valid_k = static_cast<size_t>(ggml_decoder->get_input_shape(name).to_shape()[0]); // 7
size_t total_valid = valid_i * valid_j * valid_k; // 96 * 32 * 7 = 21504 size_t total_valid = valid_i * valid_j * valid_k; // 96 * 32 * 7 = 21504
ov::Shape flat_input_shape = { total_valid }; size_t stride_j = static_cast<size_t>(input_stride[1]) / element_size; // 672
input_tensor = ov::Tensor(ggml_decoder->get_input_type(name), flat_input_shape, input_data); size_t stride_k = static_cast<size_t>(input_stride[0]) / element_size; // 96
} else if (op_node_name == "MUL_MAT") {
ov::Shape flat_shape = { ggml_decoder->get_input_shape(name).to_shape()[0] * std::vector<float> contiguous_data(total_valid);
ggml_decoder->get_input_shape(name).to_shape()[1] * const float *src_data = reinterpret_cast<const float*>(input_data);
ggml_decoder->get_input_shape(name).to_shape()[2] }; for (size_t k = 0; k < valid_k; k++) {
input_tensor = ov::Tensor(ggml_decoder->get_input_type(name), flat_shape, input_data); for (size_t j = 0; j < valid_j; j++) {
for (size_t i = 0; i < valid_i; i++) {
size_t out_index = k * (valid_i * valid_j) + j * valid_i + i;
size_t src_index = j * stride_j + k * stride_k + i;
contiguous_data[out_index] = src_data[src_index];
}
}
}
input_tensor = ov::Tensor(ggml_decoder->get_input_type(name),
ggml_decoder->get_input_shape(name).to_shape(),
contiguous_data.data());
// } else if (op_node_name == "MUL_MAT") {
// ov::Shape flat_shape = { ggml_decoder->get_input_shape(name).to_shape()[0] *
// ggml_decoder->get_input_shape(name).to_shape()[1] *
// ggml_decoder->get_input_shape(name).to_shape()[2] };
// input_tensor = ov::Tensor(ggml_decoder->get_input_type(name), flat_shape, input_data);
} else { } else {
input_tensor = ov::Tensor(ggml_decoder->get_input_type(name), ggml_decoder->get_input_shape(name).to_shape(), input_data); input_tensor = ov::Tensor(ggml_decoder->get_input_type(name), ggml_decoder->get_input_shape(name).to_shape(), input_data);
} }