From 7451b841052f12fcca971380cbe5a902d0d101b5 Mon Sep 17 00:00:00 2001 From: bluebread Date: Thu, 4 Dec 2025 13:26:53 +0000 Subject: [PATCH] mtmd: fix tensor names for image newlines and view separator --- examples/eval-callback/eval-callback.cpp | 18 ++++---- gguf-py/gguf/constants.py | 6 +-- gguf-py/gguf/gguf_writer.py | 2 +- gguf-py/gguf/tensor_mapping.py | 2 - tools/mtmd/clip-impl.h | 5 +-- tools/mtmd/clip.cpp | 54 ++++++++++-------------- 6 files changed, 38 insertions(+), 49 deletions(-) diff --git a/examples/eval-callback/eval-callback.cpp b/examples/eval-callback/eval-callback.cpp index 039bf19c99..80c693ce61 100644 --- a/examples/eval-callback/eval-callback.cpp +++ b/examples/eval-callback/eval-callback.cpp @@ -74,19 +74,19 @@ static void ggml_print_tensor(uint8_t * data, ggml_type type, const int64_t * ne } } for (int64_t i3 = 0; i3 < ne[3]; i3++) { - LOG(" [\n"); + LOG(" [\n"); for (int64_t i2 = 0; i2 < ne[2]; i2++) { if (i2 == n && ne[2] > 2*n) { - LOG(" ..., \n"); + LOG(" ..., \n"); i2 = ne[2] - n; } - LOG(" [\n"); + LOG(" [\n"); for (int64_t i1 = 0; i1 < ne[1]; i1++) { if (i1 == n && ne[1] > 2*n) { - LOG(" ..., \n"); + LOG(" ..., \n"); i1 = ne[1] - n; } - LOG(" ["); + LOG(" ["); for (int64_t i0 = 0; i0 < ne[0]; i0++) { if (i0 == n && ne[0] > 2*n) { LOG("..., "); @@ -98,10 +98,10 @@ static void ggml_print_tensor(uint8_t * data, ggml_type type, const int64_t * ne } LOG("],\n"); } - LOG(" ],\n"); + LOG(" ],\n"); } - LOG(" ]\n"); - LOG(" sum = %f\n", sum); + LOG(" ]\n"); + LOG(" sum = %f\n", sum); } // TODO: make this abort configurable/optional? @@ -136,7 +136,7 @@ static bool ggml_debug(struct ggml_tensor * t, bool ask, void * user_data) { snprintf(src1_str, sizeof(src1_str), "%s{%s}", src1->name, ggml_ne_string(src1).c_str()); } - LOG("%s: %16s = (%s) %10s(%s{%s}, %s}) = {%s}\n", __func__, + LOG("%s: %24s = (%s) %10s(%s{%s}, %s}) = {%s}\n", __func__, t->name, ggml_type_name(t->type), ggml_op_desc(t), src0->name, ggml_ne_string(src0).c_str(), src1 ? src1_str : "", diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index c7c06ceecd..884ce68de5 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -1077,7 +1077,7 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.V_MM_GATE: "mm.gate", MODEL_TENSOR.V_TOK_BOI: "v.boi", MODEL_TENSOR.V_TOK_EOI: "v.eoi", - # DeepSeek-OCR sam_model + # DeepSeek-OCR SAM MODEL_TENSOR.V_SAM_POS_EMBD: "v.sam.pos_embd", MODEL_TENSOR.V_SAM_PATCH_EMBD: "v.sam.patch_embd", MODEL_TENSOR.V_SAM_PRE_NORM: "v.sam.blk.{bid}.pre_ln", @@ -1091,8 +1091,8 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.V_SAM_NECK: "v.sam.neck.{bid}", MODEL_TENSOR.V_SAM_NET_2: "v.sam.net_2", MODEL_TENSOR.V_SAM_NET_3: "v.sam.net_3", - MODEL_TENSOR.V_ENC_EMBD_IMGNL: "model.image_newline", # Deepseek-OCR - MODEL_TENSOR.V_ENC_EMBD_VSEP: "model.view_seperator", # Deepseek-OCR + MODEL_TENSOR.V_ENC_EMBD_IMGNL: "v.image_newline", # Deepseek-OCR + MODEL_TENSOR.V_ENC_EMBD_VSEP: "v.view_seperator", # Deepseek-OCR # audio (mtmd) MODEL_TENSOR.A_ENC_EMBD_POS: "a.position_embd", MODEL_TENSOR.A_ENC_CONV1D: "a.conv1d.{bid}", diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index e0de7f8e72..15c318e11c 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -1127,12 +1127,12 @@ class GGUFWriter: def add_vision_is_deepstack_layers(self, layers: Sequence[bool]) -> None: self.add_array(Keys.ClipVision.IS_DEEPSTACK_LAYERS, layers) - def add_vision_sam_layers_count(self, value: int) -> None: self.add_uint32(Keys.ClipVision.SAM.BLOCK_COUNT, value) def add_vision_sam_embedding_length(self, value: int) -> None: self.add_uint32(Keys.ClipVision.SAM.EMBEDDING_LENGTH, value) + # audio models def add_audio_projection_dim(self, value: int) -> None: diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 5032328723..90491b15da 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -2,8 +2,6 @@ from __future__ import annotations from typing import Sequence -from numpy.f2py.auxfuncs import throw_error - from .constants import MODEL_ARCH, MODEL_TENSOR, MODEL_TENSORS, TENSOR_NAMES diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index a486ee1384..3e9b5639ba 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -86,8 +86,8 @@ #define TN_MVLM_PROJ_MLP "mm.model.mlp.%d.%s" #define TN_MVLM_PROJ_BLOCK "mm.model.mb_block.%d.block.%d.%s" #define TN_MVLM_PROJ_PEG "mm.model.peg.%d.%s" -#define TN_IMAGE_NEWLINE "model.image_newline" -#define TN_IMAGE_SEPERATOR "model.view_seperator" +#define TN_IMAGE_NEWLINE "v.image_newline" +#define TN_IMAGE_SEPERATOR "v.view_seperator" #define TN_MM_INP_NORM "mm.input_norm.weight" #define TN_MM_INP_NORM_B "mm.input_norm.bias" #define TN_MM_INP_PROJ "mm.input_projection.weight" // gemma3 @@ -443,7 +443,6 @@ static std::string gguf_kv_to_str(const struct gguf_context * ctx_gguf, int i) { // debugging // - static std::string to_ne_string(const ggml_tensor * t) { std::string str; for (int i = 0; i < GGML_MAX_DIMS; ++i) { diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index a50006ca9d..62c56a3c55 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -561,9 +561,9 @@ struct clip_graph { hparams(model.hparams), img(img), patch_size(hparams.patch_size), - n_patches_x(img.nx / patch_size), // sam 1024 / 16 = 64 - n_patches_y(img.ny / patch_size), // sam 1024 / 16 = 64 - n_patches(n_patches_x * n_patches_y), // sam 64 * 64 = 4096 + n_patches_x(img.nx / patch_size), + n_patches_y(img.ny / patch_size), + n_patches(n_patches_x * n_patches_y), n_embd(hparams.n_embd), n_head(hparams.n_head), d_head(n_embd / n_head), @@ -1302,7 +1302,7 @@ struct clip_graph { norm_t, hparams.ffn_op, model.position_embeddings, - nullptr); // shape [1024, 16, 16] + nullptr); // remove CLS token cur = ggml_view_2d(ctx0, cur, @@ -2399,18 +2399,15 @@ private: // build the input after conv2d (inp_raw --> patches) // returns tensor with shape [n_embd, n_patches] ggml_tensor * build_inp() { - // Image to Patch Embedding. - ggml_tensor * inp_raw = build_inp_raw(); // sam shape = [1024, 1024, 3] - // sam patch_embeddings_0 shape = [768, 3, 16, 16] - ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1); // sam shape = [64, 64, 768] - inp = ggml_reshape_2d(ctx0, inp, n_patches, n_embd); // sam shape = [4096, 768] - inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); // sam shape = [768, 4096] + ggml_tensor * inp_raw = build_inp_raw(); + ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1); + inp = ggml_reshape_2d(ctx0, inp, n_patches, n_embd); + inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); if (model.patch_bias) { - // sam patch_bias shape = [768] inp = ggml_add(ctx0, inp, model.patch_bias); cb(inp, "patch_bias", -1); } - return inp; // shape = [n_embd, n_patches] same as [768, 4096] + return inp; } ggml_tensor * build_inp_raw(int channels = 3) { @@ -3710,22 +3707,19 @@ struct clip_model_loader { layer.ff_down_w = get_tensor(string_format(TN_SAM_FFN_DOWN, il, "weight")); layer.ff_down_b = get_tensor(string_format(TN_SAM_FFN_DOWN, il, "bias")); } - model.neck_0_w = get_tensor(string_format(TN_SAM_NECK, 0, "weight")); - model.neck_1_b = get_tensor(string_format(TN_SAM_NECK, 1, "bias")); - model.neck_1_w = get_tensor(string_format(TN_SAM_NECK, 1, "weight")); - model.neck_2_w = get_tensor(string_format(TN_SAM_NECK, 2, "weight")); - model.neck_3_b = get_tensor(string_format(TN_SAM_NECK, 3, "bias")); - model.neck_3_w = get_tensor(string_format(TN_SAM_NECK, 3, "weight")); - model.net_2 = get_tensor(string_format(TN_SAM_NET, 2, "weight")); - model.net_3 = get_tensor(string_format(TN_SAM_NET, 3, "weight")); - } - model.image_newline = get_tensor(TN_IMAGE_NEWLINE, false); - model.view_seperator = get_tensor(TN_IMAGE_SEPERATOR, false); - model.fc_w = get_tensor(string_format(TN_MM_PROJECTOR, "weight")); - model.fc_b = get_tensor(string_format(TN_MM_PROJECTOR, "bias")); - - - break; + model.neck_0_w = get_tensor(string_format(TN_SAM_NECK, 0, "weight")); + model.neck_1_b = get_tensor(string_format(TN_SAM_NECK, 1, "bias")); + model.neck_1_w = get_tensor(string_format(TN_SAM_NECK, 1, "weight")); + model.neck_2_w = get_tensor(string_format(TN_SAM_NECK, 2, "weight")); + model.neck_3_b = get_tensor(string_format(TN_SAM_NECK, 3, "bias")); + model.neck_3_w = get_tensor(string_format(TN_SAM_NECK, 3, "weight")); + model.net_2 = get_tensor(string_format(TN_SAM_NET, 2, "weight")); + model.net_3 = get_tensor(string_format(TN_SAM_NET, 3, "weight")); + model.image_newline = get_tensor(TN_IMAGE_NEWLINE); + model.view_seperator = get_tensor(TN_IMAGE_SEPERATOR); + model.fc_w = get_tensor(string_format(TN_MM_PROJECTOR, "weight")); + model.fc_b = get_tensor(string_format(TN_MM_PROJECTOR, "bias")); + } break; default: GGML_ASSERT(false && "unknown projector type"); } @@ -5847,11 +5841,9 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima case PROJECTOR_TYPE_VOXTRAL: case PROJECTOR_TYPE_JANUS_PRO: case PROJECTOR_TYPE_COGVLM: - { - // do nothing - } break; case PROJECTOR_TYPE_DEEPSEEKOCR: { + // do nothing } break; case PROJECTOR_TYPE_LLAMA4: {