happyz
/
llama.cpp
mirror of https://github.com/ggerganov/llama.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

mtmd: add llama-mtmd-debug binary (#20508) 

* mtmd: add llama-mtmd-debug binary

* adapt

* fixes

* fix compile error

* fix windows compile error

* rm legacy clip_debug_encode()

* add MTMD_API to fix build
This commit is contained in:
Xuan-Son Nguyen 2026-03-14 15:52:29 +01:00 committed by GitHub
parent a93c0ef0fa
commit 94d0262277
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 392 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
tools/mtmd/CMakeLists.txt
View File

@ -62,6 +62,10 @@ set_target_properties(mtmd
PROPERTIES
PUBLIC_HEADER "${MTMD_PUBLIC_HEADERS}")
set_target_properties(mtmd
PROPERTIES
PRIVATE_HEADER debug/mtmd-debug.h)
install(TARGETS mtmd LIBRARY PUBLIC_HEADER)
if (NOT MSVC)
@ -96,3 +100,9 @@ if(LLAMA_TOOLS_INSTALL)
endif()
target_link_libraries (${TARGET} PRIVATE common mtmd Threads::Threads)
target_compile_features(${TARGET} PRIVATE cxx_std_17)
# mtmd-debug tool
add_executable(llama-mtmd-debug debug/mtmd-debug.cpp)
set_target_properties(llama-mtmd-debug PROPERTIES OUTPUT_NAME llama-mtmd-debug)
target_link_libraries(llama-mtmd-debug PRIVATE common mtmd Threads::Threads)
target_compile_features(llama-mtmd-debug PRIVATE cxx_std_17)

3
tools/mtmd/clip-impl.h
View File

@ -579,10 +579,9 @@ static void print_tensor_data(ggml_tensor * t, uint8_t * data, int64_t n) {
}
}
void clip_debug_encode(clip_ctx * ctx, int h, int w, float fill_value);
//
// API used internally with mtmd
//
projector_type clip_get_projector_type(const struct clip_ctx * ctx);
void clip_set_debug_output_embeddings(struct clip_ctx * ctx, bool debug);

21
tools/mtmd/clip.cpp
View File

@ -159,6 +159,8 @@ struct clip_ctx {
clip_flash_attn_type flash_attn_type = CLIP_FLASH_ATTN_TYPE_AUTO;
bool is_allocated = false;
bool debug_output_embeddings = false;
clip_ctx(clip_context_params & ctx_params) {
flash_attn_type = ctx_params.flash_attn_type;
backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr);
@ -205,6 +207,8 @@ struct clip_ctx {
if (ctx_params.cb_eval != nullptr) {
ggml_backend_sched_set_eval_callback(sched.get(), ctx_params.cb_eval, ctx_params.cb_eval_user_data);
}
debug_output_embeddings = std::getenv("MTMD_DEBUG_EMBEDDINGS") != nullptr;
}
~clip_ctx() {
@ -2193,8 +2197,6 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params
// TODO: we don't support audio for Gemma 3N, but GGUF contains audio tensors
// we can remove this check when we implement audio support for Gemma 3N
skip_audio = ctx_vision->model.proj_type == PROJECTOR_TYPE_GEMMA3NV;
// clip_debug_encode(ctx_vision, 24*14, 24*14, 0.5f);
}
if (loader.has_audio && !skip_audio) {
@ -3981,7 +3983,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
}
// Debug: dump final embeddings if MTMD_DEBUG_EMBEDDINGS is set
if (std::getenv("MTMD_DEBUG_EMBEDDINGS") != nullptr) {
if (ctx->debug_output_embeddings) {
const int64_t n_embd = embeddings->ne[0];
const int64_t n_tokens = embeddings->ne[1];
std::vector<float> emb_data(n_embd * n_tokens);
@ -4160,14 +4162,7 @@ const clip_hparams * clip_get_hparams(const struct clip_ctx * ctx) {
//
// API for debugging
//
void clip_debug_encode(clip_ctx * ctx, int h, int w, float fill_value) {
clip_image_f32 img;
img.nx = w;
img.ny = h;
img.buf.resize(h * w * 3);
for (int i = 0; i < h * w * 3; i++) {
img.buf[i] = static_cast<float>(fill_value);
}
clip_image_encode(ctx, 1, &img, nullptr);
GGML_ASSERT(img.buf.empty() && "expected, always stop here");
void clip_set_debug_output_embeddings(clip_ctx * ctx, bool enable) {
ctx->debug_output_embeddings = enable;
}

229
tools/mtmd/debug/mtmd-debug.cpp Normal file
View File

@ -0,0 +1,229 @@
#include "mtmd-debug.h"
#include "arg.h"
#include "debug.h"
#include "log.h"
#include "common.h"
#include "llama.h"
#include "ggml.h"
#include "mtmd.h"
#include "mtmd-helper.h"
#include <vector>
#include <cmath>
#include <limits.h>
#include <cinttypes>
#include <clocale>
// INTERNAL TOOL FOR DEBUGGING PURPOSES ONLY
// NOT INTENDED FOR PUBLIC USE
static void show_additional_info(int /*argc*/, char ** argv) {
LOG(
"Internal debugging tool for mtmd; See mtmd-debug.md for the pytorch equivalent code\n"
"Note: we repurpose some args from other examples, they will have different meaning here\n"
"\n"
"Usage: %s -m <model> --mmproj <mmproj> -p <mode> -n <size> --image <image> --audio <audio>\n"
"\n"
" -n <size>: number of pixels per edge for image (always square image), or number of samples for audio\n"
"\n"
" -p \"encode\" (debugging encode pass, default case):\n"
" --image can be:\n"
" \"white\", \"black\", \"gray\": filled 1.0f, 0.0f and 0.5f respectively\n"
" \"cb\": checkerboard pattern, alternate 1.0f and 0.0f\n"
" --audio can be:\n"
" \"one\", \"zero\", \"half\": filled 1.0f, 0.0f and 0.5f respectively\n"
" \"1010\": checkerboard pattern, alternate 1.0f and 0.0f\n"
"\n"
" -p \"preproc\" (debugging preprocessing pass):\n"
" --image can be:\n"
" \"white\", \"black\", \"gray\": filled image with respective colors\n"
" \"cb\": checkerboard pattern\n"
" --audio can be:\n"
" \"one\", \"zero\", \"half\": filled 1.0f, 0.0f and 0.5f respectively\n"
" \"440\": sine wave with 440 Hz frequency\n"
"\n",
argv[0]
);
}
int main(int argc, char ** argv) {
std::setlocale(LC_NUMERIC, "C");
ggml_time_init();
common_params params;
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_MTMD, show_additional_info)) {
return 1;
}
common_init();
mtmd_helper_log_set(common_log_default_callback, nullptr);
if (params.mmproj.path.empty()) {
show_additional_info(argc, argv);
LOG_ERR("ERR: Missing --mmproj argument\n");
return 1;
}
LOG_INF("%s: loading model: %s\n", __func__, params.model.path.c_str());
mtmd::context_ptr ctx_mtmd;
common_init_result_ptr llama_init;
base_callback_data cb_data;
llama_init = common_init_from_params(params);
{
auto * model = llama_init->model();
const char * clip_path = params.mmproj.path.c_str();
mtmd_context_params mparams = mtmd_context_params_default();
mparams.use_gpu = params.mmproj_use_gpu;
mparams.print_timings = true;
mparams.n_threads = params.cpuparams.n_threads;
mparams.flash_attn_type = params.flash_attn_type;
mparams.warmup = params.warmup;
mparams.image_min_tokens = params.image_min_tokens;
mparams.image_max_tokens = params.image_max_tokens;
{
// always enable debug callback
mparams.cb_eval_user_data = &cb_data;
mparams.cb_eval = common_debug_cb_eval<false>;
}
ctx_mtmd.reset(mtmd_init_from_file(clip_path, model, mparams));
if (!ctx_mtmd.get()) {
LOG_ERR("Failed to load vision model from %s\n", clip_path);
exit(1);
}
}
std::string input;
int32_t inp_size = params.n_predict;
if (params.image.empty()) {
LOG_ERR("ERR: At least one of --image or --audio must be specified\n");
return 1;
}
if (inp_size <= 0) {
LOG_ERR("ERR: Invalid size specified with -n, must be greater than 0\n");
return 1;
}
input = params.image[0];
if (params.prompt.empty() || params.prompt == "encode") {
std::vector<std::vector<float>> image;
std::vector<float> samples;
if (input == "black") {
for (int i = 0; i < inp_size; ++i) {
auto row = std::vector<float>(inp_size * 3, 0.0f);
image.push_back(row);
}
} else if (input == "white") {
for (int i = 0; i < inp_size; ++i) {
auto row = std::vector<float>(inp_size * 3, 1.0f);
image.push_back(row);
}
} else if (input == "gray") {
for (int i = 0; i < inp_size; ++i) {
auto row = std::vector<float>(inp_size * 3, 0.5f);
image.push_back(row);
}
} else if (input == "cb") {
for (int i = 0; i < inp_size; ++i) {
auto row = std::vector<float>(inp_size * 3, 0.0f);
image.push_back(row);
}
for (int y = 0; y < inp_size; ++y) {
for (int x = 0; x < inp_size; ++x) {
float v = ((x + y) % 2) ? 0.0f : 1.0f;
image[y][x * 3 + 0] = v;
image[y][x * 3 + 1] = v;
image[y][x * 3 + 2] = v;
}
}
} else if (input == "one") {
samples = std::vector<float>(inp_size, 1.0f);
} else if (input == "zero") {
samples = std::vector<float>(inp_size, 0.0f);
} else if (input == "half") {
samples = std::vector<float>(inp_size, 0.5f);
} else if (input == "1010") {
samples.resize(inp_size);
for (int i = 0; i < inp_size; ++i) {
samples[i] = (i % 2) ? 0.0f : 1.0f;
}
} else {
LOG_ERR("ERR: Invalid input specified with --image/--audio\n");
show_additional_info(argc, argv);
return 1;
}
// run encode pass
LOG_INF("Running encode pass for input type: %s\n", input.c_str());
if (samples.size() > 0) {
LOG_INF("Input audio with %zu samples, type: %s\n", samples.size(), input.c_str());
mtmd_debug_encode_audio(ctx_mtmd.get(), samples);
} else {
LOG_INF("Input image with dimensions %d x %d, type: %s\n", inp_size, inp_size, input.c_str());
mtmd_debug_encode_image(ctx_mtmd.get(), image);
}
} else if (params.prompt == "preproc") {
std::vector<uint8_t> rgb_values;
std::vector<float> pcm_samples;
if (input == "black") {
rgb_values = std::vector<uint8_t>(inp_size * inp_size * 3, 0);
} else if (input == "white") {
rgb_values = std::vector<uint8_t>(inp_size * inp_size * 3, 255);
} else if (input == "gray") {
rgb_values = std::vector<uint8_t>(inp_size * inp_size * 3, 128);
} else if (input == "cb") {
rgb_values.resize(inp_size * inp_size * 3);
for (int y = 0; y < inp_size; ++y) {
for (int x = 0; x < inp_size; ++x) {
uint8_t v = ((x + y) % 2) ? 0 : 255;
rgb_values[(y * inp_size + x) * 3 + 0] = v;
rgb_values[(y * inp_size + x) * 3 + 1] = v;
rgb_values[(y * inp_size + x) * 3 + 2] = v;
}
}
} else if (input == "one") {
pcm_samples = std::vector<float>(inp_size, 1.0f);
} else if (input == "zero") {
pcm_samples = std::vector<float>(inp_size, 0.0f);
} else if (input == "half") {
pcm_samples = std::vector<float>(inp_size, 0.5f);
} else if (input == "440") {
pcm_samples.resize(inp_size);
float freq = 440.0f;
float sample_rate = mtmd_get_audio_sample_rate(ctx_mtmd.get());
float pi = 3.14159265f;
for (int i = 0; i < inp_size; ++i) {
pcm_samples[i] = sinf(2 * pi * freq * i / sample_rate);
}
} else {
LOG_ERR("ERR: Invalid input specified with --image/--audio\n");
show_additional_info(argc, argv);
return 1;
}
// run preprocessing pass
LOG_INF("Running preprocessing pass for input type: %s\n", input.c_str());
if (pcm_samples.size() > 0) {
LOG_INF("Input audio with %zu samples, type: %s\n", pcm_samples.size(), input.c_str());
mtmd_debug_preprocess_audio(ctx_mtmd.get(), pcm_samples);
} else {
LOG_INF("Input image with dimensions %d x %d, type: %s\n", inp_size, inp_size, input.c_str());
mtmd_debug_preprocess_image(ctx_mtmd.get(), rgb_values, inp_size, inp_size);
}
} else {
LOG_ERR("ERR: Invalid mode specified with -p\n");
show_additional_info(argc, argv);
return 1;
}
return 0;
}

17
tools/mtmd/debug/mtmd-debug.h Normal file
View File

@ -0,0 +1,17 @@
#pragma once
#include "mtmd.h"
#include <vector>
// INTERNAL HEADER FOR DEBUGGING PURPOSES ONLY
// NOT INTENDED FOR PUBLIC USE
// Do not raise issues related to this debugging API
// encode take the pre-processed f32 values, print the intermidiate values via cb_eval callback
MTMD_API void mtmd_debug_encode_image(mtmd_context * ctx, const std::vector<std::vector<float>> & image);
MTMD_API void mtmd_debug_encode_audio(mtmd_context * ctx, const std::vector<float> & input); // will be broadcasted to fit n_mel
// preprocess take the raw input values
MTMD_API void mtmd_debug_preprocess_image(mtmd_context * ctx, const std::vector<uint8_t> & rgb_values, int nx, int ny);
MTMD_API void mtmd_debug_preprocess_audio(mtmd_context * ctx, const std::vector<float> & pcm_samples);

25
tools/mtmd/debug/mtmd-debug.md Normal file
View File

@ -0,0 +1,25 @@
# mtmd-debug
## Debugging encode pass
Example of debugging an input gray image (raw, not preprocessed):
```py
from transformers import AutoModel
model = AutoModel.from_pretrained(...)
def test_vision():
img_size = 896 # number of patches per side
pixel_values = torch.zeros(1, 3, img_size, img_size) + 0.5 # gray image
with torch.no_grad():
outputs = model.model.get_image_features(pixel_values=pixel_values)
print("last_hidden_state shape:", outputs.last_hidden_state.shape)
print("last_hidden_state:", outputs.last_hidden_state)
test_vision()
```
## Debugging preprocess pass
(TODO)

102
tools/mtmd/mtmd.cpp
View File

@ -2,6 +2,7 @@
#include "clip-impl.h"
#include "mtmd.h"
#include "mtmd-audio.h"
#include "debug/mtmd-debug.h"
#include "llama.h"
@ -1157,3 +1158,104 @@ void mtmd_log_set(ggml_log_callback log_callback, void * user_data) {
g_logger_state.log_callback = log_callback ? log_callback : clip_log_callback_default;
g_logger_state.log_callback_user_data = user_data;
}
//
// Debugging API (NOT intended for public use)
//
static void mtmd_debug_encode_impl(mtmd_context * ctx, clip_ctx * ctx_clip, clip_image_f32 & image) {
clip_set_debug_output_embeddings(ctx_clip, true);
int n_mmproj_embd = clip_n_mmproj_embd(ctx_clip);
int n_tokens = clip_n_output_tokens(ctx_clip, &image);
std::vector<float> embd_output(n_tokens * n_mmproj_embd, 0.0f);
bool ok = clip_image_encode(
ctx_clip,
ctx->n_threads,
&image,
embd_output.data());
if (!ok) {
LOG_ERR("%s: failed to encode image\n", __func__);
}
}
void mtmd_debug_encode_image(mtmd_context * ctx, const std::vector<std::vector<float>> & image) {
if (!ctx->ctx_v) {
LOG_ERR("%s: model does not support vision input\n", __func__);
return;
}
clip_image_f32 inp_image;
inp_image.nx = image.size();
inp_image.ny = inp_image.nx;
inp_image.buf.reserve(inp_image.nx * inp_image.ny);
for (const auto & row : image) {
inp_image.buf.insert(inp_image.buf.end(), row.begin(), row.end());
}
LOG_INF("%s: created input image with nx=%d, ny=%d\n", __func__, inp_image.nx, inp_image.ny);
mtmd_debug_encode_impl(ctx, ctx->ctx_v, inp_image);
}
void mtmd_debug_encode_audio(mtmd_context * ctx, const std::vector<float> & input) {
if (!ctx->ctx_a) {
LOG_ERR("%s: model does not support audio input\n", __func__);
return;
}
int n_mel = clip_get_hparams(ctx->ctx_a)->n_mel_bins;
clip_image_f32 inp_audio;
inp_audio.nx = input.size();
inp_audio.ny = n_mel;
inp_audio.buf.resize(input.size() * n_mel);
for (size_t i = 0; i < input.size(); i++) {
for (int j = 0; j < n_mel; j++) {
inp_audio.buf[j * inp_audio.nx + i] = input[i];
}
}
LOG_INF("%s: created input audio with nx=%d, ny=%d\n", __func__, inp_audio.nx, inp_audio.ny);
mtmd_debug_encode_impl(ctx, ctx->ctx_a, inp_audio);
}
void mtmd_debug_preprocess_image(mtmd_context * ctx, const std::vector<uint8_t> & rgb_values, int nx, int ny) {
if (!ctx->ctx_v) {
LOG_ERR("%s: model does not support vision input\n", __func__);
return;
}
clip_image_u8 img_u8;
img_u8.nx = nx;
img_u8.ny = ny;
img_u8.buf = rgb_values;
clip_image_f32_batch batch_f32;
bool ok = clip_image_preprocess(ctx->ctx_v, &img_u8, &batch_f32);
if (!ok) {
LOG_ERR("%s: failed to preprocess image\n", __func__);
return;
}
LOG_INF("%s: preprocessed image to batch_f32 with %d entries\n", __func__, (int)batch_f32.entries.size());
for (size_t i = 0; i < batch_f32.entries.size(); i++) {
LOG_INF("%s: entry %zu has nx=%d, ny=%d\n", __func__, i, batch_f32.entries[i]->nx, batch_f32.entries[i]->ny);
// TODO: better way to dump entry content?
}
}
void mtmd_debug_preprocess_audio(mtmd_context * ctx, const std::vector<float> & samples) {
if (!ctx->ctx_a) {
LOG_ERR("%s: model does not support audio input\n", __func__);
return;
}
std::vector<mtmd_audio_mel> mel_spec_chunks;
bool ok = ctx->audio_preproc->preprocess(samples.data(), samples.size(), mel_spec_chunks);
if (!ok) {
LOG_ERR("%s: failed to preprocess audio\n", __func__);
return;
}
LOG_INF("%s: preprocessed audio to %zu mel spec chunks\n", __func__, mel_spec_chunks.size());
for (size_t i = 0; i < mel_spec_chunks.size(); i++) {
LOG_INF("%s: mel spec chunk %zu has n_len=%d, n_mel=%d\n", __func__, i, mel_spec_chunks[i].n_len, mel_spec_chunks[i].n_mel);
// dump mel entries: data is stored as [n_mel][n_len] (mel-major)
const auto & mel = mel_spec_chunks[i];
for (int m = 0; m < mel.n_mel; m++) {
for (int t = 0; t < mel.n_len; t++) {
LOG_INF("mel[%zu][m=%d][t=%d] = %f\n", i, m, t, mel.data[m * mel.n_len + t]);
}
}
}
}