From 4e20fcdc9510c5e11b32450225e62618bc5c36cc Mon Sep 17 00:00:00 2001 From: ylwango613 <1217816127@qq.com> Date: Sun, 4 Jan 2026 17:11:37 +0800 Subject: [PATCH 1/2] mtmd : fix integer overflow when n_tokens equals INT32_MIN --- tools/mtmd/mtmd.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index fca55b76f8..3dce7958f8 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -774,6 +774,10 @@ struct mtmd_tokenizer { int n_tokens = text.length() + 2 * add_special; std::vector result(n_tokens); n_tokens = llama_tokenize(vocab, text.data(), text.length(), result.data(), result.size(), add_special, parse_special); + // -2147483648 is std::numeric_limits::min() + if (n_tokens == -2147483648) { + throw std::runtime_error("Tokenization failed: input text too large, tokenization result exceeds int32_t limit"); + } if (n_tokens < 0) { result.resize(-n_tokens); int check = llama_tokenize(vocab, text.data(), text.length(), result.data(), result.size(), add_special, parse_special); From f32778568ab5a250632192b25c8e2a26810b81d6 Mon Sep 17 00:00:00 2001 From: ylwango613 <128395302+ylwango613@users.noreply.github.com> Date: Sun, 4 Jan 2026 19:16:35 +0800 Subject: [PATCH 2/2] Update tools/mtmd/mtmd.cpp Co-authored-by: Aaron Teo --- tools/mtmd/mtmd.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 3dce7958f8..2fb87987ce 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -774,8 +774,7 @@ struct mtmd_tokenizer { int n_tokens = text.length() + 2 * add_special; std::vector result(n_tokens); n_tokens = llama_tokenize(vocab, text.data(), text.length(), result.data(), result.size(), add_special, parse_special); - // -2147483648 is std::numeric_limits::min() - if (n_tokens == -2147483648) { + if (n_tokens == std::numeric_limits::min()) { throw std::runtime_error("Tokenization failed: input text too large, tokenization result exceeds int32_t limit"); } if (n_tokens < 0) {