happyz
/
llama.cpp
mirror of https://github.com/ggerganov/llama.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

vocab : add byte token handling to BPE detokenizer for Gemma4 (#21488)

This commit is contained in:
Aldehir Rojas 2026-04-06 09:08:37 -05:00 committed by GitHub
parent 941146b3f1
commit 4aa962e2b0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/llama-vocab.cpp
View File

@ -2813,7 +2813,9 @@ uint8_t llama_vocab::impl::token_to_byte(llama_token id) const {
return strtol(buf.c_str(), NULL, 16);
}
case LLAMA_VOCAB_TYPE_BPE: {
GGML_ABORT("fatal error");
// Gemma4 uses BPE with SPM-style byte fallback tokens (<0xXX>)
auto buf = token_data.text.substr(3, 2);
return strtol(buf.c_str(), NULL, 16);
}
case LLAMA_VOCAB_TYPE_WPM: {
GGML_ABORT("fatal error");
@ -3294,6 +3296,10 @@ int32_t llama_vocab::impl::token_to_piece(llama_token token, char * buf, int32_t
std::string result = llama_decode_text(token_text);
return _try_copy(result.data(), result.size());
}
if (attr & LLAMA_TOKEN_ATTR_BYTE) {
char byte = (char) token_to_byte(token);
return _try_copy((char*) &byte, 1);
}
break;
}
case LLAMA_VOCAB_TYPE_RWKV: {