happyz
/
llama.cpp
mirror of https://github.com/ggerganov/llama.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update src/llama-vocab.cpp 

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>
This commit is contained in:
Ryan Mangeno 2025-10-10 11:47:39 -04:00 committed by GitHub
parent 97e1de457c
commit 4187cf5a94
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/llama-vocab.cpp
View File

@ -2505,8 +2505,8 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
for (const auto * token : {"<unk>", "<s>", "<|endoftext|>"}) { for (const auto * token : {"<unk>", "<s>", "<|endoftext|>"}) {
_set_token_attr(token, LLAMA_TOKEN_ATTR_RSTRIP, false); _set_token_attr(token, LLAMA_TOKEN_ATTR_RSTRIP, false);
} }
} else if ( _contains_any(model_name, {"modern-bert"})) { } else if (_contains_any(model_name, {"modern-bert"})) {
if ( token_to_id.count("MASK") == 0 ) { if (token_to_id.count("[MASK]") == 0 ) {
LLAMA_LOG_WARN("%s: Mask token missing in vocab!\n", __func__); LLAMA_LOG_WARN("%s: Mask token missing in vocab!\n", __func__);
} }
else { else {