happyz
/
llama.cpp
mirror of https://github.com/ggerganov/llama.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

chore: remove some missed traces

This commit is contained in:
suhyun-hwang 2026-01-14 23:48:58 +09:00
parent 56c89a1216
commit ad04d34047
1 changed files with 3 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/llama-vocab.cpp
View File

@ -525,7 +525,6 @@ struct llm_tokenizer_bpe_session {
void tokenize(const std::string & text, std::vector<llama_token> & output) {
int final_prev_index = -1;
const auto word_collection = unicode_regex_split(text, tokenizer.regex_exprs);
symbols_final.clear();
@ -617,9 +616,9 @@ struct llm_tokenizer_bpe_session {
if (token == LLAMA_TOKEN_NULL) {
for (auto j = str.begin(); j != str.end(); ++j) {
std::string byte_str(1, *j);
auto token_byte = vocab.text_to_token(byte_str);
if (token_byte != LLAMA_TOKEN_NULL) {
output.push_back(token_byte);
auto token_multibyte = vocab.text_to_token(byte_str);
if (token_multibyte != LLAMA_TOKEN_NULL) {
output.push_back(token_multibyte);
}
}
} else {