llama : fix command-r detokenization
This commit is contained in:
parent
31a1b0eeaa
commit
d6fe269ced
|
|
@ -5407,6 +5407,7 @@ static void llm_load_vocab(
|
|||
} else if (
|
||||
tokenizer_pre == "command-r") {
|
||||
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_COMMAND_R;
|
||||
vocab.tokenizer_clean_spaces = false;
|
||||
} else if (
|
||||
tokenizer_pre == "qwen2") {
|
||||
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_QWEN2;
|
||||
|
|
|
|||
Loading…
Reference in New Issue