happyz
/
llama.cpp
mirror of https://github.com/ggerganov/llama.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

vocab: add Falcon-H1-Tiny-Coder FIM tokens (#19249)

This commit is contained in:
Alexey Dubrov 2026-02-03 09:31:01 +03:00 committed by GitHub
parent aeb827a3cc
commit 1efb5f7ae1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 3 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/llama-vocab.cpp
View File

@ -2262,6 +2262,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
|| t.first == "<PRE>" || t.first == "<PRE>"
|| t.first == "▁<PRE>" // CodeLlama || t.first == "▁<PRE>" // CodeLlama
|| t.first == "<|code_prefix|>" // GLM-4.5 || t.first == "<|code_prefix|>" // GLM-4.5
|| t.first == "<|prefix|>" // Falcon-H1-Tiny-Coder
) { ) {
special_fim_pre_id = t.second; special_fim_pre_id = t.second;
if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) { if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
@ -2282,6 +2283,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
|| t.first == "<SUF>" || t.first == "<SUF>"
|| t.first == "▁<SUF>" // CodeLlama || t.first == "▁<SUF>" // CodeLlama
|| t.first == "<|code_suffix|>" // GLM-4.5 || t.first == "<|code_suffix|>" // GLM-4.5
|| t.first == "<|suffix|>" // Falcon-H1-Tiny-Coder
) { ) {
special_fim_suf_id = t.second; special_fim_suf_id = t.second;
if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) { if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
@ -2302,6 +2304,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
|| t.first == "<MID>" || t.first == "<MID>"
|| t.first == "▁<MID>" // CodeLlama || t.first == "▁<MID>" // CodeLlama
|| t.first == "<|code_middle|>" // GLM-4.5 || t.first == "<|code_middle|>" // GLM-4.5
|| t.first == "<|middle|>" // Falcon-H1-Tiny-Coder
) { ) {
special_fim_mid_id = t.second; special_fim_mid_id = t.second;
if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) { if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {