happyz
/
llama.cpp
mirror of https://github.com/ggerganov/llama.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

conflict solve

This commit is contained in:
ibrahimkhadraoui 2025-07-04 16:28:48 +04:00
parent 2aa48dd853
commit 9760c8bc9d
1 changed files with 0 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
convert_hf_to_gguf.py
View File

@ -607,14 +607,7 @@ class TextModel(ModelBase):
from transformers import AutoTokenizer from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(self.dir_model) tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
<<<<<<< HEAD
vocab_size = self.hparams.get("vocab_size", len(tokenizer.vocab)) vocab_size = self.hparams.get("vocab_size", len(tokenizer.vocab))
=======
vocab_size = min(
self.hparams.get("vocab_size", len(tokenizer.vocab)),
len(tokenizer.vocab)
)
>>>>>>> 250b4f1074da288399e71577a925aa83acf316d5
assert max(tokenizer.vocab.values()) < vocab_size assert max(tokenizer.vocab.values()) < vocab_size
tokpre = self.get_vocab_base_pre(tokenizer) tokpre = self.get_vocab_base_pre(tokenizer)