feat: set proper vocab settings

2024-04-22 18:02:48 +02:00 · 2024-04-22 18:02:48 +02:00 · 71ff763e0e
parent 64cd4b1339
commit 71ff763e0e
1 changed files with 5 additions and 0 deletions
--- a/convert-hf-to-gguf.py
+++ b/convert-hf-to-gguf.py
@ -2736,6 +2736,11 @@ class JinaBertModel(BertModel):

            yield name, data

+    def set_vocab(self, *args, **kwargs):
+        super().set_vocab()
+        self.gguf_writer.add_add_bos_token(True)
+        self.gguf_writer.add_add_eos_token(True)
+

 JinaBertForMaskedML = JinaBertModel