convert : revert "added_tokens_decoder" change

2023-12-16 16:05:35 +02:00 · 2023-12-16 16:05:35 +02:00 · 0b6ffa580c
parent a878be4cb1
commit 0b6ffa580c
1 changed files with 1 additions and 1 deletions
--- a/convert-hf-to-gguf.py
+++ b/convert-hf-to-gguf.py
@ -249,7 +249,7 @@ class Model:
                toktypes.append(gguf.TokenType.USER_DEFINED)
            elif reverse_vocab[i] in added_vocab:
                tokens.append(reverse_vocab[i])
-                if hasattr(tokenizer, "added_tokens_decoder") and tokenizer.added_tokens_decoder[i].special:
+                if tokenizer.added_tokens_decoder[i].special:
                    toktypes.append(gguf.TokenType.CONTROL)
                else:
                    toktypes.append(gguf.TokenType.USER_DEFINED)