Fix default value for WPM special_add_eos

Default values for special_add_bos/eos
Allow lstrip for 'added_tokens'
2024-05-26 01:11:53 +02:00 · 2024-05-25 23:17:09 +02:00 · 2024-05-25 21:45:32 +02:00 · 2024-05-25 21:39:50 +02:00 · 2024-05-25 21:30:26 +02:00
1 changed files with 59 additions and 40 deletions
--- a/llama.cpp
+++ b/llama.cpp
@ -2072,16 +2072,18 @@ struct llama_vocab {
    id special_cls_id  = -1;
    id special_mask_id = -1;

-    int special_add_bos = -1; // -1 unknown, 1 add, 0 don't add.
-    int special_add_eos = -1; // -1 unknown, 1 add, 0 don't add.
-
    id linefeed_id       = 13;
    id special_prefix_id = -1;
    id special_suffix_id = -1;
    id special_middle_id = -1;
    id special_eot_id    = -1; // TODO: move above after "eos_id", and here add "file separator" token

-    bool add_space_prefix = true;
+    // tokenizer flags
+    bool tokenizer_add_space_prefix = true;
+    bool tokenizer_special_add_bos  = false;
+    bool tokenizer_special_add_eos  = false;
+    bool tokenizer_ignore_merges    = false;
+    bool tokenizer_mask_lstrip      = false;

    int find_bpe_rank(const std::string & token_left, const std::string & token_right) const {
        GGML_ASSERT(token_left.find(' ') == std::string::npos);
@ -4435,7 +4437,7 @@ static void llm_load_vocab(

            const int add_space_prefix_keyidx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_ADD_PREFIX).c_str());
            if (add_space_prefix_keyidx != -1) {
-                vocab.add_space_prefix = gguf_get_val_bool(ctx, add_space_prefix_keyidx);
+                vocab.tokenizer_add_space_prefix = gguf_get_val_bool(ctx, add_space_prefix_keyidx);
            } // The default value of add_space_prefix is true.
        } else if (tokenizer_model == "bert") {
            vocab.type = LLAMA_VOCAB_TYPE_WPM;
@ -4448,7 +4450,7 @@ static void llm_load_vocab(
            vocab.special_pad_id  = 0;
            vocab.special_cls_id  = 101;
            vocab.special_mask_id = 103;
-            vocab.add_space_prefix = false;
+            vocab.tokenizer_add_space_prefix = false;
        } else {
            if (tokenizer_model == "gpt2") {
                vocab.type = LLAMA_VOCAB_TYPE_BPE;
@ -4512,6 +4514,8 @@ static void llm_load_vocab(
                    tokenizer_pre == "llama-v3" ||
                    tokenizer_pre == "llama-bpe") {
                vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_LLAMA3;
+                vocab.tokenizer_ignore_merges = true;
+                vocab.tokenizer_special_add_bos = true;
            } else if (
                    tokenizer_pre == "deepseek-llm") {
                vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM;
@ -4534,6 +4538,7 @@ static void llm_load_vocab(
                    tokenizer_pre == "jina-v2-es" ||
                    tokenizer_pre == "jina-v2-de") {
                vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_GPT2;
+                vocab.tokenizer_mask_lstrip = tokenizer_pre.find("jina-v2") < std::string::npos;
            } else if (
                    tokenizer_pre == "refact") {
                vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_REFACT;
@ -4555,8 +4560,16 @@ static void llm_load_vocab(
            } else {
                throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str()));
            }
-        } else {
+        } else if (vocab.type == LLAMA_VOCAB_TYPE_SPM) {
            vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
+            vocab.tokenizer_special_add_bos = true;
+            vocab.tokenizer_special_add_eos = false;
+        } else if (vocab.type == LLAMA_VOCAB_TYPE_WPM) {
+            vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
+            vocab.tokenizer_special_add_bos = true;
+            vocab.tokenizer_special_add_eos = false;
+        } else {
+            throw std::runtime_error(format("unknown vocab type: '%d'", (int) vocab.type));
        }
    }

@ -4647,10 +4660,10 @@ static void llm_load_vocab(
            bool temp = true;

            if (ml.get_key(LLM_KV_TOKENIZER_ADD_BOS, temp, false)) {
-                vocab.special_add_bos = int(temp);
+                vocab.tokenizer_special_add_bos = temp;
            }
            if (ml.get_key(LLM_KV_TOKENIZER_ADD_EOS, temp, false)) {
-                vocab.special_add_eos = int(temp);
+                vocab.tokenizer_special_add_eos = temp;
            }
        }

@ -12274,12 +12287,8 @@ struct llm_bigram_bpe {
 struct llm_tokenizer_bpe {
    llm_tokenizer_bpe(const llama_vocab & vocab): vocab(vocab) {
        GGML_ASSERT(vocab.type == LLAMA_VOCAB_TYPE_BPE);
-        special_add_bos = vocab.special_add_bos == 1;
-        special_add_eos = vocab.special_add_eos == 1;
        switch (vocab.type_pre) {
            case LLAMA_VOCAB_PRE_TYPE_LLAMA3:
-                special_add_bos = true;
-                ignore_merges = true;
                regex_exprs = {
                    // original regex from tokenizer.json
                    //"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
@ -12365,8 +12374,12 @@ struct llm_tokenizer_bpe {
        }
    }

-    bool add_special_bos(std::vector<llama_vocab::id> & output) const {
-        if (special_add_bos) {
+    void append(const llama_vocab::id token_id, std::vector<llama_vocab::id> & output) const {
+        output.push_back(token_id);
+    }
+
+    bool append_bos(std::vector<llama_vocab::id> & output) const {
+        if (vocab.tokenizer_special_add_bos) {
            GGML_ASSERT(vocab.special_bos_id != -1);
            output.push_back(vocab.special_bos_id);
            return true;
@ -12374,8 +12387,8 @@ struct llm_tokenizer_bpe {
        return false;
    }

-    bool add_special_eos(std::vector<llama_vocab::id> & output) const {
-        if (special_add_eos) {
+    bool append_eos(std::vector<llama_vocab::id> & output) const {
+        if (vocab.tokenizer_special_add_eos) {
            GGML_ASSERT(vocab.special_eos_id != -1);
            output.push_back(vocab.special_eos_id);
            return true;
@ -12383,14 +12396,14 @@ struct llm_tokenizer_bpe {
        return false;
    }

-    void check_add_special(const std::vector<llama_vocab::id> & output) const {
-        if (special_add_bos && output.size() >= 2 && output[1] == vocab.special_bos_id) {
+    void check_double_bos_eos(const std::vector<llama_vocab::id> & output) const {
+        if (vocab.tokenizer_special_add_bos && output.size() >= 2 && output[1] == vocab.special_bos_id) {
            LLAMA_LOG_WARN(
                "%s: Added a BOS token to the prompt as specified by the model but the prompt "
                "also starts with a BOS token. So now the final prompt starts with 2 BOS tokens. "
                "Are you sure this is what you want?\n", __FUNCTION__);
        }
-        if (special_add_eos && output.size() >= 2 && *(output.end()-2) == vocab.special_eos_id) {
+        if (vocab.tokenizer_special_add_eos && output.size() >= 2 && *(output.end()-2) == vocab.special_eos_id) {
            LLAMA_LOG_WARN(
                "%s: Added a EOS token to the prompt as specified by the model but the prompt "
                "also ends with a EOS token. So now the final prompt ends with 2 EOS tokens. "
@ -12412,7 +12425,7 @@ struct llm_tokenizer_bpe {
            int index = 0;
            size_t offset = 0;

-            if (ignore_merges && vocab.token_to_id.find(word) != vocab.token_to_id.end()) {
+            if (vocab.tokenizer_ignore_merges && vocab.token_to_id.find(word) != vocab.token_to_id.end()) {
                symbols.emplace_back(llm_symbol{-1, -1, word.c_str(), word.size()});
                offset = word.size();
            }
@ -12536,9 +12549,6 @@ private:
    const llama_vocab & vocab;

    std::vector<std::string> regex_exprs;
-    bool special_add_bos = false;
-    bool special_add_eos = false;
-    bool ignore_merges = false;

    std::vector<llm_symbol> symbols;
    std::vector<llm_symbol> symbols_final;
@ -12721,7 +12731,7 @@ static void tokenizer_st_partition(const llama_vocab & vocab, std::forward_list<

            // if a fragment is text ( not yet processed )
            if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {
-                auto * raw_text = &(fragment.raw_text);
+                auto & raw_text = fragment.raw_text;

                auto raw_text_base_offset = fragment.offset;
                auto raw_text_base_length = fragment.length;
@ -12731,7 +12741,7 @@ static void tokenizer_st_partition(const llama_vocab & vocab, std::forward_list<
                    // find the first occurrence of a given special token in this fragment
                    //  passing offset argument only limit the "search area" but match coordinates
                    //  are still relative to the source full raw_text
-                    auto match = raw_text->find(special_token, raw_text_base_offset);
+                    auto match = raw_text.find(special_token, raw_text_base_offset);

                    // no occurrences found, stop processing this fragment for a given special token
                    if (match == std::string::npos) break;
@ -12749,13 +12759,22 @@ static void tokenizer_st_partition(const llama_vocab & vocab, std::forward_list<
                    if (match > raw_text_base_offset) {
                        // left
                        const int64_t left_reminder_offset = raw_text_base_offset + 0;
-                        const int64_t left_reminder_length = match - raw_text_base_offset;
-                        buffer.emplace_after(it, (*raw_text), left_reminder_offset, left_reminder_length);
+                        int64_t left_reminder_length = match - raw_text_base_offset;
+
+                        if (vocab.tokenizer_mask_lstrip && special_id == vocab.special_mask_id) {  //TODO: generalize, this only checks special_mask_id
+                            while (left_reminder_length > 0 && isspace(raw_text[left_reminder_offset + left_reminder_length - 1])) {
+                                left_reminder_length--;
+                            }
+                        }
+
+                        if (left_reminder_length > 0) {
+                            buffer.emplace_after(it, raw_text, left_reminder_offset, left_reminder_length);
+                            it++;
+                        }

 #ifdef PRETOKENIZERDEBUG
                        LLAMA_LOG_WARN("FL: (%ld %ld) '%s'\n", left_reminder_offset, left_reminder_length, raw_text->substr(left_reminder_offset, left_reminder_length).c_str());
 #endif
-                        it++;
                    }

                    // special token
@ -12766,7 +12785,7 @@ static void tokenizer_st_partition(const llama_vocab & vocab, std::forward_list<
                    if (match + special_token.length() < raw_text_base_offset + raw_text_base_length) {
                        const int64_t right_reminder_offset = match + special_token.length();
                        const int64_t right_reminder_length = raw_text_base_length - ((match - raw_text_base_offset) + special_token.length());
-                        buffer.emplace_after(it, (*raw_text), right_reminder_offset, right_reminder_length);
+                        buffer.emplace_after(it, raw_text, right_reminder_offset, right_reminder_length);

 #ifdef PRETOKENIZERDEBUG
                        LLAMA_LOG_WARN("FR: (%ld %ld) '%s'\n", right_reminder_offset, right_reminder_length, raw_text->substr(right_reminder_offset, right_reminder_length).c_str());
@ -12823,7 +12842,7 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &
                bool is_prev_special = false;
                bool special_token_rtrim = false;

-                if (add_special && vocab.special_add_bos != 0) {
+                if (add_special && vocab.tokenizer_special_add_bos) {
                    GGML_ASSERT(vocab.special_bos_id != -1);
                    output.push_back(vocab.special_bos_id);
                    is_prev_special = true;
@ -12850,7 +12869,7 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &
                            raw_text = raw_text.substr(num_whitespaces);
                        }

-                        if (vocab.add_space_prefix) {
+                        if (vocab.tokenizer_add_space_prefix) {
                            if (!output.size() || is_prev_special) {  // prefix with space if first token
                                raw_text = " " + raw_text;
                            }
@ -12873,14 +12892,14 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &
                    }
                }

-                if (add_special && vocab.special_add_bos != 0 && output.size() >= 2 && output[1] == vocab.special_bos_id) {
+                if (add_special && vocab.tokenizer_special_add_bos && output.size() >= 2 && output[1] == vocab.special_bos_id) {
                    LLAMA_LOG_WARN(
                        "%s: Added a BOS token to the prompt as specified by the model but the prompt "
                        "also starts with a BOS token. So now the final prompt starts with 2 BOS tokens. "
                        "Are you sure this is what you want?\n", __FUNCTION__);
                }

-                if (add_special && vocab.special_add_eos == 1) {
+                if (add_special && vocab.tokenizer_special_add_eos) {
                    GGML_ASSERT(vocab.special_eos_id != -1);
                    output.push_back(vocab.special_eos_id);
                }
@ -12890,7 +12909,7 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &
                llm_tokenizer_bpe tokenizer(vocab);

                if (add_special) {
-                    tokenizer.add_special_bos(output);
+                    tokenizer.append_bos(output);
                }

                for (const auto & fragment : fragment_buffer) {
@ -12902,13 +12921,13 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &
 #endif
                        tokenizer.tokenize(raw_text, output);
                    } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)
-                        output.push_back(fragment.token);
+                        tokenizer.append(fragment.token, output);
                    }
                }

                if (add_special) {
-                    tokenizer.add_special_eos(output);
-                    tokenizer.check_add_special(output);
+                    tokenizer.append_eos(output);
+                    tokenizer.check_double_bos_eos(output);
                }
            } break;
        case LLAMA_VOCAB_TYPE_WPM:
@ -17690,11 +17709,11 @@ llama_token llama_token_nl(const struct llama_model * model) {
 }

 int32_t llama_add_bos_token(const struct llama_model * model) {
-    return model->vocab.special_add_bos;
+    return model->vocab.tokenizer_special_add_bos;
 }

 int32_t llama_add_eos_token(const struct llama_model * model) {
-    return model->vocab.special_add_eos;
+    return model->vocab.tokenizer_special_add_eos;
 }

 llama_token llama_token_prefix(const struct llama_model * model) {
Author	SHA1	Message	Date
jaime-m-p	7a5578f211	Fix default value for WPM special_add_eos	2024-05-26 01:11:53 +02:00
jaime-m-p	f84b04f1be	Default values for special_add_bos/eos	2024-05-25 23:17:09 +02:00
jaime-m-p	615f425aab	Allow lstrip for 'added_tokens' For now, only <mask> token, needed for 'jina-v2'.	2024-05-25 21:45:32 +02:00
jaime-m-p	c83ea1a1f8	Move tokenizer flags to vocab structure. Initialize values when loading the model vocab.	2024-05-25 21:39:50 +02:00
jaime-m-p	1d2f3ad471	Better name functions to append token/bos/eos	2024-05-25 21:30:26 +02:00