Fixed issue with gpt2 regex custom preprocessor

2024-04-17 07:40:40 +06:00 · 2024-04-17 07:40:40 +06:00 · 36d983262e
parent 753580360b
commit 36d983262e
1 changed files with 1 additions and 0 deletions
--- a/unicode.cpp
+++ b/unicode.cpp
@ -357,6 +357,7 @@ static std::vector<size_t> unicode_gpt2_regex_preprocess(const std::wstring & wt
                token += utf_char;
            }
        }
+        start += offset;
    }

    return bpe_offsets;