diff --git a/gemma/gemma.cc b/gemma/gemma.cc index 45edbc7..d0ed6a0 100644 --- a/gemma/gemma.cc +++ b/gemma/gemma.cc @@ -171,15 +171,15 @@ class GemmaTokenizer::Impl { return spp_ && spp_->Encode(input, pieces).ok(); } - bool Encode(const std::string& input, std::vector* pieces) const { + bool Encode(const std::string& input, std::vector* ids) const { if constexpr (kShowTokenization) { - bool is_ok = spp_ && spp_->Encode(input, pieces).ok(); - for (int i = 0; i < static_cast(pieces->size()); i++) { - fprintf(stderr, "%3d: %d\n", i, (*pieces)[i]); + bool is_ok = spp_ && spp_->Encode(input, ids).ok(); + for (int i = 0; i < static_cast(ids->size()); i++) { + fprintf(stderr, "%3d: %d\n", i, (*ids)[i]); } return is_ok; } else { - return spp_ && spp_->Encode(input, pieces).ok(); + return spp_ && spp_->Encode(input, ids).ok(); } } diff --git a/gemma/gemma.h b/gemma/gemma.h index 403740f..49fe40d 100644 --- a/gemma/gemma.h +++ b/gemma/gemma.h @@ -60,7 +60,7 @@ class GemmaTokenizer { GemmaTokenizer& operator=(GemmaTokenizer&& other); bool Encode(const std::string& input, std::vector* pieces) const; - bool Encode(const std::string& input, std::vector* pieces) const; + bool Encode(const std::string& input, std::vector* ids) const; bool Decode(const std::vector& ids, std::string* detokenized) const; private: