Merge c4107cc9e7 into 38f1ea9b80

Move this function outside of HWY_NAMESPACE since it doesn't need to be
optimized for any particular architecture.

PiperOrigin-RevId: 626098641

gemma/gemma.cc

@@ -550,6 +550,13 @@ struct GemmaImpl : public GemmaInterface {
   hwy::AlignedUniquePtr<Activations<Config, 1>> state;
 };
 
+template <class TConfig>
+std::string TokenString(GemmaImpl<TConfig>& gemma, int token) {
+  std::string token_str;
+  gemma.Tokenizer()->Decode({token}, &token_str);
+  return "'" + std::regex_replace(token_str, std::regex("\n"), "\\n") + "'";
+}
+
 }  // namespace gcpp
 #endif  // GEMMA_ONCE
 
@@ -1123,13 +1130,6 @@ void GenerateImpl(GemmaImpl<TConfig>& gemma, size_t max_tokens,
   }
 }
 
-template <class TConfig>
-std::string TokenString(GemmaImpl<TConfig>& gemma, int token) {
-  std::string token_str;
-  gemma.Tokenizer()->Decode({token}, &token_str);
-  return "'" + std::regex_replace(token_str, std::regex("\n"), "\\n") + "'";
-}
-
 #define TOKEN(token_id) TokenString(gemma, token_id).c_str()
 
 template <class TConfig>