From 5d323c00fe6871fe5ba68afd2a995fc5667ee1f7 Mon Sep 17 00:00:00 2001
From: austinvhuang <austinh@alum.mit.edu>
Date: Sun, 10 Mar 2024 13:23:16 -0400
Subject: [PATCH] fix tokenizer scope

---
 gemma.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/gemma.cc b/gemma.cc
index 6f2e664..c9066eb 100644
--- a/gemma.cc
+++ b/gemma.cc
@@ -797,10 +797,10 @@ void GemmaImpl<ConfigGemma7B>::Generate(
 Gemma::Gemma(const Path& tokenizer_path, const Path& compressed_weights_path,
              const Path& weights_path, Model model_type,
              hwy::ThreadPool& pool) {
+  std::unique_ptr<sentencepiece::SentencePieceProcessor> tokenizer;
   {
     PROFILER_ZONE("Startup.tokenizer");
-    std::unique_ptr<sentencepiece::SentencePieceProcessor> tokenizer =
-        std::make_unique<sentencepiece::SentencePieceProcessor>();
+    tokenizer = std::make_unique<sentencepiece::SentencePieceProcessor>();
     if (!tokenizer->Load(tokenizer_path.path).ok()) {
       HWY_ABORT("Failed to load the tokenizer file.");
     }