happyz
/
gemma.cpp
mirror of https://github.com/google/gemma.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Experiment with sentencepiece 0.2.0 

PiperOrigin-RevId: 695436079
This commit is contained in:
Paul Chang 2024-11-11 12:05:28 -08:00 committed by Copybara-Service
parent e54d9cbddd
commit d071c4f38b
3 changed files with 126 additions and 1688 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
MODULE.bazel
View File

@ -9,6 +9,7 @@ bazel_dep(name = "googletest", version = "1.15.2")
bazel_dep(name = "highway", version = "1.1.0") bazel_dep(name = "highway", version = "1.1.0")
bazel_dep(name = "nlohmann_json", version = "3.11.3") bazel_dep(name = "nlohmann_json", version = "3.11.3")
bazel_dep(name = "platforms", version = "0.0.10") bazel_dep(name = "platforms", version = "0.0.10")
bazel_dep(name = "protobuf", version = "28.3")
bazel_dep(name = "rules_cc", version = "0.0.9") bazel_dep(name = "rules_cc", version = "0.0.9")
bazel_dep(name = "rules_license", version = "0.0.7") bazel_dep(name = "rules_license", version = "0.0.7")
bazel_dep(name = "google_benchmark", version = "1.8.5") bazel_dep(name = "google_benchmark", version = "1.8.5")
@ -27,9 +28,9 @@ http_archive(
build_file = "@//bazel:sentencepiece.bazel", build_file = "@//bazel:sentencepiece.bazel",
patch_args = ["-p1"], patch_args = ["-p1"],
patches = ["@//bazel:sentencepiece.patch"], patches = ["@//bazel:sentencepiece.patch"],
sha256 = "8409b0126ebd62b256c685d5757150cf7fcb2b92a2f2b98efb3f38fc36719754", sha256 = "67c34082deb3d89f589d8614b1ae58aa4af250c79e18737d8103242ec62a28f9",
strip_prefix = "sentencepiece-0.1.96", strip_prefix = "sentencepiece-0.2.0",
urls = ["https://github.com/google/sentencepiece/archive/refs/tags/v0.1.96.zip"], urls = ["https://github.com/google/sentencepiece/archive/refs/tags/v0.2.0.zip"],
) )
# For sentencepiece # For sentencepiece

5
bazel/sentencepiece.bazel
View File

@ -66,6 +66,7 @@ cc_library(
"src/char_model.h", "src/char_model.h",
"src/filesystem.h", "src/filesystem.h",
"src/freelist.h", "src/freelist.h",
"src/init.h",
"src/model_factory.h", "src/model_factory.h",
"src/model_interface.h", "src/model_interface.h",
"src/normalizer.h", "src/normalizer.h",
@ -75,7 +76,7 @@ cc_library(
"src/util.h", "src/util.h",
"src/word_model.h", "src/word_model.h",
], ],
defines = ["_USE_TF_STRING_VIEW"], defines = ["_USE_TF_STRING_VIEW", "_USE_EXTERNAL_PROTOBUF"],
includes = [ includes = [
".", ".",
"src", "src",
@ -88,10 +89,12 @@ cc_library(
":sentencepiece_model_cc_proto", ":sentencepiece_model_cc_proto",
"@abseil-cpp//absl/container:flat_hash_map", "@abseil-cpp//absl/container:flat_hash_map",
"@abseil-cpp//absl/container:flat_hash_set", "@abseil-cpp//absl/container:flat_hash_set",
"@abseil-cpp//absl/flags:parse",
"@abseil-cpp//absl/memory", "@abseil-cpp//absl/memory",
"@abseil-cpp//absl/status", "@abseil-cpp//absl/status",
"@abseil-cpp//absl/strings", "@abseil-cpp//absl/strings",
"@abseil-cpp//absl/strings:str_format", "@abseil-cpp//absl/strings:str_format",
"@protobuf//:protobuf_lite",
"@darts_clone", "@darts_clone",
], ],
) )

1800
bazel/sentencepiece.patch
View File

File diff suppressed because it is too large Load Diff