From af8eb2fde30e6e009aca80bf8cb346b5540a123e Mon Sep 17 00:00:00 2001 From: Jan Wassenberg Date: Mon, 1 Jul 2024 09:50:27 -0700 Subject: [PATCH] Declutter gemma/ directory, move binaries to evals/ and util/. PiperOrigin-RevId: 648400795 --- BUILD.bazel | 14 +- CMakeLists.txt | 14 +- {gemma => evals}/benchmark.cc | 2 +- {gemma => evals}/benchmarks.cc | 0 {gemma => evals}/cross_entropy.cc | 2 +- {gemma => evals}/cross_entropy.h | 6 +- debug_prompt.cc => evals/debug_prompt.cc | 0 evals/gemma_test.cc | 155 +++++++++++++++++++++++ {gemma => evals}/run_mmlu.cc | 0 gemma/benchmark_helper.cc | 6 +- {gemma => util}/compress_weights.cc | 2 +- 11 files changed, 178 insertions(+), 23 deletions(-) rename {gemma => evals}/benchmark.cc (99%) rename {gemma => evals}/benchmarks.cc (100%) rename {gemma => evals}/cross_entropy.cc (99%) rename {gemma => evals}/cross_entropy.h (84%) rename debug_prompt.cc => evals/debug_prompt.cc (100%) create mode 100644 evals/gemma_test.cc rename {gemma => evals}/run_mmlu.cc (100%) rename {gemma => util}/compress_weights.cc (99%) diff --git a/BUILD.bazel b/BUILD.bazel index 88b7b0e..a4d960c 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -128,8 +128,8 @@ cc_library( cc_library( name = "cross_entropy", - srcs = ["gemma/cross_entropy.cc"], - hdrs = ["gemma/cross_entropy.h"], + srcs = ["evals/cross_entropy.cc"], + hdrs = ["evals/cross_entropy.h"], deps = [ ":common", ":gemma_lib", @@ -224,7 +224,7 @@ cc_binary( cc_binary( name = "compress_weights", - srcs = ["gemma/compress_weights.cc"], + srcs = ["util/compress_weights.cc"], deps = [ ":args", ":common", @@ -242,7 +242,7 @@ cc_binary( cc_binary( name = "single_benchmark", - srcs = ["gemma/benchmark.cc"], + srcs = ["evals/benchmark.cc"], deps = [ ":app", ":args", @@ -260,7 +260,7 @@ cc_binary( cc_binary( name = "benchmarks", - srcs = ["gemma/benchmarks.cc"], + srcs = ["evals/benchmarks.cc"], deps = [ ":benchmark_helper", "@benchmark//:benchmark", @@ -270,7 +270,7 @@ cc_binary( cc_binary( name = "debug_prompt", srcs = [ - "debug_prompt.cc", + "evals/debug_prompt.cc", ], deps = [ ":app", @@ -286,7 +286,7 @@ cc_binary( cc_binary( name = "gemma_mmlu", - srcs = ["gemma/run_mmlu.cc"], + srcs = ["evals/run_mmlu.cc"], deps = [ ":app", ":args", diff --git a/CMakeLists.txt b/CMakeLists.txt index ae4fb73..5e0a9d1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -60,14 +60,14 @@ set(SOURCES backprop/forward_scalar.h backprop/optimizer.cc backprop/optimizer.h + evals/cross_entropy.cc + evals/cross_entropy.h gemma/configs.h gemma/activations.h gemma/benchmark_helper.cc gemma/benchmark_helper.h gemma/common.cc gemma/common.h - gemma/cross_entropy.cc - gemma/cross_entropy.h gemma/gemma.cc gemma/gemma.h gemma/ops.h @@ -103,13 +103,13 @@ add_executable(gemma gemma/run.cc) target_link_libraries(gemma libgemma hwy hwy_contrib) install(TARGETS gemma DESTINATION bin) -add_executable(single_benchmark gemma/benchmark.cc) +add_executable(single_benchmark evals/benchmark.cc) target_link_libraries(single_benchmark libgemma hwy hwy_contrib nlohmann_json::nlohmann_json) -add_executable(benchmarks gemma/benchmarks.cc) +add_executable(benchmarks evals/benchmarks.cc) target_link_libraries(benchmarks libgemma hwy hwy_contrib nlohmann_json::nlohmann_json benchmark) -add_executable(debug_prompt debug_prompt.cc) +add_executable(debug_prompt evals/debug_prompt.cc) target_link_libraries(debug_prompt libgemma hwy hwy_contrib nlohmann_json::nlohmann_json) ## Tests @@ -124,7 +124,7 @@ set(GEMMA_TEST_FILES backprop/backward_scalar_test.cc backprop/optimize_test.cc gemma/ops_test.cc - gemma/gemma_test.cc + evals/gemma_test.cc ) foreach (TESTFILE IN LISTS GEMMA_TEST_FILES) @@ -145,5 +145,5 @@ endif() # GEMMA_ENABLE_TESTS ## Tools -add_executable(compress_weights gemma/compress_weights.cc) +add_executable(compress_weights util/compress_weights.cc) target_link_libraries(compress_weights libgemma hwy hwy_contrib) diff --git a/gemma/benchmark.cc b/evals/benchmark.cc similarity index 99% rename from gemma/benchmark.cc rename to evals/benchmark.cc index af05142..323cc96 100644 --- a/gemma/benchmark.cc +++ b/evals/benchmark.cc @@ -10,9 +10,9 @@ #include #include "compression/io.h" // Path +#include "evals/cross_entropy.h" #include "gemma/benchmark_helper.h" #include "gemma/common.h" -#include "gemma/cross_entropy.h" #include "gemma/gemma.h" #include "util/args.h" #include "hwy/base.h" diff --git a/gemma/benchmarks.cc b/evals/benchmarks.cc similarity index 100% rename from gemma/benchmarks.cc rename to evals/benchmarks.cc diff --git a/gemma/cross_entropy.cc b/evals/cross_entropy.cc similarity index 99% rename from gemma/cross_entropy.cc rename to evals/cross_entropy.cc index f5ae88a..627dd84 100644 --- a/gemma/cross_entropy.cc +++ b/evals/cross_entropy.cc @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "gemma/cross_entropy.h" +#include "evals/cross_entropy.h" #include #include diff --git a/gemma/cross_entropy.h b/evals/cross_entropy.h similarity index 84% rename from gemma/cross_entropy.h rename to evals/cross_entropy.h index 32436fc..202ce22 100644 --- a/gemma/cross_entropy.h +++ b/evals/cross_entropy.h @@ -13,8 +13,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef THIRD_PARTY_GEMMA_CPP_GEMMA_CROSS_ENTROPY_H_ -#define THIRD_PARTY_GEMMA_CPP_GEMMA_CROSS_ENTROPY_H_ +#ifndef THIRD_PARTY_GEMMA_CPP_EVALS_CROSS_ENTROPY_H_ +#define THIRD_PARTY_GEMMA_CPP_EVALS_CROSS_ENTROPY_H_ #include @@ -30,4 +30,4 @@ float ComputeCrossEntropy(Gemma& gemma, size_t max_tokens, } // namespace gcpp -#endif // THIRD_PARTY_GEMMA_CPP_GEMMA_CROSS_ENTROPY_H_ +#endif // THIRD_PARTY_GEMMA_CPP_EVALS_CROSS_ENTROPY_H_ diff --git a/debug_prompt.cc b/evals/debug_prompt.cc similarity index 100% rename from debug_prompt.cc rename to evals/debug_prompt.cc diff --git a/evals/gemma_test.cc b/evals/gemma_test.cc new file mode 100644 index 0000000..b885195 --- /dev/null +++ b/evals/gemma_test.cc @@ -0,0 +1,155 @@ +// Copyright 2024 Google LLC +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gemma/gemma.h" + +#include + +#include +#include + +#include "gemma/benchmark_helper.h" +#include "gemma/common.h" +#include "hwy/tests/hwy_gtest.h" + +namespace gcpp { +namespace { + +// Shared state. Requires argc/argv, so construct in main and use the same raw +// pointer approach as in benchmarks.cc. Note that the style guide forbids +// non-local static variables with dtors. +GemmaEnv* s_env = nullptr; + +class GemmaTest : public ::testing::Test { + protected: + std::string GemmaReply(const std::string& prompt) { + s_env->SetMaxGeneratedTokens(2048); + s_env->MutableConfig().temperature = 0.0f; // deterministic + s_env->MutableConfig().verbosity = 0; + // Using the turn structure worsens results. + const std::vector tokens = s_env->TokenizeAndPrependBOS(prompt); + auto [response, n] = s_env->QueryModel(tokens); + return response; + } + + void TestQuestions(const char* kQA[][2], size_t num_questions) { + if (!s_env->GetModel()) return; + for (size_t i = 0; i < num_questions; ++i) { + fprintf(stderr, "Question %zu\n\n", i + 1); + std::string response = GemmaReply(kQA[i][0]); + fprintf(stderr, "'%s'\n\n", response.c_str()); + EXPECT_TRUE(response.find(kQA[i][1]) != std::string::npos); // NOLINT + } + } +}; + +TEST_F(GemmaTest, Geography) { + static const char* kQA[][2] = { + {"What is the capital of Hungary?", "Budapest"}, + {"How many states does the US have?", "50"}, + }; + static const size_t kNum = sizeof(kQA) / sizeof(kQA[0]); + TestQuestions(kQA, kNum); +} + +TEST_F(GemmaTest, History) { + static const char* kQA[][2] = { + {"When was the battle of Hastings?", "1066"}, + }; + static const size_t kNum = sizeof(kQA) / sizeof(kQA[0]); + TestQuestions(kQA, kNum); +} + +TEST_F(GemmaTest, Arithmetic) { + static const char* kQA[][2] = { + {"what is 13 + 14?", "27"}, + {"what is 7 * 8?", "56"}, + }; + static const size_t kNum = sizeof(kQA) / sizeof(kQA[0]); + TestQuestions(kQA, kNum); +} + +static const char kJingleBells[] = R"( +Dashing through the snow +In a one-horse open sleigh +O'er the fields we go +Laughing all the way +Bells on bobtails ring +Making spirits bright +What fun it is to ride and sing +A sleighing song tonight +)"; + +// The "Hay Draft" of the Gettysburg Address. +static const char kGettysburg[] = { + "Four score and seven years ago our fathers brought forth, upon this " + "continent, a new nation, conceived in Liberty, and dedicated to the " + "proposition that all men are created equal.\n\nNow we are engaged in a " + "great civil war, testing whether that nation, or any nation, so " + "conceived, and so dedicated, can long endure. We are met here on a great " + "battlefield of that war. We have come to dedicate a portion of it as a " + "final resting place for those who here gave their lives that that nation " + "might live. It is altogether fitting and proper that we should do " + "this.\n\nBut in a larger sense we can not dedicate -- we can not " + "consecrate -- we can not hallow this ground. The brave men, living and " + "dead, who struggled, here, have consecrated it far above our poor power " + "to add or detract. The world will little note, nor long remember, what we " + "say here, but can never forget what they did here. It is for us, the " + "living, rather to be dedicated here to the unfinished work which they " + "have, thus far, so nobly carried on. It is rather for us to be here " + "dedicated to the great task remaining before us -- that from these " + "honored dead we take increased devotion to that cause for which they here " + "gave the last full measure of devotion -- that we here highly resolve " + "that these dead shall not have died in vain; that this nation shall have " + "a new birth of freedom; and that this government of the people, by the " + "people, for the people, shall not perish from the earth.\n"}; + +TEST_F(GemmaTest, CrossEntropySmall) { + if (!s_env->GetModel()) return; + static const char kSmall[] = + "The capital of Hungary is Budapest which is located in Europe."; + float entropy = s_env->CrossEntropy(kSmall); + fprintf(stderr, "per-byte entropy: %f\n", entropy); + EXPECT_LT(entropy, + (s_env->ModelType() == gcpp::Model::GEMMA_7B) ? 2.1f : 2.0f); +} + +TEST_F(GemmaTest, CrossEntropyJingleBells) { + if (!s_env->GetModel()) return; + float entropy = s_env->CrossEntropy(kJingleBells); + fprintf(stderr, "per-byte entropy: %f\n", entropy); + EXPECT_LT(entropy, + (s_env->ModelType() == gcpp::Model::GEMMA_7B) ? 0.9f : 1.8f); +} + +TEST_F(GemmaTest, CrossEntropyGettysburg) { + if (!s_env->GetModel()) return; + float entropy = s_env->CrossEntropy(kGettysburg); + fprintf(stderr, "per-byte entropy: %f\n", entropy); + EXPECT_LT(entropy, + (s_env->ModelType() == gcpp::Model::GEMMA_7B) ? 0.8f : 1.2f); +} + +} // namespace +} // namespace gcpp + +int main(int argc, char** argv) { + gcpp::GemmaEnv env(argc, argv); + gcpp::s_env = &env; + + testing::InitGoogleTest(&argc, argv); + + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/gemma/run_mmlu.cc b/evals/run_mmlu.cc similarity index 100% rename from gemma/run_mmlu.cc rename to evals/run_mmlu.cc diff --git a/gemma/benchmark_helper.cc b/gemma/benchmark_helper.cc index 68dbbee..d5c3fea 100644 --- a/gemma/benchmark_helper.cc +++ b/gemma/benchmark_helper.cc @@ -25,14 +25,14 @@ #include #include #include -#include // NOLINT +#include // NOLINT #include // std::pair #include // Placeholder for internal header, do not modify. #include "compression/compress.h" // TypeName -#include "gemma/common.h" // StringFromType -#include "gemma/cross_entropy.h" +#include "evals/cross_entropy.h" +#include "gemma/common.h" // StringFromType #include "gemma/gemma.h" #include "util/app.h" #include "util/args.h" diff --git a/gemma/compress_weights.cc b/util/compress_weights.cc similarity index 99% rename from gemma/compress_weights.cc rename to util/compress_weights.cc index 2504274..cc14c42 100644 --- a/gemma/compress_weights.cc +++ b/util/compress_weights.cc @@ -19,7 +19,7 @@ // which we pass the filename via macro 'argument'. #undef HWY_TARGET_INCLUDE #define HWY_TARGET_INCLUDE \ - "gemma/compress_weights.cc" // NOLINT + "util/compress_weights.cc" // NOLINT #include "hwy/foreach_target.h" // IWYU pragma: keep // Must come after foreach_target.h to avoid redefinition errors. #include "compression/compress-inl.h"