Move fields, io* and blob* from compression/ into io/

PiperOrigin-RevId: 755445712

BUILD.bazel

@@ -37,7 +37,7 @@ cc_library(
     hdrs = ["util/args.h"],
     deps = [
         ":basics",
-        "//compression:io",  # Path
+        "//io",  # Path
         "@highway//:hwy",
     ],
 )
@@ -131,8 +131,8 @@ cc_library(
     hdrs = ["gemma/configs.h"],
     deps = [
         ":basics",
-        "//compression:fields",
         "//compression:shared",
+        "//io:fields",
         "@highway//:hwy",  # base.h
     ],
 )
@@ -143,8 +143,8 @@ cc_test(
     deps = [
         ":configs",
         "@googletest//:gtest_main",  # buildcleaner: keep
-        "//compression:fields",
         "//compression:shared",
+        "//io:fields",
     ],
 )
 
@@ -168,8 +168,8 @@ cc_library(
         ":basics",
         ":tensor_info",
         ":threading_context",
-        "//compression:fields",
         "//compression:shared",
+        "//io:fields",
         "@highway//:hwy",
         "@highway//:profiler",
         "@highway//:thread_pool",
@@ -200,10 +200,10 @@ cc_library(
         ":tensor_info",
         ":threading_context",
         ":tokenizer",
-        "//compression:blob_store",
-        "//compression:fields",
-        "//compression:io",
         "//compression:shared",
+        "//io",
+        "//io:blob_store",
+        "//io:fields",
         "@highway//:hwy",
         "@highway//:thread_pool",
     ],
@@ -218,8 +218,8 @@ cc_library(
         ":mat",
         ":model_store",
         ":tensor_info",
-        "//compression:blob_store",
         "//compression:compress",
+        "//io:blob_store",
         "@highway//:hwy",
         "@highway//:profiler",
         "@highway//:stats",
@@ -426,7 +426,7 @@ cc_library(
         ":args",
         ":basics",
         ":ops",  # matmul.h
-        "//compression:io",
+        "//io",
         "@highway//:hwy",
     ],
 )
@@ -467,8 +467,8 @@ cc_library(
         ":threading_context",
         ":weights",
         # Placeholder for internal dep, do not remove.,
-        "//compression:blob_store",
+        "//io:blob_store",
-        "//compression:io",
+        "//io",
         "//compression:shared",
         "//paligemma:image",
         "@highway//:hwy",
@@ -603,7 +603,7 @@ cc_binary(
         ":benchmark_helper",
         ":cross_entropy",
         ":gemma_lib",
-        "//compression:io",
+        "//io",
         "@highway//:hwy",
         "@highway//:nanobenchmark",
         "@nlohmann_json//:json",
@@ -630,7 +630,7 @@ cc_binary(
         ":args",
         ":benchmark_helper",
         ":gemma_lib",
-        "//compression:io",
+        "//io",
         "@highway//:hwy",
         "@nlohmann_json//:json",
     ],
@@ -643,7 +643,7 @@ cc_binary(
         ":args",
         ":benchmark_helper",
         ":gemma_lib",
-        "//compression:io",
+        "//io",
         "@highway//:hwy",
         "@highway//:profiler",
         "@nlohmann_json//:json",

CMakeLists.txt

@@ -41,32 +41,25 @@ FetchContent_MakeAvailable(benchmark)
 
 # Base source files
 set(SOURCES
-  compression/blob_store.cc
+  backprop/activations.h
-  compression/blob_store.h
+  backprop/backward_scalar.h
+  backprop/backward-inl.h
+  backprop/backward.cc
+  backprop/backward.h
+  backprop/common_scalar.h
+  backprop/forward_scalar.h
+  backprop/forward-inl.h
+  backprop/forward.cc
+  backprop/forward.h
+  backprop/optimizer.cc
+  backprop/optimizer.h
+  compression/compress-inl.h
   compression/compress.cc
   compression/compress.h
-  compression/compress-inl.h
-  compression/fields.cc
-  compression/fields.h
-  compression/io_win.cc
-  compression/io.cc
-  compression/io.h
   compression/nuq-inl.h
   compression/sfp-inl.h
   compression/shared.h
   compression/test_util-inl.h
-  backprop/activations.h
-  backprop/backward.cc
-  backprop/backward.h
-  backprop/backward-inl.h
-  backprop/backward_scalar.h
-  backprop/common_scalar.h
-  backprop/forward.cc
-  backprop/forward.h
-  backprop/forward-inl.h
-  backprop/forward_scalar.h
-  backprop/optimizer.cc
-  backprop/optimizer.h
   evals/benchmark_helper.cc
   evals/benchmark_helper.h
   evals/cross_entropy.cc
@@ -94,6 +87,13 @@ set(SOURCES
   gemma/tokenizer.h
   gemma/weights.cc
   gemma/weights.h
+  io/blob_store.cc
+  io/blob_store.h
+  io/fields.cc
+  io/fields.h
+  io/io_win.cc
+  io/io.cc
+  io/io.h
   ops/dot-inl.h
   ops/matmul-inl.h
   ops/matmul.cc
@@ -110,10 +110,10 @@ set(SOURCES
   util/mat.cc
   util/mat.h
   util/test_util.h
-  util/threading.cc
-  util/threading.h
   util/threading_context.cc
   util/threading_context.h
+  util/threading.cc
+  util/threading.h
   util/topology.cc
   util/topology.h
 )
@@ -200,14 +200,14 @@ include(GoogleTest)
 set(GEMMA_TEST_FILES
   backprop/backward_test.cc
   backprop/optimize_test.cc
-  compression/blob_store_test.cc
   compression/compress_test.cc
   compression/distortion_test.cc
-  compression/fields_test.cc
   compression/nuq_test.cc
   compression/sfp_test.cc
   evals/gemma_test.cc
   gemma/tensor_info_test.cc
+  io/blob_store_test.cc
+  io/fields_test.cc
   ops/bench_matmul.cc
   ops/dot_test.cc
   ops/gemma_matvec_test.cc
@@ -240,5 +240,5 @@ endif()  # GEMMA_ENABLE_TESTS
 
 ## Tools
 
-add_executable(migrate_weights compression/migrate_weights.cc)
+add_executable(migrate_weights io/migrate_weights.cc)
 target_link_libraries(migrate_weights libgemma hwy hwy_contrib)

compression/BUILD.bazel

@@ -1,4 +1,4 @@
-# Weight compression, I/O and analysis
+# Weight compression and analysis.
 
 package(
     default_applicable_licenses = [
@@ -20,77 +20,6 @@ config_setting(
     visibility = ["//visibility:private"],
 )
 
-FILE_DEPS = select({
-    "//conditions:default": [
-        # Placeholder for io deps, do not remove
-    ],
-    ":android": [],
-    # Placeholder for internal build rules, do not remove
-})
-
-cc_library(
-    name = "io",
-    srcs = [
-        "io.cc",
-        # Placeholder for io backend, do not remove
-    ],
-    hdrs = ["io.h"],
-    local_defines = select({
-        # Placeholder for internal build rules, do not remove
-        "//conditions:default": [],
-    }),
-    deps = [
-        "//:allocator",
-        "@highway//:hwy",
-    ] + FILE_DEPS,
-)
-
-cc_library(
-    name = "fields",
-    srcs = ["fields.cc"],
-    hdrs = ["fields.h"],
-    deps = [
-        "@highway//:hwy",
-    ],
-)
-
-cc_test(
-    name = "fields_test",
-    srcs = ["fields_test.cc"],
-    deps = [
-        ":fields",
-        "@googletest//:gtest_main",  # buildcleaner: keep
-        "@highway//:hwy_test_util",
-    ],
-)
-
-cc_library(
-    name = "blob_store",
-    srcs = ["blob_store.cc"],
-    hdrs = ["blob_store.h"],
-    deps = [
-        ":io",
-        "//:basics",
-        "//:threading_context",
-        "@highway//:hwy",
-        "@highway//:profiler",
-        "@highway//:thread_pool",
-    ],
-)
-
-cc_test(
-    name = "blob_store_test",
-    srcs = ["blob_store_test.cc"],
-    deps = [
-        ":blob_store",
-        ":io",
-        "@googletest//:gtest_main",  # buildcleaner: keep
-        "//:basics",
-        "//:threading_context",
-        "@highway//:hwy_test_util",
-    ],
-)
-
 cc_library(
     name = "distortion",
     hdrs = [
@@ -262,39 +191,3 @@ cc_library(
         "@highway//hwy/contrib/sort:vqsort",
     ],
 )
-
-cc_library(
-    name = "io_win",
-    srcs = ["io_win.cc"],
-    deps = [
-        ":io",
-        "//:allocator",
-        "@highway//:hwy",
-    ],
-)
-
-cc_binary(
-    name = "blob_compare",
-    srcs = ["blob_compare.cc"],
-    deps = [
-        ":blob_store",
-        ":io",
-        "//:basics",
-        "//:threading",
-        "//:threading_context",
-        "@highway//:hwy",
-        "@highway//:hwy_test_util",
-        "@highway//:nanobenchmark",
-        "@highway//:thread_pool",
-    ],
-)
-
-cc_binary(
-    name = "migrate_weights",
-    srcs = ["migrate_weights.cc"],
-    deps = [
-        "//:args",
-        "//:benchmark_helper",
-        "//:gemma_lib",
-    ],
-)

compression/python/BUILD.bazel

@@ -21,9 +21,9 @@ cc_library(
         "//:tensor_info",
         "//:threading_context",
         "//:tokenizer",
-        "//compression:blob_store",
         "//compression:compress",
-        "//compression:io",
+        "//io",
+        "//io:blob_store",
         "@highway//:hwy",
         "@highway//:thread_pool",
     ],

compression/python/compression_clif_aux.cc

@@ -22,13 +22,13 @@
 #include <string>
 #include <vector>
 
-#include "compression/blob_store.h"  // BlobWriter
+#include "compression/compress.h"  // ScaleWeights
-#include "compression/compress.h"    // ScaleWeights
+#include "gemma/configs.h"         // ModelConfig
-#include "compression/io.h"          // Path
+#include "gemma/model_store.h"     // ModelStore
-#include "gemma/configs.h"           // ModelConfig
+#include "gemma/tensor_info.h"     // TensorInfo
-#include "gemma/model_store.h"       // ModelStore
-#include "gemma/tensor_info.h"       // TensorInfo
 #include "gemma/tokenizer.h"
+#include "io/blob_store.h"  // BlobWriter
+#include "io/io.h"          // Path
 #include "util/basics.h"
 #include "util/mat.h"
 #include "util/threading_context.h"

compression/python/compression_clif_aux.h

@@ -21,11 +21,11 @@
 #include <memory>
 #include <string>
 
-#include "compression/blob_store.h"
 #include "compression/shared.h"  // Type
 #include "gemma/configs.h"
 #include "gemma/model_store.h"
 #include "gemma/tensor_info.h"
+#include "io/blob_store.h"
 #include "util/mat.h"
 #include "hwy/aligned_allocator.h"  // Span
 

evals/benchmark.cc

@@ -8,10 +8,10 @@
 #include <string>
 #include <vector>
 
-#include "compression/io.h"  // Path
 #include "evals/benchmark_helper.h"
 #include "evals/cross_entropy.h"
 #include "gemma/gemma.h"
+#include "io/io.h"  // Path
 #include "util/args.h"
 #include "hwy/base.h"
 #include "hwy/timer.h"

evals/debug_prompt.cc

@@ -18,9 +18,9 @@
 #include <string>
 #include <vector>
 
-#include "compression/io.h"
 #include "evals/benchmark_helper.h"
 #include "gemma/gemma.h"  // LayersOutputFunc
+#include "io/io.h"
 #include "util/args.h"
 #include "hwy/base.h"
 #include "nlohmann/json.hpp"

evals/run_mmlu.cc

@@ -19,9 +19,9 @@
 #include <string>
 #include <vector>
 
-#include "compression/io.h"  // Path
 #include "evals/benchmark_helper.h"
 #include "gemma/gemma.h"  // Gemma
+#include "io/io.h"        // Path
 #include "util/args.h"
 #include "hwy/base.h"
 #include "hwy/highway.h"

gemma/configs.cc

@@ -21,8 +21,8 @@
 #include <string>
 #include <vector>
 
-#include "compression/fields.h"  // IFields
 #include "compression/shared.h"  // Type
+#include "io/fields.h"           // IFields
 #include "hwy/base.h"
 
 namespace gcpp {

gemma/configs.h

@@ -25,8 +25,8 @@
 #include <string>
 #include <vector>
 
-#include "compression/fields.h"  // IFieldsVisitor
 #include "compression/shared.h"  // Type
+#include "io/fields.h"           // IFieldsVisitor
 #include "util/basics.h"
 
 namespace gcpp {

gemma/configs_test.cc

@@ -6,8 +6,8 @@
 #include <vector>
 
 #include "gtest/gtest.h"
-#include "compression/fields.h"  // Type
 #include "compression/shared.h"  // Type
+#include "io/fields.h"           // Type
 
 namespace gcpp {
 

gemma/gemma.cc

@@ -28,13 +28,13 @@
 #include <vector>
 
 // Placeholder for internal header, do not modify.
-#include "compression/blob_store.h"
-#include "compression/io.h"  // Path
 #include "compression/shared.h"
 #include "gemma/configs.h"
 #include "gemma/model_store.h"
 #include "gemma/tokenizer.h"
 #include "gemma/weights.h"
+#include "io/blob_store.h"
+#include "io/io.h"  // Path
 #include "ops/matmul.h"
 #include "paligemma/image.h"
 #include "util/threading_context.h"

gemma/gemma.h

@@ -21,14 +21,14 @@
 #include <memory>
 
 // IWYU pragma: begin_exports
-#include "compression/blob_store.h"
-#include "compression/io.h"  // Path
 #include "gemma/activations.h"
 #include "gemma/configs.h"
 #include "gemma/gemma_args.h"
 #include "gemma/kv_cache.h"
 #include "gemma/model_store.h"
 #include "gemma/weights.h"
+#include "io/blob_store.h"
+#include "io/io.h"       // Path
 #include "ops/matmul.h"  // MatMulEnv
 #include "paligemma/image.h"
 #include "util/basics.h"  // TokenAndProb

gemma/gemma_args.h

@@ -25,7 +25,7 @@
 #include <random>
 #include <string>
 
-#include "compression/io.h"  // Path
+#include "io/io.h"           // Path
 #include "ops/matmul.h"      // MMStorage::kMax*
 #include "util/args.h"
 #include "util/basics.h"          // Tristate

gemma/model_store.cc

@@ -24,13 +24,13 @@
 #include <cstring>  // strcmp
 #include <string>
 
-#include "compression/blob_store.h"
-#include "compression/fields.h"
-#include "compression/io.h"  // Path
 #include "compression/shared.h"
 #include "gemma/configs.h"  // ModelConfig
 #include "gemma/tensor_info.h"
 #include "gemma/tokenizer.h"
+#include "io/blob_store.h"
+#include "io/fields.h"
+#include "io/io.h"  // Path
 #include "util/basics.h"
 #include "util/threading_context.h"
 #include "hwy/base.h"

gemma/model_store.h

@@ -27,10 +27,10 @@
 #include <vector>
 
 // IWYU pragma: begin_exports
-#include "compression/blob_store.h"
+#include "gemma/configs.h"  // ModelConfig
-#include "compression/io.h"  // Path
-#include "gemma/configs.h"   // ModelConfig
 #include "gemma/tokenizer.h"
+#include "io/blob_store.h"
+#include "io/io.h"        // Path
 #include "util/basics.h"  // Tristate
 #include "util/mat.h"     // MatPtr
 // IWYU pragma: end_exports

gemma/weights.cc

@@ -25,11 +25,11 @@
 #include <string>
 #include <vector>
 
-#include "compression/blob_store.h"
 #include "compression/compress.h"
 #include "compression/shared.h"
 #include "gemma/configs.h"
 #include "gemma/model_store.h"
+#include "io/blob_store.h"
 #include "util/mat.h"
 #include "hwy/base.h"
 #include "hwy/contrib/thread_pool/thread_pool.h"

gemma/weights.h

@@ -25,12 +25,12 @@
 #include <string>
 #include <vector>
 
-#include "compression/blob_store.h"  // BlobWriter
+#include "compression/shared.h"  // IsF32
-#include "compression/shared.h"      // IsF32
+#include "gemma/configs.h"       // ModelConfig
-#include "gemma/configs.h"           // ModelConfig
+#include "gemma/model_store.h"   // ModelStore
-#include "gemma/model_store.h"       // ModelStore
+#include "gemma/tensor_info.h"   // TensorInfoRegistry
-#include "gemma/tensor_info.h"       // TensorInfoRegistry
+#include "io/blob_store.h"       // BlobWriter
-#include "util/mat.h"                // MatPtr
+#include "util/mat.h"            // MatPtr
 #include "hwy/contrib/thread_pool/thread_pool.h"
 
 namespace gcpp {

io/BUILD.bazel

@@ -0,0 +1,118 @@
+# File I/O and model loading
+
+package(
+    default_applicable_licenses = [
+        "//:license",  # Placeholder comment, do not modify
+    ],
+    # Placeholder for internal compatible_with
+    default_visibility = [
+        # Placeholder for internal visibility,
+        # Users require gcpp::Path etc., which are defined in this package.
+        "//visibility:public",
+    ],
+)
+
+config_setting(
+    name = "android",
+    constraint_values = [
+        "@platforms//os:android",
+    ],
+    visibility = ["//visibility:private"],
+)
+
+FILE_DEPS = select({
+    "//conditions:default": [
+        # Placeholder for io deps, do not remove
+    ],
+    ":android": [],
+    # Placeholder for internal build rules, do not remove
+})
+
+cc_library(
+    name = "io",
+    srcs = [
+        "io.cc",
+        # Placeholder for io backend, do not remove
+    ],
+    hdrs = ["io.h"],
+    local_defines = select({
+        # Placeholder for internal build rules, do not remove
+        "//conditions:default": [],
+    }),
+    deps = [
+        "//:allocator",
+        "@highway//:hwy",
+    ] + FILE_DEPS,
+)
+
+cc_library(
+    name = "fields",
+    srcs = ["fields.cc"],
+    hdrs = ["fields.h"],
+    deps = [
+        "@highway//:hwy",
+    ],
+)
+
+cc_test(
+    name = "fields_test",
+    srcs = ["fields_test.cc"],
+    deps = [
+        ":fields",
+        "@googletest//:gtest_main",  # buildcleaner: keep
+        "@highway//:hwy_test_util",
+    ],
+)
+
+cc_library(
+    name = "blob_store",
+    srcs = ["blob_store.cc"],
+    hdrs = ["blob_store.h"],
+    deps = [
+        ":io",
+        "//:basics",
+        "//:threading_context",
+        "@highway//:hwy",
+        "@highway//:profiler",
+        "@highway//:thread_pool",
+    ],
+)
+
+cc_test(
+    name = "blob_store_test",
+    srcs = ["blob_store_test.cc"],
+    deps = [
+        ":blob_store",
+        ":io",
+        "@googletest//:gtest_main",  # buildcleaner: keep
+        "//:basics",
+        "//:threading_context",
+        "@highway//:hwy_test_util",
+    ],
+)
+
+cc_binary(
+    name = "blob_compare",
+    srcs = ["blob_compare.cc"],
+    deps = [
+        ":blob_store",
+        ":io",
+        "//:basics",
+        "//:threading",
+        "//:threading_context",
+        "@highway//:hwy",
+        "@highway//:hwy_test_util",
+        "@highway//:nanobenchmark",
+        "@highway//:thread_pool",
+    ],
+)
+
+cc_binary(
+    name = "migrate_weights",
+    srcs = ["migrate_weights.cc"],
+    deps = [
+        "//:args",
+        "//:benchmark_helper",
+        "//:gemma_lib",
+    ],
+)

io/blob_compare.cc

@@ -22,8 +22,8 @@
 #include <string>
 #include <vector>
 
-#include "compression/blob_store.h"
+#include "io/blob_store.h"
-#include "compression/io.h"  // Path
+#include "io/io.h"        // Path
 #include "util/basics.h"  // IndexRange
 #include "util/threading.h"
 #include "util/threading_context.h"

io/blob_store.cc

@@ -13,7 +13,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "compression/blob_store.h"
+#include "io/blob_store.h"
 
 #include <stddef.h>
 #include <stdint.h>
@@ -24,7 +24,7 @@
 #include <utility>  // std::move
 #include <vector>
 
-#include "compression/io.h"
+#include "io/io.h"
 #include "util/threading_context.h"
 #include "hwy/aligned_allocator.h"  // Span
 #include "hwy/base.h"

io/blob_store.h

@@ -13,8 +13,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef THIRD_PARTY_GEMMA_CPP_COMPRESSION_BLOB_STORE_H_
+#ifndef THIRD_PARTY_GEMMA_CPP_IO_BLOB_STORE_H_
-#define THIRD_PARTY_GEMMA_CPP_COMPRESSION_BLOB_STORE_H_
+#define THIRD_PARTY_GEMMA_CPP_IO_BLOB_STORE_H_
 
 // Reads/writes arrays of bytes from/to file.
 
@@ -26,7 +26,7 @@
 #include <unordered_map>
 #include <vector>
 
-#include "compression/io.h"       // File, Path, MapPtr
+#include "io/io.h"                // File, Path, MapPtr
 #include "util/basics.h"          // Tristate
 #include "hwy/aligned_allocator.h"  // Span
 #include "hwy/base.h"               // HWY_ASSERT
@@ -178,4 +178,4 @@ class BlobWriter {
 
 }  // namespace gcpp
 
-#endif  // THIRD_PARTY_GEMMA_CPP_COMPRESSION_BLOB_STORE_H_
+#endif  // THIRD_PARTY_GEMMA_CPP_IO_BLOB_STORE_H_

io/blob_store_test.cc

@@ -13,7 +13,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "compression/blob_store.h"
+#include "io/blob_store.h"
 
 #include <stdio.h>
 
@@ -23,7 +23,7 @@
 #include <string>
 #include <vector>
 
-#include "compression/io.h"
+#include "io/io.h"
 #include "util/basics.h"
 #include "util/threading_context.h"
 #include "hwy/tests/hwy_gtest.h"

io/fields.cc

@@ -13,7 +13,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "compression/fields.h"
+#include "io/fields.h"
 
 #include <stdarg.h>
 #include <stddef.h>

io/fields_test.cc

@@ -13,7 +13,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "compression/fields.h"
+#include "io/fields.h"
 
 #include <stddef.h>
 #include <stdint.h>

io/io.cc

@@ -44,7 +44,7 @@
 
 #include <memory>
 
-#include "compression/io.h"
+#include "io/io.h"
 #include "util/allocator.h"
 #include "hwy/base.h"  // HWY_ASSERT
 

io/io_win.cc

@@ -21,7 +21,7 @@
 #include <stddef.h>
 #include <stdint.h>
 
-#include "compression/io.h"
+#include "io/io.h"
 #include "util/allocator.h"
 #include "hwy/base.h"  // HWY_ASSERT
 #ifndef WIN32_LEAN_AND_MEAN

paligemma/BUILD.bazel

@@ -13,7 +13,7 @@ cc_library(
     srcs = ["image.cc"],
     hdrs = ["image.h"],
     deps = [
-        "//compression:io",
+        "//io",
         "@highway//:hwy",
         "@highway//:profiler",
     ],

paligemma/image.cc

@@ -29,7 +29,7 @@
 #include <utility>
 #include <vector>
 
-#include "compression/io.h"
+#include "io/io.h"
 #include "hwy/aligned_allocator.h"  // hwy::Span
 #include "hwy/base.h"
 #include "hwy/profiler.h"

util/args.h

@@ -23,7 +23,7 @@
 #include <algorithm>  // std::transform
 #include <string>
 
-#include "compression/io.h"  // Path
+#include "io/io.h"        // Path
 #include "util/basics.h"  // Tristate
 #include "hwy/base.h"  // HWY_ABORT
 

util/mat.h

@@ -25,9 +25,9 @@
 #include <vector>
 
 // IWYU pragma: begin_exports
-#include "compression/fields.h"
 #include "compression/shared.h"  // Type
 #include "gemma/tensor_info.h"
+#include "io/fields.h"
 #include "util/allocator.h"
 #include "util/basics.h"  // Extents2D
 // IWYU pragma: end_exports