happyz
/
gemma.cpp
mirror of https://github.com/google/gemma.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Make BlobWriter::Add() accept const void* 

PiperOrigin-RevId: 634780483
This commit is contained in:
Paul Chang 2024-05-17 08:10:26 -07:00 committed by Copybara-Service
parent 82623bdc7f
commit cfce314715
2 changed files with 8 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
compression/blob_store.cc
View File

@ -124,7 +124,7 @@ class BlobStore {
} }
static std::vector<BlobIO> PrepareWriteRequests( static std::vector<BlobIO> PrepareWriteRequests(
const hwy::uint128_t keys[], const hwy::Span<uint8_t> blobs[], const hwy::uint128_t keys[], const hwy::Span<const uint8_t> blobs[],
size_t num_blobs, BlobStore* bs) { size_t num_blobs, BlobStore* bs) {
// Sanity check and ensure the cast below is safe. // Sanity check and ensure the cast below is safe.
HWY_ASSERT(num_blobs < (1ULL << 20)); HWY_ASSERT(num_blobs < (1ULL << 20));
@ -164,7 +164,8 @@ class BlobStore {
bs->keys_[num_blobs + i].lo = offset; bs->keys_[num_blobs + i].lo = offset;
bs->keys_[num_blobs + i].hi = blobs[i].size(); bs->keys_[num_blobs + i].hi = blobs[i].size();
EnqueueChunkRequests(offset, blobs[i].size(), blobs[i].data(), requests); EnqueueChunkRequests(offset, blobs[i].size(),
const_cast<uint8_t*>(blobs[i].data()), requests);
offset += blobs[i].size(); offset += blobs[i].size();
const size_t padded_size = hwy::RoundUpTo(blobs[i].size(), kBlobAlign); const size_t padded_size = hwy::RoundUpTo(blobs[i].size(), kBlobAlign);
if (padded_size != blobs[i].size()) { if (padded_size != blobs[i].size()) {
@ -270,8 +271,8 @@ BlobError BlobWriter::WriteAll(hwy::ThreadPool& pool, const Path& filename) {
// Concatenate blobs in memory. // Concatenate blobs in memory.
const size_t header_size = BlobStore::HeaderSize(keys_.size()); const size_t header_size = BlobStore::HeaderSize(keys_.size());
const size_t padded_header_size = hwy::RoundUpTo(header_size, kBlobAlign); const size_t padded_header_size = hwy::RoundUpTo(header_size, kBlobAlign);
BlobStorePtr bs = BlobStore::Allocate(padded_header_size); const BlobStorePtr bs = BlobStore::Allocate(padded_header_size);
std::vector<BlobIO> requests = BlobStore::PrepareWriteRequests( const std::vector<BlobIO> requests = BlobStore::PrepareWriteRequests(
keys_.data(), blobs_.data(), keys_.size(), bs.get()); keys_.data(), blobs_.data(), keys_.size(), bs.get());
// Create/replace existing file. // Create/replace existing file.

6
compression/blob_store.h
View File

@ -80,9 +80,9 @@ class BlobReader {
class BlobWriter { class BlobWriter {
public: public:
void Add(hwy::uint128_t key, void* data, size_t size) { void Add(hwy::uint128_t key, const void* data, size_t size) {
keys_.push_back(key); keys_.push_back(key);
blobs_.emplace_back(static_cast<uint8_t*>(data), size); blobs_.emplace_back(static_cast<const uint8_t*>(data), size);
} }
// Stores all blobs to disk in the given order with padding for alignment. // Stores all blobs to disk in the given order with padding for alignment.
@ -90,7 +90,7 @@ class BlobWriter {
private: private:
std::vector<hwy::uint128_t> keys_; std::vector<hwy::uint128_t> keys_;
std::vector<hwy::Span<uint8_t>> blobs_; std::vector<hwy::Span<const uint8_t>> blobs_;
}; };
} // namespace gcpp } // namespace gcpp