diff --git a/io/blob_store.cc b/io/blob_store.cc index 9e1bda1..6b33186 100644 --- a/io/blob_store.cc +++ b/io/blob_store.cc @@ -411,17 +411,19 @@ void BlobWriter::WriteAll(hwy::ThreadPool& pool, const Path& filename) { std::unique_ptr file = OpenFileOrNull(filename, "w+"); if (!file) HWY_ABORT("Failed to open for writing %s", filename.path.c_str()); - pool.Run(0, writes.size(), - [this, &file, &writes](uint64_t i, size_t /*thread*/) { - const BlobRange& range = writes[i].range; + hwy::ThreadPool null_pool(0); + hwy::ThreadPool& pool_or_serial = file->IsAppendOnly() ? null_pool : pool; + pool_or_serial.Run( + 0, writes.size(), [this, &file, &writes](uint64_t i, size_t /*thread*/) { + const BlobRange& range = writes[i].range; - if (!file->Write(writes[i].data, range.bytes, range.offset)) { - const std::string& key = StringFromKey(keys_[range.key_idx]); - HWY_ABORT("Write failed for %s from %zu, %zu bytes to %p.", - key.c_str(), static_cast(range.offset), - range.bytes, writes[i].data); - } - }); + if (!file->Write(writes[i].data, range.bytes, range.offset)) { + const std::string& key = StringFromKey(keys_[range.key_idx]); + HWY_ABORT("Write failed for %s from %zu, %zu bytes to %p.", + key.c_str(), static_cast(range.offset), range.bytes, + writes[i].data); + } + }); } } // namespace gcpp diff --git a/io/io.cc b/io/io.cc index 1f7eda6..df39f7b 100644 --- a/io/io.cc +++ b/io/io.cc @@ -88,6 +88,9 @@ class FilePosix : public File { } } + // pwrite is thread-safe and allows arbitrary offsets. + bool IsAppendOnly() const override { return false; } + uint64_t FileSize() const override { static_assert(sizeof(off_t) == 8, "64-bit off_t required"); const off_t size = lseek(fd_, 0, SEEK_END); diff --git a/io/io.h b/io/io.h index a14889b..d9481bc 100644 --- a/io/io.h +++ b/io/io.h @@ -46,6 +46,10 @@ class File { File(const File& other) = delete; const File& operator=(const File& other) = delete; + // If true, Write() should only be called with `offset` equal to the number + // of bytes already written to the file, which rules out parallel writes. + virtual bool IsAppendOnly() const = 0; + // Returns size in bytes or 0. virtual uint64_t FileSize() const = 0; diff --git a/io/io_win.cc b/io/io_win.cc index 1f35a96..34773d3 100644 --- a/io/io_win.cc +++ b/io/io_win.cc @@ -47,6 +47,9 @@ class FileWin : public File { } } + // WriteFile is thread-safe and allows arbitrary offsets. + bool IsAppendOnly() const override { return false; } + uint64_t FileSize() const override { DWORD hi; const DWORD lo = GetFileSize(hFile_, &hi);