From f6d79fe1b1dddbcc8678f918394f5430b37b339e Mon Sep 17 00:00:00 2001 From: JTischbein Date: Tue, 16 Dec 2025 13:58:43 +0100 Subject: [PATCH] Remove branching in llama-model-loader.cpp and reduce code duplications in llama-mmap.cpp --- src/llama-mmap.cpp | 180 +++++++++++++--------------------- src/llama-mmap.h | 7 +- src/llama-model-loader.cpp | 196 +++++++++++++++---------------------- 3 files changed, 150 insertions(+), 233 deletions(-) diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp index 9d03c41d96..044d9aa902 100644 --- a/src/llama-mmap.cpp +++ b/src/llama-mmap.cpp @@ -75,7 +75,7 @@ struct llama_file::impl { return ret; } - impl(const char * fname, const char * mode) { + impl(const char * fname, const char * mode, const bool use_direct_io = false) { fp = ggml_fopen(fname, mode); if (fp == NULL) { throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno))); @@ -154,43 +154,50 @@ struct llama_file::impl { write_raw(&val, sizeof(val)); } + bool has_direct_io() const { + return false; + } + + void read_aligned_chunk(size_t offset, void * dest, size_t size, size_t alignment) const { + throw std::runtime_error("DirectIO is not implemented on Windows."); + } + ~impl() { if (fp) { std::fclose(fp); } } -#elif defined(__linux__) - impl(const char * fname, const char * mode) : impl(fname, mode, false) {} - - impl(const char * fname, const char * mode, bool uncached_read) { - if (uncached_read) { +#else + impl(const char * fname, const char * mode, const bool use_direct_io = false) { +#ifdef __linux__ + // Try unbuffered I/O for read only + if (use_direct_io && std::strcmp(mode, "rb") == 0) { fd = open(fname, O_RDONLY | O_DIRECT); - if (fd == -1 && (errno == EINVAL || errno == EOPNOTSUPP)) { - fd = open(fname, O_RDONLY); // retry without O_DIRECT + + if (fd != -1) { + struct stat file_stats{}; + fstat(fd, &file_stats); + + size = file_stats.st_size; + + off_t ret = lseek(fd, 0, SEEK_SET); + if (ret == -1) { + throw std::runtime_error(format("seek error: %s", strerror(errno))); + } + return; } - if (fd == -1) { - throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno))); - } - - struct stat file_stats{}; - fstat(fd, &file_stats); - - size = file_stats.st_size; - - off_t ret = lseek(fd, 0, SEEK_SET); - if (ret == -1) { - throw std::runtime_error(format("seek error: %s", strerror(errno))); - } - } else { - fp = ggml_fopen(fname, mode); - if (fp == NULL) { - throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno))); - } - seek(0, SEEK_END); - size = tell(); - seek(0, SEEK_SET); + LLAMA_LOG_WARN("Failed to open model %s with error: %s. Falling back to buffered I/O", + fname, strerror(errno)); } +#endif + fp = ggml_fopen(fname, mode); + if (fp == NULL) { + throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno))); + } + seek(0, SEEK_END); + size = tell(); + seek(0, SEEK_SET); } size_t tell() const { @@ -226,8 +233,8 @@ struct llama_file::impl { if (len == 0) { return; } + errno = 0; if (fd == -1) { - errno = 0; std::size_t ret = std::fread(ptr, len, 1, fp); if (ferror(fp)) { throw std::runtime_error(format("read error: %s", strerror(errno))); @@ -255,86 +262,27 @@ struct llama_file::impl { } } - uint32_t read_u32() const { - uint32_t ret; - read_raw(&ret, sizeof(ret)); - return ret; - } + void read_aligned_chunk(size_t offset, void * dest, size_t size, size_t alignment) const { + off_t aligned_offset = offset & ~(alignment - 1); + off_t offset_from_alignment = offset - aligned_offset; + size_t bytes_to_read = (offset_from_alignment + size + alignment - 1) & ~(alignment - 1); - void write_raw(const void * ptr, size_t len) const { - if (len == 0) { - return; - } - errno = 0; - size_t ret = std::fwrite(ptr, len, 1, fp); - if (ret != 1) { - throw std::runtime_error(format("write error: %s", strerror(errno))); - } - } - - void write_u32(uint32_t val) const { - write_raw(&val, sizeof(val)); - } - - ~impl() { - if (fp) { - std::fclose(fp); - } else if (fd != -1) { - close(fd); - } - } - - int fd = -1; - -#else - impl(const char * fname, const char * mode) { - fp = ggml_fopen(fname, mode); - if (fp == NULL) { - throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno))); - } - seek(0, SEEK_END); - size = tell(); - seek(0, SEEK_SET); - } - - size_t tell() const { -// TODO: this ifdef is never true? -#ifdef _WIN32 - __int64 ret = _ftelli64(fp); -#else - long ret = std::ftell(fp); -#endif - if (ret == -1) { - throw std::runtime_error(format("ftell error: %s", strerror(errno))); - } - - return (size_t) ret; - } - - void seek(size_t offset, int whence) const { -// TODO: this ifdef is never true? -#ifdef _WIN32 - int ret = _fseeki64(fp, (__int64) offset, whence); -#else - int ret = std::fseek(fp, (long) offset, whence); -#endif + void * raw_buffer = nullptr; + int ret = posix_memalign(&raw_buffer, alignment, bytes_to_read); if (ret != 0) { - throw std::runtime_error(format("seek error: %s", strerror(errno))); + throw std::runtime_error(format("posix_memalign failed with error %d", ret)); } - } - void read_raw(void * ptr, size_t len) const { - if (len == 0) { - return; - } - errno = 0; - std::size_t ret = std::fread(ptr, len, 1, fp); - if (ferror(fp)) { - throw std::runtime_error(format("read error: %s", strerror(errno))); - } - if (ret != 1) { - throw std::runtime_error("unexpectedly reached end of file"); - } + struct aligned_buffer_deleter { + void operator()(void * p) const { free(p); } + }; + std::unique_ptr buffer(raw_buffer); + + seek(aligned_offset, SEEK_SET); + read_raw(buffer.get(), bytes_to_read); + + uintptr_t actual_data = reinterpret_cast(buffer.get()) + offset_from_alignment; + memcpy(dest, reinterpret_cast(actual_data), size); } uint32_t read_u32() const { @@ -358,26 +306,33 @@ struct llama_file::impl { write_raw(&val, sizeof(val)); } + bool has_direct_io() const { + return fd != -1; + } + ~impl() { - if (fp) { + if (fd != -1) { + close(fd); + } else { std::fclose(fp); } } + int fd = -1; #endif FILE * fp{}; size_t size{}; }; -llama_file::llama_file(const char * fname, const char * mode) : pimpl(std::make_unique(fname, mode)) {} -#if defined(__linux__) -llama_file::llama_file(const char * fname, const char * mode, bool uncached_read) : pimpl(std::make_unique(fname, mode, uncached_read)) {} -#endif +llama_file::llama_file(const char * fname, const char * mode, const bool use_direct_io) : + pimpl(std::make_unique(fname, mode, use_direct_io)) {} llama_file::~llama_file() = default; size_t llama_file::tell() const { return pimpl->tell(); } size_t llama_file::size() const { return pimpl->size; } +bool llama_file::has_direct_io() const { return pimpl->has_direct_io(); } + int llama_file::file_id() const { #ifdef _WIN32 return _fileno(pimpl->fp); @@ -392,6 +347,9 @@ int llama_file::file_id() const { void llama_file::seek(size_t offset, int whence) const { pimpl->seek(offset, whence); } void llama_file::read_raw(void * ptr, size_t len) const { pimpl->read_raw(ptr, len); } +void llama_file::read_aligned_chunk(size_t offset, void * dest, size_t size, size_t alignment) const + { pimpl->read_aligned_chunk(offset, dest, size, alignment); } + uint32_t llama_file::read_u32() const { return pimpl->read_u32(); } diff --git a/src/llama-mmap.h b/src/llama-mmap.h index 985404d0f5..5a9361e4c3 100644 --- a/src/llama-mmap.h +++ b/src/llama-mmap.h @@ -13,10 +13,7 @@ using llama_mmaps = std::vector>; using llama_mlocks = std::vector>; struct llama_file { - llama_file(const char * fname, const char * mode); -#if defined(__linux__) - llama_file(const char * fname, const char * mode, bool uncached_read); -#endif + llama_file(const char * fname, const char * mode, bool use_direct_io = false); ~llama_file(); size_t tell() const; @@ -27,11 +24,13 @@ struct llama_file { void seek(size_t offset, int whence) const; void read_raw(void * ptr, size_t len) const; + void read_aligned_chunk(size_t offset, void * dest, size_t size, size_t alignment) const; uint32_t read_u32() const; void write_raw(const void * ptr, size_t len) const; void write_u32(uint32_t val) const; + bool has_direct_io() const; private: struct impl; std::unique_ptr pimpl; diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 9faf85a050..6e47d992bb 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -503,11 +503,7 @@ llama_model_loader::llama_model_loader( get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false); llm_kv = LLM_KV(llm_arch_from_string(arch_name)); -#if defined(__linux__) files.emplace_back(new llama_file(fname.c_str(), "rb", !use_mmap)); -#else - files.emplace_back(new llama_file(fname.c_str(), "rb")); -#endif contexts.emplace_back(ctx); // Save tensors data offset of the main file. @@ -575,11 +571,7 @@ llama_model_loader::llama_model_loader( } } -#if defined(__linux__) - files.emplace_back(new llama_file(fname_split, "rb", !use_mmap)); -#else files.emplace_back(new llama_file(fname_split, "rb")); -#endif contexts.emplace_back(ctx); // Save tensors data offset info of the shard. @@ -941,14 +933,17 @@ bool llama_model_loader::load_all_data( // 4 staging buffers for async uploads, each sized 1MB seems to be a good default for single NVMe drives. // NVMe raid configurations might require more / larger buffers. constexpr size_t n_buffers = 4; -#if defined(__linux__) - constexpr size_t alignment = 4 * 1024; // 4 KiB for Direct I/O + + + bool direct_io = false; + for (const auto& file : files) { + direct_io |= file->has_direct_io(); + } + + constexpr size_t alignment = 4 * 1024; // 4 KB for Direct I/O // Buffer size: balance between memory usage and I/O efficiency // 64MB works well for NVMe drives - constexpr size_t buffer_size = 64 * 1024 * 1024; // 64 MiB -#else - constexpr size_t buffer_size = 1 * 1024 * 1024; // 1MB -#endif + const size_t buffer_size = direct_io ? 64 * 1024 * 1024 + 2 * alignment : 1 * 1024 * 1024; std::vector host_buffers; std::vector events; @@ -997,11 +992,8 @@ bool llama_model_loader::load_all_data( // If the backend is supported, create pinned memory buffers and events for synchronisation. for (size_t idx = 0; idx < n_buffers; ++idx) { -#if defined(__linux__) - auto * buf = ggml_backend_buft_alloc_buffer(host_buft, buffer_size + 2 * alignment); -#else auto * buf = ggml_backend_buft_alloc_buffer(host_buft, buffer_size); -#endif + if (!buf) { LLAMA_LOG_DEBUG("%s: failed to allocate host buffer for async uploads for device %s\n", func, ggml_backend_dev_name(dev)); @@ -1038,35 +1030,6 @@ bool llama_model_loader::load_all_data( ggml_backend_name(upload_backend)); } -#if defined(__linux__) - auto read_aligned_chunk = [](const llama_file * file, - size_t offset, - void * dest, - size_t size, - size_t alignment) { - off_t aligned_offset = offset & ~(alignment - 1); - off_t offset_from_alignment = offset - aligned_offset; - size_t bytes_to_read = (offset_from_alignment + size + alignment - 1) & ~(alignment - 1); - - void * raw_buffer = nullptr; - int ret = posix_memalign(&raw_buffer, alignment, bytes_to_read); - if (ret != 0) { - throw std::runtime_error(format("posix_memalign failed with error %d", ret)); - } - - struct aligned_buffer_deleter { - void operator()(void * p) const { free(p); } - }; - std::unique_ptr buffer(raw_buffer); - - file->seek(aligned_offset, SEEK_SET); - file->read_raw(buffer.get(), bytes_to_read); - - uintptr_t actual_data = reinterpret_cast(buffer.get()) + offset_from_alignment; - memcpy(dest, reinterpret_cast(actual_data), size); - }; -#endif - for (struct ggml_tensor * cur = ggml_get_first_tensor(ctx); cur != NULL; cur = ggml_get_next_tensor(ctx, cur)) { const auto * weight = get_weight(ggml_get_name(cur)); if (weight == nullptr) { @@ -1112,100 +1075,97 @@ bool llama_model_loader::load_all_data( } } else { const auto & file = files.at(weight->idx); -#if defined(__linux__) - auto offset = (off_t) weight->offs; - off_t aligned_offset = offset & ~(alignment - 1); - off_t offset_from_alignment = offset - aligned_offset; -#endif + if (ggml_backend_buffer_is_host(cur->buffer)) { -#if defined(__linux__) - read_aligned_chunk(file.get(), weight->offs, cur->data, n_size, alignment); -#else - file->seek(weight->offs, SEEK_SET); - file->read_raw(cur->data, n_size); -#endif + if (file->has_direct_io()) { + file->read_aligned_chunk(weight->offs, cur->data, n_size, alignment); + } else { + file->seek(weight->offs, SEEK_SET); + file->read_raw(cur->data, n_size); + } if (check_tensors) { validation_result.emplace_back(std::async(std::launch::async, [cur, n_size] { return std::make_pair(cur, ggml_validate_row_data(cur->type, cur->data, n_size)); })); } } else { + file->seek(weight->offs, SEEK_SET); // If upload_backend is valid load the tensor in chunks to pinned memory and upload the buffers asynchronously to the GPU. if (upload_backend) { -#if defined(__linux__) - // Calculate aligned read boundaries - size_t read_start = aligned_offset; - size_t read_end = (offset + n_size + alignment - 1) & ~(alignment - 1); + if (file->has_direct_io()) { + auto offset = (off_t) weight->offs; + off_t aligned_offset = offset & ~(alignment - 1); + off_t offset_from_alignment = offset - aligned_offset; - size_t bytes_read = 0; - size_t data_read = 0; // Actual tensor data copied (excluding padding) + // Calculate aligned read boundaries + size_t read_start = aligned_offset; + size_t read_end = (offset + n_size + alignment - 1) & ~(alignment - 1); - file->seek(aligned_offset, SEEK_SET); + size_t bytes_read = 0; + size_t data_read = 0; // Actual tensor data copied (excluding padding) - while (bytes_read < read_end - read_start) { - size_t read_size = std::min(buffer_size, read_end - read_start - bytes_read); + while (bytes_read < read_end - read_start) { + size_t read_size = std::min(buffer_size, read_end - read_start - bytes_read); - // Align the destination pointer within the pinned buffer - uintptr_t ptr_dest_aligned = (reinterpret_cast(host_ptrs[buffer_idx]) + alignment - 1) & ~(alignment - 1); + // Align the destination pointer within the pinned buffer + uintptr_t ptr_dest_aligned = (reinterpret_cast(host_ptrs[buffer_idx]) + alignment - 1) & ~(alignment - 1); - // Wait for previous upload to complete before reusing buffer - ggml_backend_event_synchronize(events[buffer_idx]); + // Wait for previous upload to complete before reusing buffer + ggml_backend_event_synchronize(events[buffer_idx]); - // Read aligned chunk from file - file->read_raw(reinterpret_cast(ptr_dest_aligned), read_size); + // Read aligned chunk from file + file->read_raw(reinterpret_cast(ptr_dest_aligned), read_size); - // Calculate actual data portion (excluding alignment padding) - uintptr_t ptr_data = ptr_dest_aligned; - size_t data_to_copy = read_size; + // Calculate actual data portion (excluding alignment padding) + uintptr_t ptr_data = ptr_dest_aligned; + size_t data_to_copy = read_size; - // Skip alignment padding at start of first chunk - if (bytes_read == 0) { - ptr_data += offset_from_alignment; - data_to_copy -= offset_from_alignment; + // Skip alignment padding at start of first chunk + if (bytes_read == 0) { + ptr_data += offset_from_alignment; + data_to_copy -= offset_from_alignment; + } + + // Trim alignment padding at end of last chunk + if (aligned_offset + bytes_read + read_size > offset + n_size) { + data_to_copy -= (read_end - (offset + n_size)); + } + + // Async upload actual data to GPU + ggml_backend_tensor_set_async(upload_backend, cur, + reinterpret_cast(ptr_data), data_read, data_to_copy); + ggml_backend_event_record(events[buffer_idx], upload_backend); + + data_read += data_to_copy; + bytes_read += read_size; + + ++buffer_idx; + buffer_idx %= n_buffers; } + } else { + size_t bytes_read = 0; - // Trim alignment padding at end of last chunk - if (aligned_offset + bytes_read + read_size > offset + n_size) { - data_to_copy -= (read_end - (offset + n_size)); + while (bytes_read < n_size) { + size_t read_iteration = std::min(buffer_size, n_size - bytes_read); + + ggml_backend_event_synchronize(events[buffer_idx]); + file->read_raw(host_ptrs[buffer_idx], read_iteration); + ggml_backend_tensor_set_async(upload_backend, cur, host_ptrs[buffer_idx], bytes_read, read_iteration); + ggml_backend_event_record(events[buffer_idx], upload_backend); + + bytes_read += read_iteration; + ++buffer_idx; + buffer_idx %= n_buffers; } - - // Async upload actual data to GPU - ggml_backend_tensor_set_async(upload_backend, cur, - reinterpret_cast(ptr_data), data_read, data_to_copy); - ggml_backend_event_record(events[buffer_idx], upload_backend); - - data_read += data_to_copy; - bytes_read += read_size; - - ++buffer_idx; - buffer_idx %= n_buffers; } -#else - file->seek(weight->offs, SEEK_SET); - - size_t bytes_read = 0; - - while (bytes_read < n_size) { - size_t read_iteration = std::min(buffer_size, n_size - bytes_read); - - ggml_backend_event_synchronize(events[buffer_idx]); - file->read_raw(host_ptrs[buffer_idx], read_iteration); - ggml_backend_tensor_set_async(upload_backend, cur, host_ptrs[buffer_idx], bytes_read, read_iteration); - ggml_backend_event_record(events[buffer_idx], upload_backend); - - bytes_read += read_iteration; - ++buffer_idx; - buffer_idx %= n_buffers; - } -#endif } else { read_buf.resize(n_size); -#if defined(__linux__) - read_aligned_chunk(file.get(), weight->offs, read_buf.data(), n_size, alignment); -#else - file->seek(weight->offs, SEEK_SET); - file->read_raw(read_buf.data(), n_size); -#endif + if (file->has_direct_io()) { + file->read_aligned_chunk(weight->offs, read_buf.data(), n_size, alignment); + } else { + file->seek(weight->offs, SEEK_SET); + file->read_raw(read_buf.data(), n_size); + } ggml_backend_tensor_set(cur, read_buf.data(), 0, n_size); if (check_tensors && !ggml_validate_row_data(cur->type, read_buf.data(), n_size)) { throw std::runtime_error(format("tensor '%s' has invalid data", ggml_get_name(cur)));