happyz
/
gemma.cpp
mirror of https://github.com/google/gemma.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix KV cache size calculation error

This commit is contained in:
RangerUFO 2024-06-21 13:06:26 +08:00
parent 0570972d43
commit d7787c8f6c
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
gemma/gemma.cc
View File

@ -114,7 +114,7 @@ struct CreateKVCache {
KVCache operator()() const { KVCache operator()() const {
KVCache kv_cache = {}; KVCache kv_cache = {};
const size_t size_cache_pos = CacheLayerSize<TConfig>()(); const size_t size_cache_pos = CachePosSize<TConfig>()();
if (size_cache_pos != 0) { if (size_cache_pos != 0) {
const size_t seq_len = TConfig::kSeqLen + kPrefillBatchSize; const size_t seq_len = TConfig::kSeqLen + kPrefillBatchSize;
kv_cache.kv_cache = hwy::AllocateAligned<float>(seq_len * size_cache_pos); kv_cache.kv_cache = hwy::AllocateAligned<float>(seq_len * size_cache_pos);