happyz
/
gemma.cpp
mirror of https://github.com/google/gemma.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Increase the prefill batch size to 64. 

PiperOrigin-RevId: 651754772
This commit is contained in:
The gemma.cpp Authors 2024-07-12 06:27:57 -07:00 committed by Copybara-Service
parent df3fb70802
commit c879133a5a
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
gemma/common.h
View File

@ -36,7 +36,7 @@ ByteStorageT AllocateSizeof() {
return hwy::AllocateAligned<uint8_t>(sizeof(T)); return hwy::AllocateAligned<uint8_t>(sizeof(T));
} }
constexpr size_t kPrefillBatchSize = 16; constexpr size_t kPrefillBatchSize = 64;
constexpr size_t kDecodeBatchSize = 1; constexpr size_t kDecodeBatchSize = 1;
constexpr size_t kBatchedQueryBatchSize = 16; constexpr size_t kBatchedQueryBatchSize = 16;
constexpr size_t kMinAdjustedPrefillBatchSize = constexpr size_t kMinAdjustedPrefillBatchSize =