happyz
/
llama.cpp
mirror of https://github.com/ggerganov/llama.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

cuda : unroll Q*K^T loop

This commit is contained in:
Georgi Gerganov 2024-02-03 16:12:20 +02:00
parent 3b1c4e7673
commit 5b263dd83a
No known key found for this signature in database
GPG Key ID: 449E073F9DC10735
1 changed files with 1 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
ggml-cuda.cu
View File

@ -6571,6 +6571,7 @@ static __global__ void flash_attn_ext_f16(
// Q*K^T // Q*K^T
{ {
#pragma unroll
for (int cc = 0; cc < C/16; ++cc) { for (int cc = 0; cc < C/16; ++cc) {
half16x16_acc mqk[Q16]; half16x16_acc mqk[Q16];
for (int j = 0; j < Q16; ++j) { for (int j = 0; j < Q16; ++j) {