happyz
/
llama.cpp
mirror of https://github.com/ggerganov/llama.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fixed where I added residual, now gives much much better embeddings~

This commit is contained in:
ryan-mangeno 2025-10-29 12:20:50 -04:00
parent 7a1f06ac35
commit 04167d9e26
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/llama-model.cpp
View File

@ -8280,11 +8280,11 @@ struct llm_build_modern_bert : public llm_graph_context {
// re-add the layer input // re-add the layer input
cur = ggml_add(ctx0, cur, inpL); cur = ggml_add(ctx0, cur, inpL);
ggml_tensor * ffn_inp = cur;
// attention layer norm // attention layer norm
cur = build_norm(cur, model.layers[il].ffn_norm, nullptr, LLM_NORM, il); cur = build_norm(cur, model.layers[il].ffn_norm, nullptr, LLM_NORM, il);
ggml_tensor * ffn_inp = cur;
cb(ffn_inp, "ffn_inp", il); cb(ffn_inp, "ffn_inp", il);
cur = build_ffn(cur, cur = build_ffn(cur,