From 544745c03419ad004271bb16d1d53c64535ffbed Mon Sep 17 00:00:00 2001 From: Colin Kealty <3266127+bartowski1182@users.noreply.github.com> Date: Tue, 10 Mar 2026 11:24:24 -0400 Subject: [PATCH] Update attn_qkv schema, change throw behaviour --- src/llama-quant.cpp | 7 +- tests/snapshots/qwen3-coder-next.schema | 634 ++++++++++++++--- tests/snapshots/qwen3.5-27b.schema | 842 +++++++++++++++++++---- tests/snapshots/qwen3.5-397b-a17b.schema | 791 +++++++++++++++++---- 4 files changed, 1907 insertions(+), 367 deletions(-) diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp index 7415328c24..79c57832bf 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -355,7 +355,7 @@ static ggml_type tensor_type_fallback(quantize_state_impl & qs, const ggml_tenso return return_type; } -ggml_type llama_tensor_get_type_impl(quantize_state_impl & qs, ggml_type new_type, const ggml_tensor * tensor, llama_ftype ftype, tensor_category category) { +static ggml_type llama_tensor_get_type_impl(quantize_state_impl & qs, ggml_type new_type, const ggml_tensor * tensor, llama_ftype ftype, tensor_category category) { const std::string name = ggml_get_name(tensor); // TODO: avoid hardcoded tensor names - use the TN_* constants @@ -782,7 +782,7 @@ ggml_type llama_ftype_get_default_type(llama_ftype ftype) { case LLAMA_FTYPE_MOSTLY_IQ3_S: case LLAMA_FTYPE_MOSTLY_IQ3_M: return GGML_TYPE_IQ3_S; - default: throw std::runtime_error(format("invalid output file type %d\n", ftype)); + default: return GGML_TYPE_COUNT; } } @@ -874,6 +874,9 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: } ggml_type default_type = llama_ftype_get_default_type(ftype); + if (default_type == GGML_TYPE_COUNT) { + throw std::runtime_error(format("invalid output file type %d\n", ftype)); + } // mmap consistently increases speed on Linux, and also increases speed on Windows with // hot cache. It may cause a slowdown on macOS, possibly related to free memory. diff --git a/tests/snapshots/qwen3-coder-next.schema b/tests/snapshots/qwen3-coder-next.schema index d71fe86a1f..4f12721f16 100644 --- a/tests/snapshots/qwen3-coder-next.schema +++ b/tests/snapshots/qwen3-coder-next.schema @@ -22,29 +22,38 @@ output.weight q6_K output.weight q6_K [Q2_K] q2_K +blk.0.attn_qkv.weight q4_K blk.0.ffn_down_exps.weight q3_K blk.0.ffn_down_shexp.weight q3_K +blk.1.attn_qkv.weight q4_K blk.1.ffn_down_shexp.weight q3_K blk.1.ffn_down_exps.weight q3_K +blk.2.attn_qkv.weight q4_K blk.2.ffn_down_shexp.weight q3_K blk.2.ffn_down_exps.weight q3_K blk.3.ffn_down_shexp.weight q3_K blk.3.attn_output.weight q3_K blk.3.attn_v.weight q4_K blk.3.ffn_down_exps.weight q3_K +blk.4.attn_qkv.weight q4_K blk.4.ffn_down_shexp.weight q3_K blk.4.ffn_down_exps.weight q3_K +blk.5.attn_qkv.weight q4_K blk.5.ffn_down_shexp.weight q3_K blk.5.ffn_down_exps.weight q3_K +blk.6.attn_qkv.weight q4_K blk.6.ffn_down_exps.weight q3_K blk.6.ffn_down_shexp.weight q3_K blk.7.ffn_down_shexp.weight q3_K blk.7.attn_output.weight q3_K blk.7.attn_v.weight q4_K blk.7.ffn_down_exps.weight q3_K +blk.8.attn_qkv.weight q4_K blk.8.ffn_down_shexp.weight q3_K blk.8.ffn_down_exps.weight q3_K +blk.9.attn_qkv.weight q4_K blk.9.ffn_down_shexp.weight q3_K +blk.10.attn_qkv.weight q4_K blk.10.ffn_down_shexp.weight q3_K blk.9.ffn_down_exps.weight q3_K blk.10.ffn_down_exps.weight q3_K @@ -52,90 +61,117 @@ blk.11.ffn_down_shexp.weight q3_K blk.11.attn_output.weight q3_K blk.11.attn_v.weight q4_K blk.11.ffn_down_exps.weight q3_K +blk.12.attn_qkv.weight q4_K blk.12.ffn_down_exps.weight q3_K blk.12.ffn_down_shexp.weight q3_K +blk.13.attn_qkv.weight q4_K blk.13.ffn_down_shexp.weight q3_K blk.13.ffn_down_exps.weight q3_K +blk.14.attn_qkv.weight q4_K blk.14.ffn_down_shexp.weight q3_K blk.14.ffn_down_exps.weight q3_K blk.15.ffn_down_shexp.weight q3_K blk.15.attn_output.weight q3_K blk.15.attn_v.weight q4_K blk.15.ffn_down_exps.weight q3_K +blk.16.attn_qkv.weight q4_K blk.16.ffn_down_shexp.weight q3_K blk.16.ffn_down_exps.weight q3_K +blk.17.attn_qkv.weight q4_K blk.17.ffn_down_exps.weight q3_K blk.17.ffn_down_shexp.weight q3_K +blk.18.attn_qkv.weight q4_K blk.18.ffn_down_shexp.weight q3_K blk.18.ffn_down_exps.weight q3_K blk.19.ffn_down_shexp.weight q3_K blk.19.attn_output.weight q3_K blk.19.attn_v.weight q4_K blk.19.ffn_down_exps.weight q3_K +blk.20.attn_qkv.weight q4_K blk.20.ffn_down_shexp.weight q3_K blk.20.ffn_down_exps.weight q3_K +blk.21.attn_qkv.weight q4_K blk.21.ffn_down_shexp.weight q3_K blk.21.ffn_down_exps.weight q3_K +blk.22.attn_qkv.weight q4_K blk.22.ffn_down_shexp.weight q3_K blk.22.ffn_down_exps.weight q3_K blk.23.ffn_down_exps.weight q3_K blk.23.ffn_down_shexp.weight q3_K blk.23.attn_output.weight q3_K blk.23.attn_v.weight q4_K +blk.24.attn_qkv.weight q4_K blk.24.ffn_down_shexp.weight q3_K blk.24.ffn_down_exps.weight q3_K +blk.25.attn_qkv.weight q4_K blk.25.ffn_down_shexp.weight q3_K blk.25.ffn_down_exps.weight q3_K +blk.26.attn_qkv.weight q4_K blk.26.ffn_down_shexp.weight q3_K blk.26.ffn_down_exps.weight q3_K blk.27.ffn_down_shexp.weight q3_K blk.27.attn_output.weight q3_K blk.27.attn_v.weight q4_K blk.27.ffn_down_exps.weight q3_K +blk.28.attn_qkv.weight q4_K blk.28.ffn_down_shexp.weight q3_K blk.28.ffn_down_exps.weight q3_K +blk.29.attn_qkv.weight q4_K blk.29.ffn_down_exps.weight q3_K blk.29.ffn_down_shexp.weight q3_K +blk.30.attn_qkv.weight q4_K blk.30.ffn_down_shexp.weight q3_K blk.30.ffn_down_exps.weight q3_K blk.31.ffn_down_shexp.weight q3_K blk.31.attn_output.weight q3_K blk.31.attn_v.weight q4_K blk.31.ffn_down_exps.weight q3_K +blk.32.attn_qkv.weight q4_K blk.32.ffn_down_shexp.weight q3_K blk.32.ffn_down_exps.weight q3_K +blk.33.attn_qkv.weight q4_K blk.33.ffn_down_shexp.weight q3_K blk.33.ffn_down_exps.weight q3_K +blk.34.attn_qkv.weight q4_K blk.34.ffn_down_exps.weight q3_K blk.34.ffn_down_shexp.weight q3_K blk.35.ffn_down_shexp.weight q3_K blk.35.attn_output.weight q3_K blk.35.attn_v.weight q4_K blk.35.ffn_down_exps.weight q3_K +blk.36.attn_qkv.weight q4_K blk.36.ffn_down_shexp.weight q3_K blk.36.ffn_down_exps.weight q3_K +blk.37.attn_qkv.weight q4_K blk.37.ffn_down_shexp.weight q3_K blk.37.ffn_down_exps.weight q3_K +blk.38.attn_qkv.weight q4_K blk.38.ffn_down_shexp.weight q3_K blk.38.ffn_down_exps.weight q3_K blk.39.ffn_down_shexp.weight q3_K blk.39.attn_output.weight q3_K blk.39.attn_v.weight q4_K blk.39.ffn_down_exps.weight q3_K +blk.40.attn_qkv.weight q4_K blk.40.ffn_down_exps.weight q3_K blk.40.ffn_down_shexp.weight q3_K +blk.41.attn_qkv.weight q4_K blk.41.ffn_down_shexp.weight q3_K blk.41.ffn_down_exps.weight q3_K +blk.42.attn_qkv.weight q4_K blk.42.ffn_down_shexp.weight q3_K blk.42.ffn_down_exps.weight q3_K blk.43.ffn_down_shexp.weight q3_K blk.43.attn_output.weight q3_K blk.43.attn_v.weight q4_K blk.43.ffn_down_exps.weight q3_K +blk.44.attn_qkv.weight q4_K blk.44.ffn_down_shexp.weight q3_K blk.44.ffn_down_exps.weight q3_K +blk.45.attn_qkv.weight q4_K blk.45.ffn_down_shexp.weight q3_K blk.45.ffn_down_exps.weight q3_K +blk.46.attn_qkv.weight q4_K blk.46.ffn_down_exps.weight q3_K blk.46.ffn_down_shexp.weight q3_K blk.47.ffn_down_shexp.weight q3_K @@ -148,10 +184,10 @@ blk.47.ffn_down_exps.weight q3_K output.weight q6_K [Q3_K_M] q3_K -blk.0.attn_qkv.weight q4_K +blk.0.attn_qkv.weight q5_K blk.0.ffn_down_exps.weight q5_K blk.0.ffn_down_shexp.weight q5_K -blk.1.attn_qkv.weight q4_K +blk.1.attn_qkv.weight q5_K blk.1.ffn_down_shexp.weight q5_K blk.1.ffn_down_exps.weight q5_K blk.2.attn_qkv.weight q4_K @@ -159,7 +195,7 @@ blk.2.ffn_down_shexp.weight q5_K blk.2.ffn_down_exps.weight q5_K blk.3.ffn_down_shexp.weight q4_K blk.3.attn_output.weight q4_K -blk.3.attn_v.weight q5_K +blk.3.attn_v.weight q4_K blk.3.ffn_down_exps.weight q4_K blk.4.attn_qkv.weight q4_K blk.4.ffn_down_shexp.weight q4_K @@ -172,7 +208,7 @@ blk.6.ffn_down_exps.weight q4_K blk.6.ffn_down_shexp.weight q4_K blk.7.ffn_down_shexp.weight q4_K blk.7.attn_output.weight q4_K -blk.7.attn_v.weight q5_K +blk.7.attn_v.weight q4_K blk.7.ffn_down_exps.weight q4_K blk.8.attn_qkv.weight q4_K blk.8.ffn_down_shexp.weight q4_K @@ -307,38 +343,38 @@ output.weight q6_K blk.47.ffn_down_exps.weight q4_K [Q3_K_L] q3_K -blk.0.attn_qkv.weight q4_K +blk.0.attn_qkv.weight q5_K blk.0.ffn_down_exps.weight q5_K blk.0.ffn_down_shexp.weight q5_K -blk.1.attn_qkv.weight q4_K +blk.1.attn_qkv.weight q5_K blk.1.ffn_down_shexp.weight q5_K blk.1.ffn_down_exps.weight q5_K -blk.2.attn_qkv.weight q4_K +blk.2.attn_qkv.weight q5_K blk.2.ffn_down_shexp.weight q5_K blk.2.ffn_down_exps.weight q5_K blk.3.ffn_down_shexp.weight q5_K blk.3.attn_output.weight q5_K blk.3.attn_v.weight q5_K blk.3.ffn_down_exps.weight q5_K -blk.4.attn_qkv.weight q4_K +blk.4.attn_qkv.weight q5_K blk.4.ffn_down_shexp.weight q5_K blk.4.ffn_down_exps.weight q5_K -blk.5.attn_qkv.weight q4_K +blk.5.attn_qkv.weight q5_K blk.5.ffn_down_shexp.weight q5_K blk.5.ffn_down_exps.weight q5_K -blk.6.attn_qkv.weight q4_K +blk.6.attn_qkv.weight q5_K blk.6.ffn_down_exps.weight q5_K blk.6.ffn_down_shexp.weight q5_K blk.7.ffn_down_shexp.weight q5_K blk.7.attn_output.weight q5_K blk.7.attn_v.weight q5_K blk.7.ffn_down_exps.weight q5_K -blk.8.attn_qkv.weight q4_K +blk.8.attn_qkv.weight q5_K blk.8.ffn_down_shexp.weight q5_K blk.8.ffn_down_exps.weight q5_K -blk.9.attn_qkv.weight q4_K +blk.9.attn_qkv.weight q5_K blk.9.ffn_down_shexp.weight q5_K -blk.10.attn_qkv.weight q4_K +blk.10.attn_qkv.weight q5_K blk.10.ffn_down_shexp.weight q5_K blk.9.ffn_down_exps.weight q5_K blk.10.ffn_down_exps.weight q5_K @@ -346,117 +382,117 @@ blk.11.ffn_down_shexp.weight q5_K blk.11.attn_output.weight q5_K blk.11.attn_v.weight q5_K blk.11.ffn_down_exps.weight q5_K -blk.12.attn_qkv.weight q4_K +blk.12.attn_qkv.weight q5_K blk.12.ffn_down_exps.weight q5_K blk.12.ffn_down_shexp.weight q5_K -blk.13.attn_qkv.weight q4_K +blk.13.attn_qkv.weight q5_K blk.13.ffn_down_shexp.weight q5_K blk.13.ffn_down_exps.weight q5_K -blk.14.attn_qkv.weight q4_K +blk.14.attn_qkv.weight q5_K blk.14.ffn_down_shexp.weight q5_K blk.14.ffn_down_exps.weight q5_K blk.15.ffn_down_shexp.weight q5_K blk.15.attn_output.weight q5_K blk.15.attn_v.weight q5_K blk.15.ffn_down_exps.weight q5_K -blk.16.attn_qkv.weight q4_K +blk.16.attn_qkv.weight q5_K blk.16.ffn_down_shexp.weight q5_K blk.16.ffn_down_exps.weight q5_K -blk.17.attn_qkv.weight q4_K +blk.17.attn_qkv.weight q5_K blk.17.ffn_down_exps.weight q5_K blk.17.ffn_down_shexp.weight q5_K -blk.18.attn_qkv.weight q4_K +blk.18.attn_qkv.weight q5_K blk.18.ffn_down_shexp.weight q5_K blk.18.ffn_down_exps.weight q5_K blk.19.ffn_down_shexp.weight q5_K blk.19.attn_output.weight q5_K blk.19.attn_v.weight q5_K blk.19.ffn_down_exps.weight q5_K -blk.20.attn_qkv.weight q4_K +blk.20.attn_qkv.weight q5_K blk.20.ffn_down_shexp.weight q5_K blk.20.ffn_down_exps.weight q5_K -blk.21.attn_qkv.weight q4_K +blk.21.attn_qkv.weight q5_K blk.21.ffn_down_shexp.weight q5_K blk.21.ffn_down_exps.weight q5_K -blk.22.attn_qkv.weight q4_K +blk.22.attn_qkv.weight q5_K blk.22.ffn_down_shexp.weight q5_K blk.22.ffn_down_exps.weight q5_K blk.23.ffn_down_exps.weight q5_K blk.23.ffn_down_shexp.weight q5_K blk.23.attn_output.weight q5_K blk.23.attn_v.weight q5_K -blk.24.attn_qkv.weight q4_K +blk.24.attn_qkv.weight q5_K blk.24.ffn_down_shexp.weight q5_K blk.24.ffn_down_exps.weight q5_K -blk.25.attn_qkv.weight q4_K +blk.25.attn_qkv.weight q5_K blk.25.ffn_down_shexp.weight q5_K blk.25.ffn_down_exps.weight q5_K -blk.26.attn_qkv.weight q4_K +blk.26.attn_qkv.weight q5_K blk.26.ffn_down_shexp.weight q5_K blk.26.ffn_down_exps.weight q5_K blk.27.ffn_down_shexp.weight q5_K blk.27.attn_output.weight q5_K blk.27.attn_v.weight q5_K blk.27.ffn_down_exps.weight q5_K -blk.28.attn_qkv.weight q4_K +blk.28.attn_qkv.weight q5_K blk.28.ffn_down_shexp.weight q5_K blk.28.ffn_down_exps.weight q5_K -blk.29.attn_qkv.weight q4_K +blk.29.attn_qkv.weight q5_K blk.29.ffn_down_exps.weight q5_K blk.29.ffn_down_shexp.weight q5_K -blk.30.attn_qkv.weight q4_K +blk.30.attn_qkv.weight q5_K blk.30.ffn_down_shexp.weight q5_K blk.30.ffn_down_exps.weight q5_K blk.31.ffn_down_shexp.weight q5_K blk.31.attn_output.weight q5_K blk.31.attn_v.weight q5_K blk.31.ffn_down_exps.weight q5_K -blk.32.attn_qkv.weight q4_K +blk.32.attn_qkv.weight q5_K blk.32.ffn_down_shexp.weight q5_K blk.32.ffn_down_exps.weight q5_K -blk.33.attn_qkv.weight q4_K +blk.33.attn_qkv.weight q5_K blk.33.ffn_down_shexp.weight q5_K blk.33.ffn_down_exps.weight q5_K -blk.34.attn_qkv.weight q4_K +blk.34.attn_qkv.weight q5_K blk.34.ffn_down_exps.weight q5_K blk.34.ffn_down_shexp.weight q5_K blk.35.ffn_down_shexp.weight q5_K blk.35.attn_output.weight q5_K blk.35.attn_v.weight q5_K blk.35.ffn_down_exps.weight q5_K -blk.36.attn_qkv.weight q4_K +blk.36.attn_qkv.weight q5_K blk.36.ffn_down_shexp.weight q5_K blk.36.ffn_down_exps.weight q5_K -blk.37.attn_qkv.weight q4_K +blk.37.attn_qkv.weight q5_K blk.37.ffn_down_shexp.weight q5_K blk.37.ffn_down_exps.weight q5_K -blk.38.attn_qkv.weight q4_K +blk.38.attn_qkv.weight q5_K blk.38.ffn_down_shexp.weight q5_K blk.38.ffn_down_exps.weight q5_K blk.39.ffn_down_shexp.weight q5_K blk.39.attn_output.weight q5_K blk.39.attn_v.weight q5_K blk.39.ffn_down_exps.weight q5_K -blk.40.attn_qkv.weight q4_K +blk.40.attn_qkv.weight q5_K blk.40.ffn_down_exps.weight q5_K blk.40.ffn_down_shexp.weight q5_K -blk.41.attn_qkv.weight q4_K +blk.41.attn_qkv.weight q5_K blk.41.ffn_down_shexp.weight q5_K blk.41.ffn_down_exps.weight q5_K -blk.42.attn_qkv.weight q4_K +blk.42.attn_qkv.weight q5_K blk.42.ffn_down_shexp.weight q5_K blk.42.ffn_down_exps.weight q5_K blk.43.ffn_down_shexp.weight q5_K blk.43.attn_output.weight q5_K blk.43.attn_v.weight q5_K blk.43.ffn_down_exps.weight q5_K -blk.44.attn_qkv.weight q4_K +blk.44.attn_qkv.weight q5_K blk.44.ffn_down_shexp.weight q5_K blk.44.ffn_down_exps.weight q5_K -blk.45.attn_qkv.weight q4_K +blk.45.attn_qkv.weight q5_K blk.45.ffn_down_shexp.weight q5_K blk.45.ffn_down_exps.weight q5_K -blk.46.attn_qkv.weight q4_K +blk.46.attn_qkv.weight q5_K blk.46.ffn_down_exps.weight q5_K blk.46.ffn_down_shexp.weight q5_K blk.47.ffn_down_shexp.weight q5_K @@ -466,10 +502,13 @@ output.weight q6_K blk.47.ffn_down_exps.weight q5_K [Q4_K_S] q4_K +blk.0.attn_qkv.weight q5_K blk.0.ffn_down_exps.weight q5_K blk.0.ffn_down_shexp.weight q5_K +blk.1.attn_qkv.weight q5_K blk.1.ffn_down_shexp.weight q5_K blk.1.ffn_down_exps.weight q5_K +blk.2.attn_qkv.weight q5_K blk.2.ffn_down_shexp.weight q5_K blk.2.ffn_down_exps.weight q5_K blk.3.ffn_down_shexp.weight q5_K @@ -479,99 +518,76 @@ blk.4.ffn_down_shexp.weight q5_K blk.4.ffn_down_exps.weight q5_K blk.5.ffn_down_shexp.weight q5_K blk.5.ffn_down_exps.weight q5_K -blk.7.attn_v.weight q5_K -blk.11.attn_v.weight q5_K -blk.15.attn_v.weight q5_K output.weight q6_K [Q4_K_M] q4_K -blk.0.attn_qkv.weight q5_K +blk.0.attn_qkv.weight q6_K blk.0.ffn_down_exps.weight q6_K blk.0.ffn_down_shexp.weight q6_K -blk.1.attn_qkv.weight q5_K +blk.1.attn_qkv.weight q6_K blk.1.ffn_down_shexp.weight q6_K blk.1.ffn_down_exps.weight q6_K -blk.2.attn_qkv.weight q5_K +blk.2.attn_qkv.weight q6_K blk.2.ffn_down_shexp.weight q6_K blk.2.ffn_down_exps.weight q6_K blk.3.ffn_down_shexp.weight q6_K blk.3.attn_v.weight q6_K blk.3.ffn_down_exps.weight q6_K -blk.4.attn_qkv.weight q5_K +blk.4.attn_qkv.weight q6_K blk.4.ffn_down_shexp.weight q6_K blk.4.ffn_down_exps.weight q6_K -blk.5.attn_qkv.weight q5_K +blk.5.attn_qkv.weight q6_K blk.5.ffn_down_shexp.weight q6_K blk.5.ffn_down_exps.weight q6_K -blk.6.attn_qkv.weight q5_K -blk.7.attn_v.weight q6_K -blk.8.attn_qkv.weight q5_K +blk.8.attn_qkv.weight q6_K blk.8.ffn_down_shexp.weight q6_K blk.8.ffn_down_exps.weight q6_K -blk.9.attn_qkv.weight q5_K -blk.10.attn_qkv.weight q5_K blk.11.ffn_down_shexp.weight q6_K blk.11.attn_v.weight q6_K blk.11.ffn_down_exps.weight q6_K -blk.12.attn_qkv.weight q5_K -blk.13.attn_qkv.weight q5_K -blk.14.attn_qkv.weight q5_K +blk.14.attn_qkv.weight q6_K blk.14.ffn_down_shexp.weight q6_K blk.14.ffn_down_exps.weight q6_K -blk.15.attn_v.weight q6_K -blk.16.attn_qkv.weight q5_K -blk.17.attn_qkv.weight q5_K +blk.17.attn_qkv.weight q6_K blk.17.ffn_down_exps.weight q6_K blk.17.ffn_down_shexp.weight q6_K -blk.18.attn_qkv.weight q5_K -blk.19.attn_v.weight q6_K -blk.20.attn_qkv.weight q5_K +blk.20.attn_qkv.weight q6_K blk.20.ffn_down_shexp.weight q6_K blk.20.ffn_down_exps.weight q6_K -blk.21.attn_qkv.weight q5_K -blk.22.attn_qkv.weight q5_K blk.23.ffn_down_exps.weight q6_K blk.23.ffn_down_shexp.weight q6_K blk.23.attn_v.weight q6_K -blk.24.attn_qkv.weight q5_K -blk.25.attn_qkv.weight q5_K -blk.26.attn_qkv.weight q5_K +blk.26.attn_qkv.weight q6_K blk.26.ffn_down_shexp.weight q6_K blk.26.ffn_down_exps.weight q6_K -blk.28.attn_qkv.weight q5_K -blk.29.attn_qkv.weight q5_K +blk.29.attn_qkv.weight q6_K blk.29.ffn_down_exps.weight q6_K blk.29.ffn_down_shexp.weight q6_K -blk.30.attn_qkv.weight q5_K -blk.32.attn_qkv.weight q5_K +blk.32.attn_qkv.weight q6_K blk.32.ffn_down_shexp.weight q6_K blk.32.ffn_down_exps.weight q6_K -blk.33.attn_qkv.weight q5_K -blk.34.attn_qkv.weight q5_K blk.35.ffn_down_shexp.weight q6_K blk.35.attn_v.weight q6_K blk.35.ffn_down_exps.weight q6_K -blk.36.attn_qkv.weight q5_K -blk.37.attn_qkv.weight q5_K -blk.38.attn_qkv.weight q5_K +blk.38.attn_qkv.weight q6_K blk.38.ffn_down_shexp.weight q6_K blk.38.ffn_down_exps.weight q6_K -blk.40.attn_qkv.weight q5_K -blk.41.attn_qkv.weight q5_K +blk.41.attn_qkv.weight q6_K blk.41.ffn_down_shexp.weight q6_K blk.41.ffn_down_exps.weight q6_K -blk.42.attn_qkv.weight q5_K +blk.42.attn_qkv.weight q6_K blk.42.ffn_down_shexp.weight q6_K blk.42.ffn_down_exps.weight q6_K blk.43.ffn_down_shexp.weight q6_K +blk.43.attn_v.weight q6_K blk.43.ffn_down_exps.weight q6_K -blk.44.attn_qkv.weight q5_K +blk.44.attn_qkv.weight q6_K blk.44.ffn_down_shexp.weight q6_K blk.44.ffn_down_exps.weight q6_K -blk.45.attn_qkv.weight q5_K +blk.45.attn_qkv.weight q6_K blk.45.ffn_down_shexp.weight q6_K blk.45.ffn_down_exps.weight q6_K -blk.46.attn_qkv.weight q5_K +blk.46.attn_qkv.weight q6_K blk.46.ffn_down_exps.weight q6_K blk.46.ffn_down_shexp.weight q6_K blk.47.ffn_down_shexp.weight q6_K @@ -601,60 +617,39 @@ blk.4.ffn_down_exps.weight q6_K blk.5.attn_qkv.weight q6_K blk.5.ffn_down_shexp.weight q6_K blk.5.ffn_down_exps.weight q6_K -blk.6.attn_qkv.weight q6_K -blk.7.attn_v.weight q6_K blk.8.attn_qkv.weight q6_K blk.8.ffn_down_shexp.weight q6_K blk.8.ffn_down_exps.weight q6_K -blk.9.attn_qkv.weight q6_K -blk.10.attn_qkv.weight q6_K blk.11.ffn_down_shexp.weight q6_K blk.11.attn_v.weight q6_K blk.11.ffn_down_exps.weight q6_K -blk.12.attn_qkv.weight q6_K -blk.13.attn_qkv.weight q6_K blk.14.attn_qkv.weight q6_K blk.14.ffn_down_shexp.weight q6_K blk.14.ffn_down_exps.weight q6_K -blk.15.attn_v.weight q6_K -blk.16.attn_qkv.weight q6_K blk.17.attn_qkv.weight q6_K blk.17.ffn_down_exps.weight q6_K blk.17.ffn_down_shexp.weight q6_K -blk.18.attn_qkv.weight q6_K -blk.19.attn_v.weight q6_K blk.20.attn_qkv.weight q6_K blk.20.ffn_down_shexp.weight q6_K blk.20.ffn_down_exps.weight q6_K -blk.21.attn_qkv.weight q6_K -blk.22.attn_qkv.weight q6_K blk.23.ffn_down_exps.weight q6_K blk.23.ffn_down_shexp.weight q6_K blk.23.attn_v.weight q6_K -blk.24.attn_qkv.weight q6_K -blk.25.attn_qkv.weight q6_K blk.26.attn_qkv.weight q6_K blk.26.ffn_down_shexp.weight q6_K blk.26.ffn_down_exps.weight q6_K -blk.28.attn_qkv.weight q6_K blk.29.attn_qkv.weight q6_K blk.29.ffn_down_exps.weight q6_K blk.29.ffn_down_shexp.weight q6_K -blk.30.attn_qkv.weight q6_K blk.32.attn_qkv.weight q6_K blk.32.ffn_down_shexp.weight q6_K blk.32.ffn_down_exps.weight q6_K -blk.33.attn_qkv.weight q6_K -blk.34.attn_qkv.weight q6_K blk.35.ffn_down_shexp.weight q6_K blk.35.attn_v.weight q6_K blk.35.ffn_down_exps.weight q6_K -blk.36.attn_qkv.weight q6_K -blk.37.attn_qkv.weight q6_K blk.38.attn_qkv.weight q6_K blk.38.ffn_down_shexp.weight q6_K blk.38.ffn_down_exps.weight q6_K -blk.40.attn_qkv.weight q6_K blk.41.attn_qkv.weight q6_K blk.41.ffn_down_shexp.weight q6_K blk.41.ffn_down_exps.weight q6_K @@ -662,6 +657,7 @@ blk.42.attn_qkv.weight q6_K blk.42.ffn_down_shexp.weight q6_K blk.42.ffn_down_exps.weight q6_K blk.43.ffn_down_shexp.weight q6_K +blk.43.attn_v.weight q6_K blk.43.ffn_down_exps.weight q6_K blk.44.attn_qkv.weight q6_K blk.44.ffn_down_shexp.weight q6_K @@ -681,79 +677,193 @@ blk.47.ffn_down_exps.weight q6_K [IQ2_XXS] iq2_xxs token_embd.weight q2_K +blk.0.attn_qkv.weight q4_K blk.0.ffn_down_exps.weight q2_K blk.0.ffn_down_shexp.weight q2_K +blk.1.attn_qkv.weight q4_K blk.1.ffn_down_shexp.weight q2_K blk.1.ffn_down_exps.weight q2_K +blk.2.attn_qkv.weight q4_K blk.2.ffn_down_shexp.weight q2_K blk.2.ffn_down_exps.weight q2_K blk.3.attn_v.weight q4_K +blk.4.attn_qkv.weight q4_K +blk.5.attn_qkv.weight q4_K +blk.6.attn_qkv.weight q4_K blk.7.attn_v.weight q4_K +blk.8.attn_qkv.weight q4_K +blk.9.attn_qkv.weight q4_K +blk.10.attn_qkv.weight q4_K blk.11.attn_v.weight q4_K +blk.12.attn_qkv.weight q4_K +blk.13.attn_qkv.weight q4_K +blk.14.attn_qkv.weight q4_K blk.15.attn_v.weight q4_K +blk.16.attn_qkv.weight q4_K +blk.17.attn_qkv.weight q4_K +blk.18.attn_qkv.weight q4_K blk.19.attn_v.weight q4_K +blk.20.attn_qkv.weight q4_K +blk.21.attn_qkv.weight q4_K +blk.22.attn_qkv.weight q4_K blk.23.attn_v.weight q4_K +blk.24.attn_qkv.weight q4_K +blk.25.attn_qkv.weight q4_K +blk.26.attn_qkv.weight q4_K blk.27.attn_v.weight q4_K +blk.28.attn_qkv.weight q4_K +blk.29.attn_qkv.weight q4_K +blk.30.attn_qkv.weight q4_K blk.31.attn_v.weight q4_K +blk.32.attn_qkv.weight q4_K +blk.33.attn_qkv.weight q4_K +blk.34.attn_qkv.weight q4_K blk.35.attn_v.weight q4_K +blk.36.attn_qkv.weight q4_K +blk.37.attn_qkv.weight q4_K +blk.38.attn_qkv.weight q4_K blk.39.attn_v.weight q4_K +blk.40.attn_qkv.weight q4_K +blk.41.attn_qkv.weight q4_K +blk.42.attn_qkv.weight q4_K blk.43.attn_v.weight q4_K +blk.44.attn_qkv.weight q4_K +blk.45.attn_qkv.weight q4_K +blk.46.attn_qkv.weight q4_K blk.47.attn_v.weight q4_K output.weight q5_K [IQ2_XS] iq2_xs token_embd.weight q2_K +blk.0.attn_qkv.weight q4_K blk.0.ffn_down_exps.weight q2_K blk.0.ffn_down_shexp.weight q2_K +blk.1.attn_qkv.weight q4_K blk.1.ffn_down_shexp.weight q2_K blk.1.ffn_down_exps.weight q2_K +blk.2.attn_qkv.weight q4_K blk.2.ffn_down_shexp.weight q2_K blk.2.ffn_down_exps.weight q2_K blk.3.attn_v.weight q4_K +blk.4.attn_qkv.weight q4_K +blk.5.attn_qkv.weight q4_K +blk.6.attn_qkv.weight q4_K blk.7.attn_v.weight q4_K +blk.8.attn_qkv.weight q4_K +blk.9.attn_qkv.weight q4_K +blk.10.attn_qkv.weight q4_K blk.11.attn_v.weight q4_K +blk.12.attn_qkv.weight q4_K +blk.13.attn_qkv.weight q4_K +blk.14.attn_qkv.weight q4_K blk.15.attn_v.weight q4_K +blk.16.attn_qkv.weight q4_K +blk.17.attn_qkv.weight q4_K +blk.18.attn_qkv.weight q4_K blk.19.attn_v.weight q4_K +blk.20.attn_qkv.weight q4_K +blk.21.attn_qkv.weight q4_K +blk.22.attn_qkv.weight q4_K blk.23.attn_v.weight q4_K +blk.24.attn_qkv.weight q4_K +blk.25.attn_qkv.weight q4_K +blk.26.attn_qkv.weight q4_K blk.27.attn_v.weight q4_K +blk.28.attn_qkv.weight q4_K +blk.29.attn_qkv.weight q4_K +blk.30.attn_qkv.weight q4_K blk.31.attn_v.weight q4_K +blk.32.attn_qkv.weight q4_K +blk.33.attn_qkv.weight q4_K +blk.34.attn_qkv.weight q4_K blk.35.attn_v.weight q4_K +blk.36.attn_qkv.weight q4_K +blk.37.attn_qkv.weight q4_K +blk.38.attn_qkv.weight q4_K blk.39.attn_v.weight q4_K +blk.40.attn_qkv.weight q4_K +blk.41.attn_qkv.weight q4_K +blk.42.attn_qkv.weight q4_K blk.43.attn_v.weight q4_K +blk.44.attn_qkv.weight q4_K +blk.45.attn_qkv.weight q4_K +blk.46.attn_qkv.weight q4_K blk.47.attn_v.weight q4_K output.weight q5_K [Q2_K_S] q2_K +blk.0.attn_qkv.weight q4_K blk.0.ffn_down_exps.weight q4_K blk.0.ffn_down_shexp.weight q4_K +blk.1.attn_qkv.weight q4_K blk.1.ffn_down_shexp.weight q4_K blk.1.ffn_down_exps.weight q4_K +blk.2.attn_qkv.weight q4_K blk.2.ffn_down_shexp.weight q4_K blk.2.ffn_down_exps.weight q4_K blk.3.ffn_down_shexp.weight q4_K blk.3.attn_v.weight q4_K blk.3.ffn_down_exps.weight q4_K +blk.4.attn_qkv.weight q4_K blk.4.ffn_down_shexp.weight q4_K blk.4.ffn_down_exps.weight q4_K +blk.5.attn_qkv.weight q4_K blk.5.ffn_down_shexp.weight q4_K blk.5.ffn_down_exps.weight q4_K +blk.6.attn_qkv.weight q4_K blk.7.attn_v.weight q4_K +blk.8.attn_qkv.weight q4_K +blk.9.attn_qkv.weight q4_K +blk.10.attn_qkv.weight q4_K blk.11.attn_v.weight q4_K +blk.12.attn_qkv.weight q4_K +blk.13.attn_qkv.weight q4_K +blk.14.attn_qkv.weight q4_K blk.15.attn_v.weight q4_K +blk.16.attn_qkv.weight q4_K +blk.17.attn_qkv.weight q4_K +blk.18.attn_qkv.weight q4_K blk.19.attn_v.weight q4_K +blk.20.attn_qkv.weight q4_K +blk.21.attn_qkv.weight q4_K +blk.22.attn_qkv.weight q4_K blk.23.attn_v.weight q4_K +blk.24.attn_qkv.weight q4_K +blk.25.attn_qkv.weight q4_K +blk.26.attn_qkv.weight q4_K blk.27.attn_v.weight q4_K +blk.28.attn_qkv.weight q4_K +blk.29.attn_qkv.weight q4_K +blk.30.attn_qkv.weight q4_K blk.31.attn_v.weight q4_K +blk.32.attn_qkv.weight q4_K +blk.33.attn_qkv.weight q4_K +blk.34.attn_qkv.weight q4_K blk.35.attn_v.weight q4_K +blk.36.attn_qkv.weight q4_K +blk.37.attn_qkv.weight q4_K +blk.38.attn_qkv.weight q4_K blk.39.attn_v.weight q4_K +blk.40.attn_qkv.weight q4_K +blk.41.attn_qkv.weight q4_K +blk.42.attn_qkv.weight q4_K blk.43.attn_v.weight q4_K +blk.44.attn_qkv.weight q4_K +blk.45.attn_qkv.weight q4_K +blk.46.attn_qkv.weight q4_K blk.47.attn_v.weight q4_K output.weight q6_K [IQ3_XS] iq3_s +blk.0.attn_qkv.weight q4_K +blk.1.attn_qkv.weight q4_K +blk.2.attn_qkv.weight q4_K blk.3.attn_k.weight iq3_xxs blk.3.attn_q.weight iq3_xxs blk.3.attn_v.weight q4_K +blk.4.attn_qkv.weight q4_K +blk.5.attn_qkv.weight q4_K +blk.6.attn_qkv.weight q4_K blk.6.ffn_gate_exps.weight iq3_xxs blk.6.ffn_up_exps.weight iq3_xxs blk.6.ffn_gate_shexp.weight iq3_xxs @@ -765,12 +875,15 @@ blk.7.attn_q.weight iq3_xxs blk.7.attn_v.weight q4_K blk.7.ffn_gate_exps.weight iq3_xxs blk.7.ffn_up_exps.weight iq3_xxs +blk.8.attn_qkv.weight q4_K blk.8.ffn_gate_shexp.weight iq3_xxs blk.8.ffn_up_shexp.weight iq3_xxs blk.8.ffn_gate_exps.weight iq3_xxs blk.8.ffn_up_exps.weight iq3_xxs +blk.9.attn_qkv.weight q4_K blk.9.ffn_gate_shexp.weight iq3_xxs blk.9.ffn_up_shexp.weight iq3_xxs +blk.10.attn_qkv.weight q4_K blk.10.ffn_gate_shexp.weight iq3_xxs blk.10.ffn_up_shexp.weight iq3_xxs blk.9.ffn_gate_exps.weight iq3_xxs @@ -784,14 +897,17 @@ blk.11.attn_q.weight iq3_xxs blk.11.attn_v.weight q4_K blk.11.ffn_gate_exps.weight iq3_xxs blk.11.ffn_up_exps.weight iq3_xxs +blk.12.attn_qkv.weight q4_K blk.12.ffn_gate_exps.weight iq3_xxs blk.12.ffn_up_exps.weight iq3_xxs blk.12.ffn_gate_shexp.weight iq3_xxs blk.12.ffn_up_shexp.weight iq3_xxs +blk.13.attn_qkv.weight q4_K blk.13.ffn_gate_shexp.weight iq3_xxs blk.13.ffn_up_shexp.weight iq3_xxs blk.13.ffn_gate_exps.weight iq3_xxs blk.13.ffn_up_exps.weight iq3_xxs +blk.14.attn_qkv.weight q4_K blk.14.ffn_gate_shexp.weight iq3_xxs blk.14.ffn_up_shexp.weight iq3_xxs blk.14.ffn_gate_exps.weight iq3_xxs @@ -803,14 +919,17 @@ blk.15.attn_q.weight iq3_xxs blk.15.attn_v.weight q4_K blk.15.ffn_gate_exps.weight iq3_xxs blk.15.ffn_up_exps.weight iq3_xxs +blk.16.attn_qkv.weight q4_K blk.16.ffn_gate_shexp.weight iq3_xxs blk.16.ffn_up_shexp.weight iq3_xxs blk.16.ffn_gate_exps.weight iq3_xxs blk.16.ffn_up_exps.weight iq3_xxs +blk.17.attn_qkv.weight q4_K blk.17.ffn_gate_exps.weight iq3_xxs blk.17.ffn_up_exps.weight iq3_xxs blk.17.ffn_gate_shexp.weight iq3_xxs blk.17.ffn_up_shexp.weight iq3_xxs +blk.18.attn_qkv.weight q4_K blk.18.ffn_gate_shexp.weight iq3_xxs blk.18.ffn_up_shexp.weight iq3_xxs blk.18.ffn_gate_exps.weight iq3_xxs @@ -822,14 +941,17 @@ blk.19.attn_q.weight iq3_xxs blk.19.attn_v.weight q4_K blk.19.ffn_gate_exps.weight iq3_xxs blk.19.ffn_up_exps.weight iq3_xxs +blk.20.attn_qkv.weight q4_K blk.20.ffn_gate_shexp.weight iq3_xxs blk.20.ffn_up_shexp.weight iq3_xxs blk.20.ffn_gate_exps.weight iq3_xxs blk.20.ffn_up_exps.weight iq3_xxs +blk.21.attn_qkv.weight q4_K blk.21.ffn_gate_shexp.weight iq3_xxs blk.21.ffn_up_shexp.weight iq3_xxs blk.21.ffn_gate_exps.weight iq3_xxs blk.21.ffn_up_exps.weight iq3_xxs +blk.22.attn_qkv.weight q4_K blk.22.ffn_gate_shexp.weight iq3_xxs blk.22.ffn_up_shexp.weight iq3_xxs blk.22.ffn_gate_exps.weight iq3_xxs @@ -841,14 +963,17 @@ blk.23.ffn_up_shexp.weight iq3_xxs blk.23.attn_k.weight iq3_xxs blk.23.attn_q.weight iq3_xxs blk.23.attn_v.weight q4_K +blk.24.attn_qkv.weight q4_K blk.24.ffn_gate_shexp.weight iq3_xxs blk.24.ffn_up_shexp.weight iq3_xxs blk.24.ffn_gate_exps.weight iq3_xxs blk.24.ffn_up_exps.weight iq3_xxs +blk.25.attn_qkv.weight q4_K blk.25.ffn_gate_shexp.weight iq3_xxs blk.25.ffn_up_shexp.weight iq3_xxs blk.25.ffn_gate_exps.weight iq3_xxs blk.25.ffn_up_exps.weight iq3_xxs +blk.26.attn_qkv.weight q4_K blk.26.ffn_gate_shexp.weight iq3_xxs blk.26.ffn_up_shexp.weight iq3_xxs blk.26.ffn_gate_exps.weight iq3_xxs @@ -860,14 +985,17 @@ blk.27.attn_q.weight iq3_xxs blk.27.attn_v.weight q4_K blk.27.ffn_gate_exps.weight iq3_xxs blk.27.ffn_up_exps.weight iq3_xxs +blk.28.attn_qkv.weight q4_K blk.28.ffn_gate_shexp.weight iq3_xxs blk.28.ffn_up_shexp.weight iq3_xxs blk.28.ffn_gate_exps.weight iq3_xxs blk.28.ffn_up_exps.weight iq3_xxs +blk.29.attn_qkv.weight q4_K blk.29.ffn_gate_exps.weight iq3_xxs blk.29.ffn_up_exps.weight iq3_xxs blk.29.ffn_gate_shexp.weight iq3_xxs blk.29.ffn_up_shexp.weight iq3_xxs +blk.30.attn_qkv.weight q4_K blk.30.ffn_gate_shexp.weight iq3_xxs blk.30.ffn_up_shexp.weight iq3_xxs blk.30.ffn_gate_exps.weight iq3_xxs @@ -879,14 +1007,17 @@ blk.31.attn_q.weight iq3_xxs blk.31.attn_v.weight q4_K blk.31.ffn_gate_exps.weight iq3_xxs blk.31.ffn_up_exps.weight iq3_xxs +blk.32.attn_qkv.weight q4_K blk.32.ffn_gate_shexp.weight iq3_xxs blk.32.ffn_up_shexp.weight iq3_xxs blk.32.ffn_gate_exps.weight iq3_xxs blk.32.ffn_up_exps.weight iq3_xxs +blk.33.attn_qkv.weight q4_K blk.33.ffn_gate_shexp.weight iq3_xxs blk.33.ffn_up_shexp.weight iq3_xxs blk.33.ffn_gate_exps.weight iq3_xxs blk.33.ffn_up_exps.weight iq3_xxs +blk.34.attn_qkv.weight q4_K blk.34.ffn_gate_exps.weight iq3_xxs blk.34.ffn_up_exps.weight iq3_xxs blk.34.ffn_gate_shexp.weight iq3_xxs @@ -898,14 +1029,17 @@ blk.35.attn_q.weight iq3_xxs blk.35.attn_v.weight q4_K blk.35.ffn_gate_exps.weight iq3_xxs blk.35.ffn_up_exps.weight iq3_xxs +blk.36.attn_qkv.weight q4_K blk.36.ffn_gate_shexp.weight iq3_xxs blk.36.ffn_up_shexp.weight iq3_xxs blk.36.ffn_gate_exps.weight iq3_xxs blk.36.ffn_up_exps.weight iq3_xxs +blk.37.attn_qkv.weight q4_K blk.37.ffn_gate_shexp.weight iq3_xxs blk.37.ffn_up_shexp.weight iq3_xxs blk.37.ffn_gate_exps.weight iq3_xxs blk.37.ffn_up_exps.weight iq3_xxs +blk.38.attn_qkv.weight q4_K blk.38.ffn_gate_shexp.weight iq3_xxs blk.38.ffn_up_shexp.weight iq3_xxs blk.38.ffn_gate_exps.weight iq3_xxs @@ -917,17 +1051,23 @@ blk.39.attn_q.weight iq3_xxs blk.39.attn_v.weight q4_K blk.39.ffn_gate_exps.weight iq3_xxs blk.39.ffn_up_exps.weight iq3_xxs +blk.40.attn_qkv.weight q4_K blk.40.ffn_gate_exps.weight iq3_xxs blk.40.ffn_up_exps.weight iq3_xxs blk.40.ffn_gate_shexp.weight iq3_xxs blk.40.ffn_up_shexp.weight iq3_xxs +blk.41.attn_qkv.weight q4_K blk.41.ffn_gate_shexp.weight iq3_xxs blk.41.ffn_up_shexp.weight iq3_xxs blk.41.ffn_gate_exps.weight iq3_xxs blk.41.ffn_up_exps.weight iq3_xxs +blk.42.attn_qkv.weight q4_K blk.43.attn_k.weight iq3_xxs blk.43.attn_q.weight iq3_xxs blk.43.attn_v.weight q4_K +blk.44.attn_qkv.weight q4_K +blk.45.attn_qkv.weight q4_K +blk.46.attn_qkv.weight q4_K blk.47.attn_k.weight iq3_xxs blk.47.attn_q.weight iq3_xxs blk.47.attn_v.weight q4_K @@ -935,10 +1075,13 @@ output.weight q6_K [IQ3_XXS] iq3_xxs token_embd.weight iq3_s +blk.0.attn_qkv.weight q4_K blk.0.ffn_down_exps.weight q4_K blk.0.ffn_down_shexp.weight q4_K +blk.1.attn_qkv.weight q4_K blk.1.ffn_down_shexp.weight q4_K blk.1.ffn_down_exps.weight q4_K +blk.2.attn_qkv.weight q4_K blk.2.ffn_down_shexp.weight q4_K blk.2.ffn_down_exps.weight q4_K blk.3.ffn_down_shexp.weight q4_K @@ -947,10 +1090,13 @@ blk.3.attn_output.weight iq3_s blk.3.attn_q.weight iq2_s blk.3.attn_v.weight q4_K blk.3.ffn_down_exps.weight q4_K +blk.4.attn_qkv.weight q4_K blk.4.ffn_down_shexp.weight q4_K blk.4.ffn_down_exps.weight q4_K +blk.5.attn_qkv.weight q4_K blk.5.ffn_down_shexp.weight q4_K blk.5.ffn_down_exps.weight q4_K +blk.6.attn_qkv.weight q4_K blk.6.ffn_down_exps.weight q3_K blk.6.ffn_down_shexp.weight q3_K blk.7.ffn_down_shexp.weight q3_K @@ -959,9 +1105,12 @@ blk.7.attn_output.weight iq3_s blk.7.attn_q.weight iq2_s blk.7.attn_v.weight q4_K blk.7.ffn_down_exps.weight q3_K +blk.8.attn_qkv.weight q4_K blk.8.ffn_down_shexp.weight q3_K blk.8.ffn_down_exps.weight q3_K +blk.9.attn_qkv.weight q4_K blk.9.ffn_down_shexp.weight q3_K +blk.10.attn_qkv.weight q4_K blk.10.ffn_down_shexp.weight q3_K blk.9.ffn_down_exps.weight q3_K blk.10.ffn_down_exps.weight q3_K @@ -971,10 +1120,13 @@ blk.11.attn_output.weight iq3_s blk.11.attn_q.weight iq2_s blk.11.attn_v.weight q4_K blk.11.ffn_down_exps.weight q3_K +blk.12.attn_qkv.weight q4_K blk.12.ffn_down_exps.weight q3_K blk.12.ffn_down_shexp.weight q3_K +blk.13.attn_qkv.weight q4_K blk.13.ffn_down_shexp.weight q3_K blk.13.ffn_down_exps.weight q3_K +blk.14.attn_qkv.weight q4_K blk.14.ffn_down_shexp.weight q3_K blk.14.ffn_down_exps.weight q3_K blk.15.ffn_down_shexp.weight q3_K @@ -983,10 +1135,13 @@ blk.15.attn_output.weight iq3_s blk.15.attn_q.weight iq2_s blk.15.attn_v.weight q4_K blk.15.ffn_down_exps.weight q3_K +blk.16.attn_qkv.weight q4_K blk.16.ffn_down_shexp.weight q3_K blk.16.ffn_down_exps.weight q3_K +blk.17.attn_qkv.weight q4_K blk.17.ffn_down_exps.weight q3_K blk.17.ffn_down_shexp.weight q3_K +blk.18.attn_qkv.weight q4_K blk.18.ffn_down_shexp.weight q3_K blk.18.ffn_down_exps.weight q3_K blk.19.ffn_down_shexp.weight q3_K @@ -995,10 +1150,13 @@ blk.19.attn_output.weight iq3_s blk.19.attn_q.weight iq2_s blk.19.attn_v.weight q4_K blk.19.ffn_down_exps.weight q3_K +blk.20.attn_qkv.weight q4_K blk.20.ffn_down_shexp.weight q3_K blk.20.ffn_down_exps.weight q3_K +blk.21.attn_qkv.weight q4_K blk.21.ffn_down_shexp.weight q3_K blk.21.ffn_down_exps.weight q3_K +blk.22.attn_qkv.weight q4_K blk.22.ffn_down_shexp.weight q3_K blk.22.ffn_down_exps.weight q3_K blk.23.ffn_down_exps.weight q3_K @@ -1007,10 +1165,13 @@ blk.23.attn_k.weight iq2_s blk.23.attn_output.weight iq3_s blk.23.attn_q.weight iq2_s blk.23.attn_v.weight q4_K +blk.24.attn_qkv.weight q4_K blk.24.ffn_down_shexp.weight q3_K blk.24.ffn_down_exps.weight q3_K +blk.25.attn_qkv.weight q4_K blk.25.ffn_down_shexp.weight q3_K blk.25.ffn_down_exps.weight q3_K +blk.26.attn_qkv.weight q4_K blk.26.ffn_down_shexp.weight q3_K blk.26.ffn_down_exps.weight q3_K blk.27.ffn_down_shexp.weight q3_K @@ -1019,10 +1180,13 @@ blk.27.attn_output.weight iq3_s blk.27.attn_q.weight iq2_s blk.27.attn_v.weight q4_K blk.27.ffn_down_exps.weight q3_K +blk.28.attn_qkv.weight q4_K blk.28.ffn_down_shexp.weight q3_K blk.28.ffn_down_exps.weight q3_K +blk.29.attn_qkv.weight q4_K blk.29.ffn_down_exps.weight q3_K blk.29.ffn_down_shexp.weight q3_K +blk.30.attn_qkv.weight q4_K blk.30.ffn_down_shexp.weight q3_K blk.30.ffn_down_exps.weight q3_K blk.31.ffn_down_shexp.weight q3_K @@ -1031,10 +1195,13 @@ blk.31.attn_output.weight iq3_s blk.31.attn_q.weight iq2_s blk.31.attn_v.weight q4_K blk.31.ffn_down_exps.weight q3_K +blk.32.attn_qkv.weight q4_K blk.32.ffn_down_shexp.weight q3_K blk.32.ffn_down_exps.weight q3_K +blk.33.attn_qkv.weight q4_K blk.33.ffn_down_shexp.weight q3_K blk.33.ffn_down_exps.weight q3_K +blk.34.attn_qkv.weight q4_K blk.34.ffn_down_exps.weight q3_K blk.34.ffn_down_shexp.weight q3_K blk.35.ffn_down_shexp.weight q3_K @@ -1043,10 +1210,13 @@ blk.35.attn_output.weight iq3_s blk.35.attn_q.weight iq2_s blk.35.attn_v.weight q4_K blk.35.ffn_down_exps.weight q3_K +blk.36.attn_qkv.weight q4_K blk.36.ffn_down_shexp.weight q3_K blk.36.ffn_down_exps.weight q3_K +blk.37.attn_qkv.weight q4_K blk.37.ffn_down_shexp.weight q3_K blk.37.ffn_down_exps.weight q3_K +blk.38.attn_qkv.weight q4_K blk.38.ffn_down_shexp.weight q3_K blk.38.ffn_down_exps.weight q3_K blk.39.ffn_down_shexp.weight q3_K @@ -1055,10 +1225,13 @@ blk.39.attn_output.weight iq3_s blk.39.attn_q.weight iq2_s blk.39.attn_v.weight q4_K blk.39.ffn_down_exps.weight q3_K +blk.40.attn_qkv.weight q4_K blk.40.ffn_down_exps.weight q3_K blk.40.ffn_down_shexp.weight q3_K +blk.41.attn_qkv.weight q4_K blk.41.ffn_down_shexp.weight q3_K blk.41.ffn_down_exps.weight q3_K +blk.42.attn_qkv.weight q4_K blk.42.ffn_down_shexp.weight q3_K blk.42.ffn_down_exps.weight q3_K blk.43.ffn_down_shexp.weight q3_K @@ -1067,10 +1240,13 @@ blk.43.attn_output.weight iq3_s blk.43.attn_q.weight iq2_s blk.43.attn_v.weight q4_K blk.43.ffn_down_exps.weight q3_K +blk.44.attn_qkv.weight q4_K blk.44.ffn_down_shexp.weight q3_K blk.44.ffn_down_exps.weight q3_K +blk.45.attn_qkv.weight q4_K blk.45.ffn_down_shexp.weight q3_K blk.45.ffn_down_exps.weight q3_K +blk.46.attn_qkv.weight q4_K blk.46.ffn_down_exps.weight q3_K blk.46.ffn_down_shexp.weight q3_K blk.47.ffn_down_shexp.weight q3_K @@ -1083,77 +1259,185 @@ blk.47.ffn_down_exps.weight q3_K [IQ1_S] iq1_s token_embd.weight q2_K +blk.0.attn_qkv.weight q4_K blk.0.ffn_down_exps.weight q2_K blk.0.ffn_down_shexp.weight q2_K +blk.1.attn_qkv.weight q4_K blk.1.ffn_down_shexp.weight q2_K blk.1.ffn_down_exps.weight q2_K +blk.2.attn_qkv.weight q4_K blk.2.ffn_down_shexp.weight q2_K blk.2.ffn_down_exps.weight q2_K blk.3.attn_output.weight iq2_xxs blk.3.attn_v.weight q4_K +blk.4.attn_qkv.weight q4_K +blk.5.attn_qkv.weight q4_K +blk.6.attn_qkv.weight q4_K blk.7.attn_output.weight iq2_xxs blk.7.attn_v.weight q4_K +blk.8.attn_qkv.weight q4_K +blk.9.attn_qkv.weight q4_K +blk.10.attn_qkv.weight q4_K blk.11.attn_output.weight iq2_xxs blk.11.attn_v.weight q4_K +blk.12.attn_qkv.weight q4_K +blk.13.attn_qkv.weight q4_K +blk.14.attn_qkv.weight q4_K blk.15.attn_output.weight iq2_xxs blk.15.attn_v.weight q4_K +blk.16.attn_qkv.weight q4_K +blk.17.attn_qkv.weight q4_K +blk.18.attn_qkv.weight q4_K blk.19.attn_output.weight iq2_xxs blk.19.attn_v.weight q4_K +blk.20.attn_qkv.weight q4_K +blk.21.attn_qkv.weight q4_K +blk.22.attn_qkv.weight q4_K blk.23.attn_output.weight iq2_xxs blk.23.attn_v.weight q4_K +blk.24.attn_qkv.weight q4_K +blk.25.attn_qkv.weight q4_K +blk.26.attn_qkv.weight q4_K blk.27.attn_output.weight iq2_xxs blk.27.attn_v.weight q4_K +blk.28.attn_qkv.weight q4_K +blk.29.attn_qkv.weight q4_K +blk.30.attn_qkv.weight q4_K blk.31.attn_output.weight iq2_xxs blk.31.attn_v.weight q4_K +blk.32.attn_qkv.weight q4_K +blk.33.attn_qkv.weight q4_K +blk.34.attn_qkv.weight q4_K blk.35.attn_output.weight iq2_xxs blk.35.attn_v.weight q4_K +blk.36.attn_qkv.weight q4_K +blk.37.attn_qkv.weight q4_K +blk.38.attn_qkv.weight q4_K blk.39.attn_output.weight iq2_xxs blk.39.attn_v.weight q4_K +blk.40.attn_qkv.weight q4_K +blk.41.attn_qkv.weight q4_K +blk.42.attn_qkv.weight q4_K blk.43.attn_output.weight iq2_xxs blk.43.attn_v.weight q4_K +blk.44.attn_qkv.weight q4_K +blk.45.attn_qkv.weight q4_K +blk.46.attn_qkv.weight q4_K blk.47.attn_output.weight iq2_xxs blk.47.attn_v.weight q4_K output.weight q5_K [IQ4_NL] iq4_nl +blk.0.attn_qkv.weight q5_K blk.0.ffn_down_exps.weight q5_K blk.0.ffn_down_shexp.weight q5_K +blk.1.attn_qkv.weight q5_K blk.1.ffn_down_shexp.weight q5_K blk.1.ffn_down_exps.weight q5_K +blk.2.attn_qkv.weight q5_K blk.2.ffn_down_shexp.weight q5_K blk.2.ffn_down_exps.weight q5_K blk.3.ffn_down_shexp.weight q5_K blk.3.attn_v.weight q5_K blk.3.ffn_down_exps.weight q5_K +blk.4.attn_qkv.weight q5_K blk.4.ffn_down_shexp.weight q5_K blk.4.ffn_down_exps.weight q5_K +blk.5.attn_qkv.weight q5_K blk.5.ffn_down_shexp.weight q5_K blk.5.ffn_down_exps.weight q5_K +blk.6.attn_qkv.weight q5_K blk.7.attn_v.weight q5_K +blk.8.attn_qkv.weight q5_K +blk.9.attn_qkv.weight q5_K +blk.10.attn_qkv.weight q5_K blk.11.attn_v.weight q5_K +blk.12.attn_qkv.weight q5_K +blk.13.attn_qkv.weight q5_K +blk.14.attn_qkv.weight q5_K blk.15.attn_v.weight q5_K +blk.16.attn_qkv.weight q5_K +blk.17.attn_qkv.weight q5_K +blk.18.attn_qkv.weight q5_K blk.19.attn_v.weight q5_K +blk.20.attn_qkv.weight q5_K +blk.21.attn_qkv.weight q5_K +blk.22.attn_qkv.weight q5_K blk.23.attn_v.weight q5_K +blk.24.attn_qkv.weight q5_K +blk.25.attn_qkv.weight q5_K +blk.26.attn_qkv.weight q5_K blk.27.attn_v.weight q5_K +blk.28.attn_qkv.weight q5_K +blk.29.attn_qkv.weight q5_K +blk.30.attn_qkv.weight q5_K blk.31.attn_v.weight q5_K +blk.32.attn_qkv.weight q5_K +blk.33.attn_qkv.weight q5_K +blk.34.attn_qkv.weight q5_K blk.35.attn_v.weight q5_K +blk.36.attn_qkv.weight q5_K +blk.37.attn_qkv.weight q5_K +blk.38.attn_qkv.weight q5_K blk.39.attn_v.weight q5_K +blk.40.attn_qkv.weight q5_K +blk.41.attn_qkv.weight q5_K +blk.42.attn_qkv.weight q5_K blk.43.attn_v.weight q5_K +blk.44.attn_qkv.weight q5_K +blk.45.attn_qkv.weight q5_K +blk.46.attn_qkv.weight q5_K blk.47.attn_v.weight q5_K output.weight q6_K [IQ3_S] iq3_s +blk.0.attn_qkv.weight q4_K +blk.1.attn_qkv.weight q4_K +blk.2.attn_qkv.weight q4_K blk.3.attn_v.weight q4_K +blk.4.attn_qkv.weight q4_K +blk.5.attn_qkv.weight q4_K +blk.6.attn_qkv.weight q4_K blk.7.attn_v.weight q4_K +blk.8.attn_qkv.weight q4_K +blk.9.attn_qkv.weight q4_K +blk.10.attn_qkv.weight q4_K blk.11.attn_v.weight q4_K +blk.12.attn_qkv.weight q4_K +blk.13.attn_qkv.weight q4_K +blk.14.attn_qkv.weight q4_K blk.15.attn_v.weight q4_K +blk.16.attn_qkv.weight q4_K +blk.17.attn_qkv.weight q4_K +blk.18.attn_qkv.weight q4_K blk.19.attn_v.weight q4_K +blk.20.attn_qkv.weight q4_K +blk.21.attn_qkv.weight q4_K +blk.22.attn_qkv.weight q4_K blk.23.attn_v.weight q4_K +blk.24.attn_qkv.weight q4_K +blk.25.attn_qkv.weight q4_K +blk.26.attn_qkv.weight q4_K blk.27.attn_v.weight q4_K +blk.28.attn_qkv.weight q4_K +blk.29.attn_qkv.weight q4_K +blk.30.attn_qkv.weight q4_K blk.31.attn_v.weight q4_K +blk.32.attn_qkv.weight q4_K +blk.33.attn_qkv.weight q4_K +blk.34.attn_qkv.weight q4_K blk.35.attn_v.weight q4_K +blk.36.attn_qkv.weight q4_K +blk.37.attn_qkv.weight q4_K +blk.38.attn_qkv.weight q4_K blk.39.attn_v.weight q4_K +blk.40.attn_qkv.weight q4_K +blk.41.attn_qkv.weight q4_K +blk.42.attn_qkv.weight q4_K blk.43.attn_v.weight q4_K +blk.44.attn_qkv.weight q4_K +blk.45.attn_qkv.weight q4_K +blk.46.attn_qkv.weight q4_K blk.47.attn_v.weight q4_K output.weight q6_K @@ -1234,129 +1518,273 @@ output.weight q6_K [IQ2_S] iq2_xs token_embd.weight iq3_s +blk.0.attn_qkv.weight q4_K blk.0.ffn_down_exps.weight iq3_s blk.0.ffn_down_shexp.weight iq3_s +blk.1.attn_qkv.weight q4_K blk.1.ffn_down_shexp.weight iq3_s blk.1.ffn_down_exps.weight iq3_s +blk.2.attn_qkv.weight q4_K blk.2.ffn_down_shexp.weight iq3_s blk.2.ffn_down_exps.weight iq3_s blk.3.attn_output.weight iq3_s blk.3.attn_v.weight q4_K +blk.4.attn_qkv.weight q4_K +blk.5.attn_qkv.weight q4_K +blk.6.attn_qkv.weight q4_K blk.7.attn_output.weight iq3_s blk.7.attn_v.weight q4_K +blk.8.attn_qkv.weight q4_K +blk.9.attn_qkv.weight q4_K +blk.10.attn_qkv.weight q4_K blk.11.attn_output.weight iq3_s blk.11.attn_v.weight q4_K +blk.12.attn_qkv.weight q4_K +blk.13.attn_qkv.weight q4_K +blk.14.attn_qkv.weight q4_K blk.15.attn_output.weight iq3_s blk.15.attn_v.weight q4_K +blk.16.attn_qkv.weight q4_K +blk.17.attn_qkv.weight q4_K +blk.18.attn_qkv.weight q4_K blk.19.attn_output.weight iq3_s blk.19.attn_v.weight q4_K +blk.20.attn_qkv.weight q4_K +blk.21.attn_qkv.weight q4_K +blk.22.attn_qkv.weight q4_K blk.23.attn_output.weight iq3_s blk.23.attn_v.weight q4_K +blk.24.attn_qkv.weight q4_K +blk.25.attn_qkv.weight q4_K +blk.26.attn_qkv.weight q4_K blk.27.attn_output.weight iq3_s blk.27.attn_v.weight q4_K +blk.28.attn_qkv.weight q4_K +blk.29.attn_qkv.weight q4_K +blk.30.attn_qkv.weight q4_K blk.31.attn_output.weight iq3_s blk.31.attn_v.weight q4_K +blk.32.attn_qkv.weight q4_K +blk.33.attn_qkv.weight q4_K +blk.34.attn_qkv.weight q4_K blk.35.attn_output.weight iq3_s blk.35.attn_v.weight q4_K +blk.36.attn_qkv.weight q4_K +blk.37.attn_qkv.weight q4_K +blk.38.attn_qkv.weight q4_K blk.39.attn_output.weight iq3_s blk.39.attn_v.weight q4_K +blk.40.attn_qkv.weight q4_K +blk.41.attn_qkv.weight q4_K +blk.42.attn_qkv.weight q4_K blk.43.attn_output.weight iq3_s blk.43.attn_v.weight q4_K +blk.44.attn_qkv.weight q4_K +blk.45.attn_qkv.weight q4_K +blk.46.attn_qkv.weight q4_K blk.47.attn_output.weight iq3_s blk.47.attn_v.weight q4_K output.weight q5_K [IQ2_M] iq2_s token_embd.weight iq3_s +blk.0.attn_qkv.weight q4_K blk.0.ffn_down_exps.weight iq3_s blk.0.ffn_down_shexp.weight iq3_s +blk.1.attn_qkv.weight q4_K blk.1.ffn_down_shexp.weight iq3_s blk.1.ffn_down_exps.weight iq3_s +blk.2.attn_qkv.weight q4_K blk.2.ffn_down_shexp.weight iq3_s blk.2.ffn_down_exps.weight iq3_s blk.3.attn_output.weight iq3_s blk.3.attn_v.weight q4_K +blk.4.attn_qkv.weight q4_K +blk.5.attn_qkv.weight q4_K +blk.6.attn_qkv.weight q4_K blk.7.attn_output.weight iq3_s blk.7.attn_v.weight q4_K +blk.8.attn_qkv.weight q4_K +blk.9.attn_qkv.weight q4_K +blk.10.attn_qkv.weight q4_K blk.11.attn_output.weight iq3_s blk.11.attn_v.weight q4_K +blk.12.attn_qkv.weight q4_K +blk.13.attn_qkv.weight q4_K +blk.14.attn_qkv.weight q4_K blk.15.attn_output.weight iq3_s blk.15.attn_v.weight q4_K +blk.16.attn_qkv.weight q4_K +blk.17.attn_qkv.weight q4_K +blk.18.attn_qkv.weight q4_K blk.19.attn_output.weight iq3_s blk.19.attn_v.weight q4_K +blk.20.attn_qkv.weight q4_K +blk.21.attn_qkv.weight q4_K +blk.22.attn_qkv.weight q4_K blk.23.attn_output.weight iq3_s blk.23.attn_v.weight q4_K +blk.24.attn_qkv.weight q4_K +blk.25.attn_qkv.weight q4_K +blk.26.attn_qkv.weight q4_K blk.27.attn_output.weight iq3_s blk.27.attn_v.weight q4_K +blk.28.attn_qkv.weight q4_K +blk.29.attn_qkv.weight q4_K +blk.30.attn_qkv.weight q4_K blk.31.attn_output.weight iq3_s blk.31.attn_v.weight q4_K +blk.32.attn_qkv.weight q4_K +blk.33.attn_qkv.weight q4_K +blk.34.attn_qkv.weight q4_K blk.35.attn_output.weight iq3_s blk.35.attn_v.weight q4_K +blk.36.attn_qkv.weight q4_K +blk.37.attn_qkv.weight q4_K +blk.38.attn_qkv.weight q4_K blk.39.attn_output.weight iq3_s blk.39.attn_v.weight q4_K +blk.40.attn_qkv.weight q4_K +blk.41.attn_qkv.weight q4_K +blk.42.attn_qkv.weight q4_K blk.43.attn_output.weight iq3_s blk.43.attn_v.weight q4_K +blk.44.attn_qkv.weight q4_K +blk.45.attn_qkv.weight q4_K +blk.46.attn_qkv.weight q4_K blk.47.attn_output.weight iq3_s blk.47.attn_v.weight q4_K output.weight q5_K [IQ4_XS] iq4_xs +blk.0.attn_qkv.weight q5_K blk.0.ffn_down_exps.weight q5_K blk.0.ffn_down_shexp.weight q5_K +blk.1.attn_qkv.weight q5_K blk.1.ffn_down_shexp.weight q5_K blk.1.ffn_down_exps.weight q5_K +blk.2.attn_qkv.weight q5_K blk.2.ffn_down_shexp.weight q5_K blk.2.ffn_down_exps.weight q5_K blk.3.ffn_down_shexp.weight q5_K blk.3.attn_v.weight q5_K blk.3.ffn_down_exps.weight q5_K +blk.4.attn_qkv.weight q5_K blk.4.ffn_down_shexp.weight q5_K blk.4.ffn_down_exps.weight q5_K +blk.5.attn_qkv.weight q5_K blk.5.ffn_down_shexp.weight q5_K blk.5.ffn_down_exps.weight q5_K +blk.6.attn_qkv.weight q5_K blk.7.attn_v.weight q5_K +blk.8.attn_qkv.weight q5_K +blk.9.attn_qkv.weight q5_K +blk.10.attn_qkv.weight q5_K blk.11.attn_v.weight q5_K +blk.12.attn_qkv.weight q5_K +blk.13.attn_qkv.weight q5_K +blk.14.attn_qkv.weight q5_K blk.15.attn_v.weight q5_K +blk.16.attn_qkv.weight q5_K +blk.17.attn_qkv.weight q5_K +blk.18.attn_qkv.weight q5_K blk.19.attn_v.weight q5_K +blk.20.attn_qkv.weight q5_K +blk.21.attn_qkv.weight q5_K +blk.22.attn_qkv.weight q5_K blk.23.attn_v.weight q5_K +blk.24.attn_qkv.weight q5_K +blk.25.attn_qkv.weight q5_K +blk.26.attn_qkv.weight q5_K blk.27.attn_v.weight q5_K +blk.28.attn_qkv.weight q5_K +blk.29.attn_qkv.weight q5_K +blk.30.attn_qkv.weight q5_K blk.31.attn_v.weight q5_K +blk.32.attn_qkv.weight q5_K +blk.33.attn_qkv.weight q5_K +blk.34.attn_qkv.weight q5_K blk.35.attn_v.weight q5_K +blk.36.attn_qkv.weight q5_K +blk.37.attn_qkv.weight q5_K +blk.38.attn_qkv.weight q5_K blk.39.attn_v.weight q5_K +blk.40.attn_qkv.weight q5_K +blk.41.attn_qkv.weight q5_K +blk.42.attn_qkv.weight q5_K blk.43.attn_v.weight q5_K +blk.44.attn_qkv.weight q5_K +blk.45.attn_qkv.weight q5_K +blk.46.attn_qkv.weight q5_K blk.47.attn_v.weight q5_K output.weight q6_K [IQ1_M] iq1_m token_embd.weight q2_K +blk.0.attn_qkv.weight q4_K blk.0.ffn_down_exps.weight q2_K blk.0.ffn_down_shexp.weight q2_K +blk.1.attn_qkv.weight q4_K blk.1.ffn_down_shexp.weight q2_K blk.1.ffn_down_exps.weight q2_K +blk.2.attn_qkv.weight q4_K blk.2.ffn_down_shexp.weight q2_K blk.2.ffn_down_exps.weight q2_K blk.3.attn_output.weight iq2_xxs blk.3.attn_v.weight q4_K +blk.4.attn_qkv.weight q4_K +blk.5.attn_qkv.weight q4_K +blk.6.attn_qkv.weight q4_K blk.7.attn_output.weight iq2_xxs blk.7.attn_v.weight q4_K +blk.8.attn_qkv.weight q4_K +blk.9.attn_qkv.weight q4_K +blk.10.attn_qkv.weight q4_K blk.11.attn_output.weight iq2_xxs blk.11.attn_v.weight q4_K +blk.12.attn_qkv.weight q4_K +blk.13.attn_qkv.weight q4_K +blk.14.attn_qkv.weight q4_K blk.15.attn_output.weight iq2_xxs blk.15.attn_v.weight q4_K +blk.16.attn_qkv.weight q4_K +blk.17.attn_qkv.weight q4_K +blk.18.attn_qkv.weight q4_K blk.19.attn_output.weight iq2_xxs blk.19.attn_v.weight q4_K +blk.20.attn_qkv.weight q4_K +blk.21.attn_qkv.weight q4_K +blk.22.attn_qkv.weight q4_K blk.23.attn_output.weight iq2_xxs blk.23.attn_v.weight q4_K +blk.24.attn_qkv.weight q4_K +blk.25.attn_qkv.weight q4_K +blk.26.attn_qkv.weight q4_K blk.27.attn_output.weight iq2_xxs blk.27.attn_v.weight q4_K +blk.28.attn_qkv.weight q4_K +blk.29.attn_qkv.weight q4_K +blk.30.attn_qkv.weight q4_K blk.31.attn_output.weight iq2_xxs blk.31.attn_v.weight q4_K +blk.32.attn_qkv.weight q4_K +blk.33.attn_qkv.weight q4_K +blk.34.attn_qkv.weight q4_K blk.35.attn_output.weight iq2_xxs blk.35.attn_v.weight q4_K +blk.36.attn_qkv.weight q4_K +blk.37.attn_qkv.weight q4_K +blk.38.attn_qkv.weight q4_K blk.39.attn_output.weight iq2_xxs blk.39.attn_v.weight q4_K +blk.40.attn_qkv.weight q4_K +blk.41.attn_qkv.weight q4_K +blk.42.attn_qkv.weight q4_K blk.43.attn_output.weight iq2_xxs blk.43.attn_v.weight q4_K +blk.44.attn_qkv.weight q4_K +blk.45.attn_qkv.weight q4_K +blk.46.attn_qkv.weight q4_K blk.47.attn_output.weight iq2_xxs blk.47.attn_v.weight q4_K output.weight q5_K diff --git a/tests/snapshots/qwen3.5-27b.schema b/tests/snapshots/qwen3.5-27b.schema index f1f655a847..b5e07f01fc 100644 --- a/tests/snapshots/qwen3.5-27b.schema +++ b/tests/snapshots/qwen3.5-27b.schema @@ -23,98 +23,146 @@ output.weight q6_K [Q2_K] q2_K output.weight q6_K +blk.0.attn_qkv.weight q4_K blk.0.ffn_down.weight q3_K +blk.1.attn_qkv.weight q4_K blk.1.ffn_down.weight q3_K +blk.2.attn_qkv.weight q4_K blk.2.ffn_down.weight q3_K blk.3.attn_output.weight q3_K blk.3.attn_v.weight q4_K blk.3.ffn_down.weight q3_K +blk.4.attn_qkv.weight q4_K blk.4.ffn_down.weight q3_K +blk.5.attn_qkv.weight q4_K blk.5.ffn_down.weight q3_K +blk.6.attn_qkv.weight q4_K blk.6.ffn_down.weight q3_K blk.7.attn_output.weight q3_K blk.7.attn_v.weight q4_K blk.7.ffn_down.weight q3_K +blk.8.attn_qkv.weight q4_K blk.8.ffn_down.weight q3_K +blk.9.attn_qkv.weight q4_K blk.9.ffn_down.weight q3_K +blk.10.attn_qkv.weight q4_K blk.10.ffn_down.weight q3_K blk.11.attn_output.weight q3_K blk.11.attn_v.weight q4_K blk.11.ffn_down.weight q3_K +blk.12.attn_qkv.weight q4_K blk.12.ffn_down.weight q3_K +blk.13.attn_qkv.weight q4_K blk.13.ffn_down.weight q3_K +blk.14.attn_qkv.weight q4_K blk.14.ffn_down.weight q3_K blk.15.attn_output.weight q3_K blk.15.attn_v.weight q4_K blk.15.ffn_down.weight q3_K +blk.16.attn_qkv.weight q4_K blk.16.ffn_down.weight q3_K +blk.17.attn_qkv.weight q4_K blk.17.ffn_down.weight q3_K +blk.18.attn_qkv.weight q4_K blk.18.ffn_down.weight q3_K blk.19.attn_output.weight q3_K blk.19.attn_v.weight q4_K blk.19.ffn_down.weight q3_K +blk.20.attn_qkv.weight q4_K blk.20.ffn_down.weight q3_K +blk.21.attn_qkv.weight q4_K blk.21.ffn_down.weight q3_K +blk.22.attn_qkv.weight q4_K blk.22.ffn_down.weight q3_K blk.23.attn_output.weight q3_K blk.23.attn_v.weight q4_K blk.23.ffn_down.weight q3_K +blk.24.attn_qkv.weight q4_K blk.24.ffn_down.weight q3_K +blk.25.attn_qkv.weight q4_K blk.25.ffn_down.weight q3_K +blk.26.attn_qkv.weight q4_K blk.26.ffn_down.weight q3_K blk.27.attn_output.weight q3_K blk.27.attn_v.weight q4_K blk.27.ffn_down.weight q3_K +blk.28.attn_qkv.weight q4_K blk.28.ffn_down.weight q3_K +blk.29.attn_qkv.weight q4_K blk.29.ffn_down.weight q3_K +blk.30.attn_qkv.weight q4_K blk.30.ffn_down.weight q3_K blk.31.attn_output.weight q3_K blk.31.attn_v.weight q4_K blk.31.ffn_down.weight q3_K +blk.32.attn_qkv.weight q4_K blk.32.ffn_down.weight q3_K +blk.33.attn_qkv.weight q4_K blk.33.ffn_down.weight q3_K +blk.34.attn_qkv.weight q4_K blk.34.ffn_down.weight q3_K blk.35.attn_output.weight q3_K blk.35.attn_v.weight q4_K blk.35.ffn_down.weight q3_K +blk.36.attn_qkv.weight q4_K blk.36.ffn_down.weight q3_K +blk.37.attn_qkv.weight q4_K blk.37.ffn_down.weight q3_K +blk.38.attn_qkv.weight q4_K blk.38.ffn_down.weight q3_K blk.39.attn_output.weight q3_K blk.39.attn_v.weight q4_K blk.39.ffn_down.weight q3_K +blk.40.attn_qkv.weight q4_K blk.40.ffn_down.weight q3_K +blk.41.attn_qkv.weight q4_K blk.41.ffn_down.weight q3_K +blk.42.attn_qkv.weight q4_K blk.42.ffn_down.weight q3_K blk.43.attn_output.weight q3_K blk.43.attn_v.weight q4_K blk.43.ffn_down.weight q3_K +blk.44.attn_qkv.weight q4_K blk.44.ffn_down.weight q3_K +blk.45.attn_qkv.weight q4_K blk.45.ffn_down.weight q3_K +blk.46.attn_qkv.weight q4_K blk.46.ffn_down.weight q3_K blk.47.attn_output.weight q3_K blk.47.attn_v.weight q4_K blk.47.ffn_down.weight q3_K +blk.48.attn_qkv.weight q4_K blk.48.ffn_down.weight q3_K +blk.49.attn_qkv.weight q4_K blk.49.ffn_down.weight q3_K +blk.50.attn_qkv.weight q4_K blk.50.ffn_down.weight q3_K blk.51.attn_output.weight q3_K blk.51.attn_v.weight q4_K blk.51.ffn_down.weight q3_K +blk.52.attn_qkv.weight q4_K blk.52.ffn_down.weight q3_K +blk.53.attn_qkv.weight q4_K blk.53.ffn_down.weight q3_K +blk.54.attn_qkv.weight q4_K blk.54.ffn_down.weight q3_K blk.55.attn_output.weight q3_K blk.55.attn_v.weight q4_K blk.55.ffn_down.weight q3_K +blk.56.attn_qkv.weight q4_K blk.56.ffn_down.weight q3_K +blk.57.attn_qkv.weight q4_K blk.57.ffn_down.weight q3_K +blk.58.attn_qkv.weight q4_K blk.58.ffn_down.weight q3_K blk.59.attn_output.weight q3_K blk.59.attn_v.weight q4_K blk.59.ffn_down.weight q3_K +blk.60.attn_qkv.weight q4_K blk.60.ffn_down.weight q3_K +blk.61.attn_qkv.weight q4_K blk.61.ffn_down.weight q3_K +blk.62.attn_qkv.weight q4_K blk.62.ffn_down.weight q3_K blk.63.attn_output.weight q3_K blk.63.attn_v.weight q4_K @@ -125,14 +173,14 @@ output.weight q6_K [Q3_K_M] q3_K output.weight q6_K -blk.0.attn_qkv.weight q4_K +blk.0.attn_qkv.weight q5_K blk.0.ffn_down.weight q5_K -blk.1.attn_qkv.weight q4_K +blk.1.attn_qkv.weight q5_K blk.1.ffn_down.weight q5_K blk.2.attn_qkv.weight q4_K blk.2.ffn_down.weight q5_K blk.3.attn_output.weight q4_K -blk.3.attn_v.weight q5_K +blk.3.attn_v.weight q4_K blk.3.ffn_down.weight q5_K blk.4.attn_qkv.weight q4_K blk.4.ffn_down.weight q4_K @@ -141,7 +189,7 @@ blk.5.ffn_down.weight q4_K blk.6.attn_qkv.weight q4_K blk.6.ffn_down.weight q4_K blk.7.attn_output.weight q4_K -blk.7.attn_v.weight q5_K +blk.7.attn_v.weight q4_K blk.7.ffn_down.weight q4_K blk.8.attn_qkv.weight q4_K blk.8.ffn_down.weight q4_K @@ -272,146 +320,146 @@ blk.63.ffn_down.weight q4_K [Q3_K_L] q3_K output.weight q6_K -blk.0.attn_qkv.weight q4_K +blk.0.attn_qkv.weight q5_K blk.0.ffn_down.weight q5_K -blk.1.attn_qkv.weight q4_K +blk.1.attn_qkv.weight q5_K blk.1.ffn_down.weight q5_K -blk.2.attn_qkv.weight q4_K +blk.2.attn_qkv.weight q5_K blk.2.ffn_down.weight q5_K blk.3.attn_output.weight q5_K blk.3.attn_v.weight q5_K blk.3.ffn_down.weight q5_K -blk.4.attn_qkv.weight q4_K +blk.4.attn_qkv.weight q5_K blk.4.ffn_down.weight q5_K -blk.5.attn_qkv.weight q4_K +blk.5.attn_qkv.weight q5_K blk.5.ffn_down.weight q5_K -blk.6.attn_qkv.weight q4_K +blk.6.attn_qkv.weight q5_K blk.6.ffn_down.weight q5_K blk.7.attn_output.weight q5_K blk.7.attn_v.weight q5_K blk.7.ffn_down.weight q5_K -blk.8.attn_qkv.weight q4_K +blk.8.attn_qkv.weight q5_K blk.8.ffn_down.weight q5_K -blk.9.attn_qkv.weight q4_K +blk.9.attn_qkv.weight q5_K blk.9.ffn_down.weight q5_K -blk.10.attn_qkv.weight q4_K +blk.10.attn_qkv.weight q5_K blk.10.ffn_down.weight q5_K blk.11.attn_output.weight q5_K blk.11.attn_v.weight q5_K blk.11.ffn_down.weight q5_K -blk.12.attn_qkv.weight q4_K +blk.12.attn_qkv.weight q5_K blk.12.ffn_down.weight q5_K -blk.13.attn_qkv.weight q4_K +blk.13.attn_qkv.weight q5_K blk.13.ffn_down.weight q5_K -blk.14.attn_qkv.weight q4_K +blk.14.attn_qkv.weight q5_K blk.14.ffn_down.weight q5_K blk.15.attn_output.weight q5_K blk.15.attn_v.weight q5_K blk.15.ffn_down.weight q5_K -blk.16.attn_qkv.weight q4_K +blk.16.attn_qkv.weight q5_K blk.16.ffn_down.weight q5_K -blk.17.attn_qkv.weight q4_K +blk.17.attn_qkv.weight q5_K blk.17.ffn_down.weight q5_K -blk.18.attn_qkv.weight q4_K +blk.18.attn_qkv.weight q5_K blk.18.ffn_down.weight q5_K blk.19.attn_output.weight q5_K blk.19.attn_v.weight q5_K blk.19.ffn_down.weight q5_K -blk.20.attn_qkv.weight q4_K +blk.20.attn_qkv.weight q5_K blk.20.ffn_down.weight q5_K -blk.21.attn_qkv.weight q4_K +blk.21.attn_qkv.weight q5_K blk.21.ffn_down.weight q5_K -blk.22.attn_qkv.weight q4_K +blk.22.attn_qkv.weight q5_K blk.22.ffn_down.weight q5_K blk.23.attn_output.weight q5_K blk.23.attn_v.weight q5_K blk.23.ffn_down.weight q5_K -blk.24.attn_qkv.weight q4_K +blk.24.attn_qkv.weight q5_K blk.24.ffn_down.weight q5_K -blk.25.attn_qkv.weight q4_K +blk.25.attn_qkv.weight q5_K blk.25.ffn_down.weight q5_K -blk.26.attn_qkv.weight q4_K +blk.26.attn_qkv.weight q5_K blk.26.ffn_down.weight q5_K blk.27.attn_output.weight q5_K blk.27.attn_v.weight q5_K blk.27.ffn_down.weight q5_K -blk.28.attn_qkv.weight q4_K +blk.28.attn_qkv.weight q5_K blk.28.ffn_down.weight q5_K -blk.29.attn_qkv.weight q4_K +blk.29.attn_qkv.weight q5_K blk.29.ffn_down.weight q5_K -blk.30.attn_qkv.weight q4_K +blk.30.attn_qkv.weight q5_K blk.30.ffn_down.weight q5_K blk.31.attn_output.weight q5_K blk.31.attn_v.weight q5_K blk.31.ffn_down.weight q5_K -blk.32.attn_qkv.weight q4_K +blk.32.attn_qkv.weight q5_K blk.32.ffn_down.weight q5_K -blk.33.attn_qkv.weight q4_K +blk.33.attn_qkv.weight q5_K blk.33.ffn_down.weight q5_K -blk.34.attn_qkv.weight q4_K +blk.34.attn_qkv.weight q5_K blk.34.ffn_down.weight q5_K blk.35.attn_output.weight q5_K blk.35.attn_v.weight q5_K blk.35.ffn_down.weight q5_K -blk.36.attn_qkv.weight q4_K +blk.36.attn_qkv.weight q5_K blk.36.ffn_down.weight q5_K -blk.37.attn_qkv.weight q4_K +blk.37.attn_qkv.weight q5_K blk.37.ffn_down.weight q5_K -blk.38.attn_qkv.weight q4_K +blk.38.attn_qkv.weight q5_K blk.38.ffn_down.weight q5_K blk.39.attn_output.weight q5_K blk.39.attn_v.weight q5_K blk.39.ffn_down.weight q5_K -blk.40.attn_qkv.weight q4_K +blk.40.attn_qkv.weight q5_K blk.40.ffn_down.weight q5_K -blk.41.attn_qkv.weight q4_K +blk.41.attn_qkv.weight q5_K blk.41.ffn_down.weight q5_K -blk.42.attn_qkv.weight q4_K +blk.42.attn_qkv.weight q5_K blk.42.ffn_down.weight q5_K blk.43.attn_output.weight q5_K blk.43.attn_v.weight q5_K blk.43.ffn_down.weight q5_K -blk.44.attn_qkv.weight q4_K +blk.44.attn_qkv.weight q5_K blk.44.ffn_down.weight q5_K -blk.45.attn_qkv.weight q4_K +blk.45.attn_qkv.weight q5_K blk.45.ffn_down.weight q5_K -blk.46.attn_qkv.weight q4_K +blk.46.attn_qkv.weight q5_K blk.46.ffn_down.weight q5_K blk.47.attn_output.weight q5_K blk.47.attn_v.weight q5_K blk.47.ffn_down.weight q5_K -blk.48.attn_qkv.weight q4_K +blk.48.attn_qkv.weight q5_K blk.48.ffn_down.weight q5_K -blk.49.attn_qkv.weight q4_K +blk.49.attn_qkv.weight q5_K blk.49.ffn_down.weight q5_K -blk.50.attn_qkv.weight q4_K +blk.50.attn_qkv.weight q5_K blk.50.ffn_down.weight q5_K blk.51.attn_output.weight q5_K blk.51.attn_v.weight q5_K blk.51.ffn_down.weight q5_K -blk.52.attn_qkv.weight q4_K +blk.52.attn_qkv.weight q5_K blk.52.ffn_down.weight q5_K -blk.53.attn_qkv.weight q4_K +blk.53.attn_qkv.weight q5_K blk.53.ffn_down.weight q5_K -blk.54.attn_qkv.weight q4_K +blk.54.attn_qkv.weight q5_K blk.54.ffn_down.weight q5_K blk.55.attn_output.weight q5_K blk.55.attn_v.weight q5_K blk.55.ffn_down.weight q5_K -blk.56.attn_qkv.weight q4_K +blk.56.attn_qkv.weight q5_K blk.56.ffn_down.weight q5_K -blk.57.attn_qkv.weight q4_K +blk.57.attn_qkv.weight q5_K blk.57.ffn_down.weight q5_K -blk.58.attn_qkv.weight q4_K +blk.58.attn_qkv.weight q5_K blk.58.ffn_down.weight q5_K blk.59.attn_output.weight q5_K blk.59.attn_v.weight q5_K blk.59.ffn_down.weight q5_K -blk.60.attn_qkv.weight q4_K +blk.60.attn_qkv.weight q5_K blk.60.ffn_down.weight q5_K -blk.61.attn_qkv.weight q4_K +blk.61.attn_qkv.weight q5_K blk.61.ffn_down.weight q5_K -blk.62.attn_qkv.weight q4_K +blk.62.attn_qkv.weight q5_K blk.62.ffn_down.weight q5_K blk.63.attn_output.weight q5_K blk.63.attn_v.weight q5_K @@ -419,110 +467,84 @@ blk.63.ffn_down.weight q5_K [Q4_K_S] q4_K output.weight q6_K +blk.0.attn_qkv.weight q5_K blk.0.ffn_down.weight q5_K +blk.1.attn_qkv.weight q5_K blk.1.ffn_down.weight q5_K +blk.2.attn_qkv.weight q5_K blk.2.ffn_down.weight q5_K blk.3.attn_v.weight q5_K blk.3.ffn_down.weight q5_K blk.4.ffn_down.weight q5_K blk.5.ffn_down.weight q5_K blk.6.ffn_down.weight q5_K -blk.7.attn_v.weight q5_K blk.7.ffn_down.weight q5_K -blk.11.attn_v.weight q5_K -blk.15.attn_v.weight q5_K [Q4_K_M] q4_K output.weight q6_K -blk.0.attn_qkv.weight q5_K +blk.0.attn_qkv.weight q6_K blk.0.ffn_down.weight q6_K -blk.1.attn_qkv.weight q5_K +blk.1.attn_qkv.weight q6_K blk.1.ffn_down.weight q6_K -blk.2.attn_qkv.weight q5_K +blk.2.attn_qkv.weight q6_K blk.2.ffn_down.weight q6_K blk.3.attn_v.weight q6_K blk.3.ffn_down.weight q6_K -blk.4.attn_qkv.weight q5_K +blk.4.attn_qkv.weight q6_K blk.4.ffn_down.weight q6_K -blk.5.attn_qkv.weight q5_K +blk.5.attn_qkv.weight q6_K blk.5.ffn_down.weight q6_K -blk.6.attn_qkv.weight q5_K +blk.6.attn_qkv.weight q6_K blk.6.ffn_down.weight q6_K blk.7.attn_v.weight q6_K blk.7.ffn_down.weight q6_K -blk.8.attn_qkv.weight q5_K -blk.9.attn_qkv.weight q5_K -blk.10.attn_qkv.weight q5_K +blk.10.attn_qkv.weight q6_K blk.10.ffn_down.weight q6_K -blk.11.attn_v.weight q6_K -blk.12.attn_qkv.weight q5_K -blk.13.attn_qkv.weight q5_K +blk.13.attn_qkv.weight q6_K blk.13.ffn_down.weight q6_K -blk.14.attn_qkv.weight q5_K -blk.15.attn_v.weight q6_K -blk.16.attn_qkv.weight q5_K +blk.16.attn_qkv.weight q6_K blk.16.ffn_down.weight q6_K -blk.17.attn_qkv.weight q5_K -blk.18.attn_qkv.weight q5_K blk.19.attn_v.weight q6_K blk.19.ffn_down.weight q6_K -blk.20.attn_qkv.weight q5_K -blk.21.attn_qkv.weight q5_K -blk.22.attn_qkv.weight q5_K +blk.22.attn_qkv.weight q6_K blk.22.ffn_down.weight q6_K -blk.23.attn_v.weight q6_K -blk.24.attn_qkv.weight q5_K -blk.25.attn_qkv.weight q5_K +blk.25.attn_qkv.weight q6_K blk.25.ffn_down.weight q6_K -blk.26.attn_qkv.weight q5_K -blk.27.attn_v.weight q6_K -blk.28.attn_qkv.weight q5_K +blk.28.attn_qkv.weight q6_K blk.28.ffn_down.weight q6_K -blk.29.attn_qkv.weight q5_K -blk.30.attn_qkv.weight q5_K blk.31.attn_v.weight q6_K blk.31.ffn_down.weight q6_K -blk.32.attn_qkv.weight q5_K -blk.33.attn_qkv.weight q5_K -blk.34.attn_qkv.weight q5_K +blk.34.attn_qkv.weight q6_K blk.34.ffn_down.weight q6_K -blk.36.attn_qkv.weight q5_K -blk.37.attn_qkv.weight q5_K +blk.37.attn_qkv.weight q6_K blk.37.ffn_down.weight q6_K -blk.38.attn_qkv.weight q5_K -blk.40.attn_qkv.weight q5_K +blk.40.attn_qkv.weight q6_K blk.40.ffn_down.weight q6_K -blk.41.attn_qkv.weight q5_K -blk.42.attn_qkv.weight q5_K blk.43.attn_v.weight q6_K blk.43.ffn_down.weight q6_K -blk.44.attn_qkv.weight q5_K -blk.45.attn_qkv.weight q5_K -blk.46.attn_qkv.weight q5_K +blk.46.attn_qkv.weight q6_K blk.46.ffn_down.weight q6_K -blk.48.attn_qkv.weight q5_K -blk.49.attn_qkv.weight q5_K +blk.49.attn_qkv.weight q6_K blk.49.ffn_down.weight q6_K -blk.50.attn_qkv.weight q5_K -blk.52.attn_qkv.weight q5_K +blk.52.attn_qkv.weight q6_K blk.52.ffn_down.weight q6_K -blk.53.attn_qkv.weight q5_K -blk.54.attn_qkv.weight q5_K blk.55.attn_v.weight q6_K blk.55.ffn_down.weight q6_K -blk.56.attn_qkv.weight q5_K +blk.56.attn_qkv.weight q6_K blk.56.ffn_down.weight q6_K -blk.57.attn_qkv.weight q5_K +blk.57.attn_qkv.weight q6_K blk.57.ffn_down.weight q6_K -blk.58.attn_qkv.weight q5_K +blk.58.attn_qkv.weight q6_K blk.58.ffn_down.weight q6_K +blk.59.attn_v.weight q6_K blk.59.ffn_down.weight q6_K -blk.60.attn_qkv.weight q5_K +blk.60.attn_qkv.weight q6_K blk.60.ffn_down.weight q6_K -blk.61.attn_qkv.weight q5_K +blk.61.attn_qkv.weight q6_K blk.61.ffn_down.weight q6_K -blk.62.attn_qkv.weight q5_K +blk.62.attn_qkv.weight q6_K blk.62.ffn_down.weight q6_K +blk.63.attn_v.weight q6_K blk.63.ffn_down.weight q6_K [Q5_K_S] q5_K @@ -546,64 +568,36 @@ blk.6.attn_qkv.weight q6_K blk.6.ffn_down.weight q6_K blk.7.attn_v.weight q6_K blk.7.ffn_down.weight q6_K -blk.8.attn_qkv.weight q6_K -blk.9.attn_qkv.weight q6_K blk.10.attn_qkv.weight q6_K blk.10.ffn_down.weight q6_K -blk.11.attn_v.weight q6_K -blk.12.attn_qkv.weight q6_K blk.13.attn_qkv.weight q6_K blk.13.ffn_down.weight q6_K -blk.14.attn_qkv.weight q6_K -blk.15.attn_v.weight q6_K blk.16.attn_qkv.weight q6_K blk.16.ffn_down.weight q6_K -blk.17.attn_qkv.weight q6_K -blk.18.attn_qkv.weight q6_K blk.19.attn_v.weight q6_K blk.19.ffn_down.weight q6_K -blk.20.attn_qkv.weight q6_K -blk.21.attn_qkv.weight q6_K blk.22.attn_qkv.weight q6_K blk.22.ffn_down.weight q6_K -blk.23.attn_v.weight q6_K -blk.24.attn_qkv.weight q6_K blk.25.attn_qkv.weight q6_K blk.25.ffn_down.weight q6_K -blk.26.attn_qkv.weight q6_K -blk.27.attn_v.weight q6_K blk.28.attn_qkv.weight q6_K blk.28.ffn_down.weight q6_K -blk.29.attn_qkv.weight q6_K -blk.30.attn_qkv.weight q6_K blk.31.attn_v.weight q6_K blk.31.ffn_down.weight q6_K -blk.32.attn_qkv.weight q6_K -blk.33.attn_qkv.weight q6_K blk.34.attn_qkv.weight q6_K blk.34.ffn_down.weight q6_K -blk.36.attn_qkv.weight q6_K blk.37.attn_qkv.weight q6_K blk.37.ffn_down.weight q6_K -blk.38.attn_qkv.weight q6_K blk.40.attn_qkv.weight q6_K blk.40.ffn_down.weight q6_K -blk.41.attn_qkv.weight q6_K -blk.42.attn_qkv.weight q6_K blk.43.attn_v.weight q6_K blk.43.ffn_down.weight q6_K -blk.44.attn_qkv.weight q6_K -blk.45.attn_qkv.weight q6_K blk.46.attn_qkv.weight q6_K blk.46.ffn_down.weight q6_K -blk.48.attn_qkv.weight q6_K blk.49.attn_qkv.weight q6_K blk.49.ffn_down.weight q6_K -blk.50.attn_qkv.weight q6_K blk.52.attn_qkv.weight q6_K blk.52.ffn_down.weight q6_K -blk.53.attn_qkv.weight q6_K -blk.54.attn_qkv.weight q6_K blk.55.attn_v.weight q6_K blk.55.ffn_down.weight q6_K blk.56.attn_qkv.weight q6_K @@ -612,6 +606,7 @@ blk.57.attn_qkv.weight q6_K blk.57.ffn_down.weight q6_K blk.58.attn_qkv.weight q6_K blk.58.ffn_down.weight q6_K +blk.59.attn_v.weight q6_K blk.59.ffn_down.weight q6_K blk.60.attn_qkv.weight q6_K blk.60.ffn_down.weight q6_K @@ -619,6 +614,7 @@ blk.61.attn_qkv.weight q6_K blk.61.ffn_down.weight q6_K blk.62.attn_qkv.weight q6_K blk.62.ffn_down.weight q6_K +blk.63.attn_v.weight q6_K blk.63.ffn_down.weight q6_K [Q6_K] q6_K @@ -626,98 +622,251 @@ blk.63.ffn_down.weight q6_K [IQ2_XXS] iq2_xxs output.weight q5_K token_embd.weight q2_K +blk.0.attn_qkv.weight q4_K blk.0.ffn_down.weight q2_K +blk.1.attn_qkv.weight q4_K blk.1.ffn_down.weight q2_K +blk.2.attn_qkv.weight q4_K blk.2.ffn_down.weight q2_K blk.3.attn_v.weight q4_K blk.3.ffn_down.weight q2_K +blk.4.attn_qkv.weight q4_K blk.4.ffn_down.weight q2_K +blk.5.attn_qkv.weight q4_K blk.5.ffn_down.weight q2_K +blk.6.attn_qkv.weight q4_K blk.6.ffn_down.weight q2_K blk.7.attn_v.weight q4_K blk.7.ffn_down.weight q2_K +blk.8.attn_qkv.weight q4_K +blk.9.attn_qkv.weight q4_K +blk.10.attn_qkv.weight q4_K blk.11.attn_v.weight q4_K +blk.12.attn_qkv.weight q4_K +blk.13.attn_qkv.weight q4_K +blk.14.attn_qkv.weight q4_K blk.15.attn_v.weight q4_K +blk.16.attn_qkv.weight q4_K +blk.17.attn_qkv.weight q4_K +blk.18.attn_qkv.weight q4_K blk.19.attn_v.weight q4_K +blk.20.attn_qkv.weight q4_K +blk.21.attn_qkv.weight q4_K +blk.22.attn_qkv.weight q4_K blk.23.attn_v.weight q4_K +blk.24.attn_qkv.weight q4_K +blk.25.attn_qkv.weight q4_K +blk.26.attn_qkv.weight q4_K blk.27.attn_v.weight q4_K +blk.28.attn_qkv.weight q4_K +blk.29.attn_qkv.weight q4_K +blk.30.attn_qkv.weight q4_K blk.31.attn_v.weight q4_K +blk.32.attn_qkv.weight q4_K +blk.33.attn_qkv.weight q4_K +blk.34.attn_qkv.weight q4_K blk.35.attn_v.weight q4_K +blk.36.attn_qkv.weight q4_K +blk.37.attn_qkv.weight q4_K +blk.38.attn_qkv.weight q4_K blk.39.attn_v.weight q4_K +blk.40.attn_qkv.weight q4_K +blk.41.attn_qkv.weight q4_K +blk.42.attn_qkv.weight q4_K blk.43.attn_v.weight q4_K +blk.44.attn_qkv.weight q4_K +blk.45.attn_qkv.weight q4_K +blk.46.attn_qkv.weight q4_K blk.47.attn_v.weight q4_K +blk.48.attn_qkv.weight q4_K +blk.49.attn_qkv.weight q4_K +blk.50.attn_qkv.weight q4_K blk.51.attn_v.weight q4_K +blk.52.attn_qkv.weight q4_K +blk.53.attn_qkv.weight q4_K +blk.54.attn_qkv.weight q4_K blk.55.attn_v.weight q4_K +blk.56.attn_qkv.weight q4_K +blk.57.attn_qkv.weight q4_K +blk.58.attn_qkv.weight q4_K blk.59.attn_v.weight q4_K +blk.60.attn_qkv.weight q4_K +blk.61.attn_qkv.weight q4_K +blk.62.attn_qkv.weight q4_K blk.63.attn_v.weight q4_K [IQ2_XS] iq2_xs output.weight q5_K token_embd.weight q2_K +blk.0.attn_qkv.weight q4_K blk.0.ffn_down.weight q2_K +blk.1.attn_qkv.weight q4_K blk.1.ffn_down.weight q2_K +blk.2.attn_qkv.weight q4_K blk.2.ffn_down.weight q2_K blk.3.attn_v.weight q4_K blk.3.ffn_down.weight q2_K +blk.4.attn_qkv.weight q4_K blk.4.ffn_down.weight q2_K +blk.5.attn_qkv.weight q4_K blk.5.ffn_down.weight q2_K +blk.6.attn_qkv.weight q4_K blk.6.ffn_down.weight q2_K blk.7.attn_v.weight q4_K blk.7.ffn_down.weight q2_K +blk.8.attn_qkv.weight q4_K +blk.9.attn_qkv.weight q4_K +blk.10.attn_qkv.weight q4_K blk.11.attn_v.weight q4_K +blk.12.attn_qkv.weight q4_K +blk.13.attn_qkv.weight q4_K +blk.14.attn_qkv.weight q4_K blk.15.attn_v.weight q4_K +blk.16.attn_qkv.weight q4_K +blk.17.attn_qkv.weight q4_K +blk.18.attn_qkv.weight q4_K blk.19.attn_v.weight q4_K +blk.20.attn_qkv.weight q4_K +blk.21.attn_qkv.weight q4_K +blk.22.attn_qkv.weight q4_K blk.23.attn_v.weight q4_K +blk.24.attn_qkv.weight q4_K +blk.25.attn_qkv.weight q4_K +blk.26.attn_qkv.weight q4_K blk.27.attn_v.weight q4_K +blk.28.attn_qkv.weight q4_K +blk.29.attn_qkv.weight q4_K +blk.30.attn_qkv.weight q4_K blk.31.attn_v.weight q4_K +blk.32.attn_qkv.weight q4_K +blk.33.attn_qkv.weight q4_K +blk.34.attn_qkv.weight q4_K blk.35.attn_v.weight q4_K +blk.36.attn_qkv.weight q4_K +blk.37.attn_qkv.weight q4_K +blk.38.attn_qkv.weight q4_K blk.39.attn_v.weight q4_K +blk.40.attn_qkv.weight q4_K +blk.41.attn_qkv.weight q4_K +blk.42.attn_qkv.weight q4_K blk.43.attn_v.weight q4_K +blk.44.attn_qkv.weight q4_K +blk.45.attn_qkv.weight q4_K +blk.46.attn_qkv.weight q4_K blk.47.attn_v.weight q4_K +blk.48.attn_qkv.weight q4_K +blk.49.attn_qkv.weight q4_K +blk.50.attn_qkv.weight q4_K blk.51.attn_v.weight q4_K +blk.52.attn_qkv.weight q4_K +blk.53.attn_qkv.weight q4_K +blk.54.attn_qkv.weight q4_K blk.55.attn_v.weight q4_K +blk.56.attn_qkv.weight q4_K +blk.57.attn_qkv.weight q4_K +blk.58.attn_qkv.weight q4_K blk.59.attn_v.weight q4_K +blk.60.attn_qkv.weight q4_K +blk.61.attn_qkv.weight q4_K +blk.62.attn_qkv.weight q4_K blk.63.attn_v.weight q4_K [Q2_K_S] q2_K output.weight q6_K +blk.0.attn_qkv.weight q4_K blk.0.ffn_down.weight q4_K +blk.1.attn_qkv.weight q4_K blk.1.ffn_down.weight q4_K +blk.2.attn_qkv.weight q4_K blk.2.ffn_down.weight q4_K blk.3.attn_v.weight q4_K blk.3.ffn_down.weight q4_K +blk.4.attn_qkv.weight q4_K blk.4.ffn_down.weight q4_K +blk.5.attn_qkv.weight q4_K blk.5.ffn_down.weight q4_K +blk.6.attn_qkv.weight q4_K blk.6.ffn_down.weight q4_K blk.7.attn_v.weight q4_K blk.7.ffn_down.weight q4_K +blk.8.attn_qkv.weight q4_K +blk.9.attn_qkv.weight q4_K +blk.10.attn_qkv.weight q4_K blk.11.attn_v.weight q4_K +blk.12.attn_qkv.weight q4_K +blk.13.attn_qkv.weight q4_K +blk.14.attn_qkv.weight q4_K blk.15.attn_v.weight q4_K +blk.16.attn_qkv.weight q4_K +blk.17.attn_qkv.weight q4_K +blk.18.attn_qkv.weight q4_K blk.19.attn_v.weight q4_K +blk.20.attn_qkv.weight q4_K +blk.21.attn_qkv.weight q4_K +blk.22.attn_qkv.weight q4_K blk.23.attn_v.weight q4_K +blk.24.attn_qkv.weight q4_K +blk.25.attn_qkv.weight q4_K +blk.26.attn_qkv.weight q4_K blk.27.attn_v.weight q4_K +blk.28.attn_qkv.weight q4_K +blk.29.attn_qkv.weight q4_K +blk.30.attn_qkv.weight q4_K blk.31.attn_v.weight q4_K +blk.32.attn_qkv.weight q4_K +blk.33.attn_qkv.weight q4_K +blk.34.attn_qkv.weight q4_K blk.35.attn_v.weight q4_K +blk.36.attn_qkv.weight q4_K +blk.37.attn_qkv.weight q4_K +blk.38.attn_qkv.weight q4_K blk.39.attn_v.weight q4_K +blk.40.attn_qkv.weight q4_K +blk.41.attn_qkv.weight q4_K +blk.42.attn_qkv.weight q4_K blk.43.attn_v.weight q4_K +blk.44.attn_qkv.weight q4_K +blk.45.attn_qkv.weight q4_K +blk.46.attn_qkv.weight q4_K blk.47.attn_v.weight q4_K +blk.48.attn_qkv.weight q4_K +blk.49.attn_qkv.weight q4_K +blk.50.attn_qkv.weight q4_K blk.51.attn_v.weight q4_K +blk.52.attn_qkv.weight q4_K +blk.53.attn_qkv.weight q4_K +blk.54.attn_qkv.weight q4_K blk.55.attn_v.weight q4_K +blk.56.attn_qkv.weight q4_K +blk.57.attn_qkv.weight q4_K +blk.58.attn_qkv.weight q4_K blk.59.attn_v.weight q4_K +blk.60.attn_qkv.weight q4_K +blk.61.attn_qkv.weight q4_K +blk.62.attn_qkv.weight q4_K blk.63.attn_v.weight q4_K [IQ3_XS] iq3_s output.weight q6_K +blk.0.attn_qkv.weight q4_K +blk.1.attn_qkv.weight q4_K +blk.2.attn_qkv.weight q4_K blk.3.attn_k.weight iq3_xxs blk.3.attn_q.weight iq3_xxs blk.3.attn_v.weight q4_K +blk.4.attn_qkv.weight q4_K +blk.5.attn_qkv.weight q4_K +blk.6.attn_qkv.weight q4_K blk.7.attn_k.weight iq3_xxs blk.7.attn_q.weight iq3_xxs blk.7.attn_v.weight q4_K +blk.8.attn_qkv.weight q4_K blk.8.ffn_gate.weight iq3_xxs blk.8.ffn_up.weight iq3_xxs +blk.9.attn_qkv.weight q4_K blk.9.ffn_gate.weight iq3_xxs blk.9.ffn_up.weight iq3_xxs +blk.10.attn_qkv.weight q4_K blk.10.ffn_gate.weight iq3_xxs blk.10.ffn_up.weight iq3_xxs blk.11.attn_k.weight iq3_xxs @@ -725,10 +874,13 @@ blk.11.attn_q.weight iq3_xxs blk.11.attn_v.weight q4_K blk.11.ffn_gate.weight iq3_xxs blk.11.ffn_up.weight iq3_xxs +blk.12.attn_qkv.weight q4_K blk.12.ffn_gate.weight iq3_xxs blk.12.ffn_up.weight iq3_xxs +blk.13.attn_qkv.weight q4_K blk.13.ffn_gate.weight iq3_xxs blk.13.ffn_up.weight iq3_xxs +blk.14.attn_qkv.weight q4_K blk.14.ffn_gate.weight iq3_xxs blk.14.ffn_up.weight iq3_xxs blk.15.attn_k.weight iq3_xxs @@ -736,10 +888,13 @@ blk.15.attn_q.weight iq3_xxs blk.15.attn_v.weight q4_K blk.15.ffn_gate.weight iq3_xxs blk.15.ffn_up.weight iq3_xxs +blk.16.attn_qkv.weight q4_K blk.16.ffn_gate.weight iq3_xxs blk.16.ffn_up.weight iq3_xxs +blk.17.attn_qkv.weight q4_K blk.17.ffn_gate.weight iq3_xxs blk.17.ffn_up.weight iq3_xxs +blk.18.attn_qkv.weight q4_K blk.18.ffn_gate.weight iq3_xxs blk.18.ffn_up.weight iq3_xxs blk.19.attn_k.weight iq3_xxs @@ -747,10 +902,13 @@ blk.19.attn_q.weight iq3_xxs blk.19.attn_v.weight q4_K blk.19.ffn_gate.weight iq3_xxs blk.19.ffn_up.weight iq3_xxs +blk.20.attn_qkv.weight q4_K blk.20.ffn_gate.weight iq3_xxs blk.20.ffn_up.weight iq3_xxs +blk.21.attn_qkv.weight q4_K blk.21.ffn_gate.weight iq3_xxs blk.21.ffn_up.weight iq3_xxs +blk.22.attn_qkv.weight q4_K blk.22.ffn_gate.weight iq3_xxs blk.22.ffn_up.weight iq3_xxs blk.23.attn_k.weight iq3_xxs @@ -758,10 +916,13 @@ blk.23.attn_q.weight iq3_xxs blk.23.attn_v.weight q4_K blk.23.ffn_gate.weight iq3_xxs blk.23.ffn_up.weight iq3_xxs +blk.24.attn_qkv.weight q4_K blk.24.ffn_gate.weight iq3_xxs blk.24.ffn_up.weight iq3_xxs +blk.25.attn_qkv.weight q4_K blk.25.ffn_gate.weight iq3_xxs blk.25.ffn_up.weight iq3_xxs +blk.26.attn_qkv.weight q4_K blk.26.ffn_gate.weight iq3_xxs blk.26.ffn_up.weight iq3_xxs blk.27.attn_k.weight iq3_xxs @@ -769,10 +930,13 @@ blk.27.attn_q.weight iq3_xxs blk.27.attn_v.weight q4_K blk.27.ffn_gate.weight iq3_xxs blk.27.ffn_up.weight iq3_xxs +blk.28.attn_qkv.weight q4_K blk.28.ffn_gate.weight iq3_xxs blk.28.ffn_up.weight iq3_xxs +blk.29.attn_qkv.weight q4_K blk.29.ffn_gate.weight iq3_xxs blk.29.ffn_up.weight iq3_xxs +blk.30.attn_qkv.weight q4_K blk.30.ffn_gate.weight iq3_xxs blk.30.ffn_up.weight iq3_xxs blk.31.attn_k.weight iq3_xxs @@ -780,10 +944,13 @@ blk.31.attn_q.weight iq3_xxs blk.31.attn_v.weight q4_K blk.31.ffn_gate.weight iq3_xxs blk.31.ffn_up.weight iq3_xxs +blk.32.attn_qkv.weight q4_K blk.32.ffn_gate.weight iq3_xxs blk.32.ffn_up.weight iq3_xxs +blk.33.attn_qkv.weight q4_K blk.33.ffn_gate.weight iq3_xxs blk.33.ffn_up.weight iq3_xxs +blk.34.attn_qkv.weight q4_K blk.34.ffn_gate.weight iq3_xxs blk.34.ffn_up.weight iq3_xxs blk.35.attn_k.weight iq3_xxs @@ -791,10 +958,13 @@ blk.35.attn_q.weight iq3_xxs blk.35.attn_v.weight q4_K blk.35.ffn_gate.weight iq3_xxs blk.35.ffn_up.weight iq3_xxs +blk.36.attn_qkv.weight q4_K blk.36.ffn_gate.weight iq3_xxs blk.36.ffn_up.weight iq3_xxs +blk.37.attn_qkv.weight q4_K blk.37.ffn_gate.weight iq3_xxs blk.37.ffn_up.weight iq3_xxs +blk.38.attn_qkv.weight q4_K blk.38.ffn_gate.weight iq3_xxs blk.38.ffn_up.weight iq3_xxs blk.39.attn_k.weight iq3_xxs @@ -802,10 +972,13 @@ blk.39.attn_q.weight iq3_xxs blk.39.attn_v.weight q4_K blk.39.ffn_gate.weight iq3_xxs blk.39.ffn_up.weight iq3_xxs +blk.40.attn_qkv.weight q4_K blk.40.ffn_gate.weight iq3_xxs blk.40.ffn_up.weight iq3_xxs +blk.41.attn_qkv.weight q4_K blk.41.ffn_gate.weight iq3_xxs blk.41.ffn_up.weight iq3_xxs +blk.42.attn_qkv.weight q4_K blk.42.ffn_gate.weight iq3_xxs blk.42.ffn_up.weight iq3_xxs blk.43.attn_k.weight iq3_xxs @@ -813,10 +986,13 @@ blk.43.attn_q.weight iq3_xxs blk.43.attn_v.weight q4_K blk.43.ffn_gate.weight iq3_xxs blk.43.ffn_up.weight iq3_xxs +blk.44.attn_qkv.weight q4_K blk.44.ffn_gate.weight iq3_xxs blk.44.ffn_up.weight iq3_xxs +blk.45.attn_qkv.weight q4_K blk.45.ffn_gate.weight iq3_xxs blk.45.ffn_up.weight iq3_xxs +blk.46.attn_qkv.weight q4_K blk.46.ffn_gate.weight iq3_xxs blk.46.ffn_up.weight iq3_xxs blk.47.attn_k.weight iq3_xxs @@ -824,10 +1000,13 @@ blk.47.attn_q.weight iq3_xxs blk.47.attn_v.weight q4_K blk.47.ffn_gate.weight iq3_xxs blk.47.ffn_up.weight iq3_xxs +blk.48.attn_qkv.weight q4_K blk.48.ffn_gate.weight iq3_xxs blk.48.ffn_up.weight iq3_xxs +blk.49.attn_qkv.weight q4_K blk.49.ffn_gate.weight iq3_xxs blk.49.ffn_up.weight iq3_xxs +blk.50.attn_qkv.weight q4_K blk.50.ffn_gate.weight iq3_xxs blk.50.ffn_up.weight iq3_xxs blk.51.attn_k.weight iq3_xxs @@ -835,10 +1014,13 @@ blk.51.attn_q.weight iq3_xxs blk.51.attn_v.weight q4_K blk.51.ffn_gate.weight iq3_xxs blk.51.ffn_up.weight iq3_xxs +blk.52.attn_qkv.weight q4_K blk.52.ffn_gate.weight iq3_xxs blk.52.ffn_up.weight iq3_xxs +blk.53.attn_qkv.weight q4_K blk.53.ffn_gate.weight iq3_xxs blk.53.ffn_up.weight iq3_xxs +blk.54.attn_qkv.weight q4_K blk.54.ffn_gate.weight iq3_xxs blk.54.ffn_up.weight iq3_xxs blk.55.attn_k.weight iq3_xxs @@ -846,9 +1028,15 @@ blk.55.attn_q.weight iq3_xxs blk.55.attn_v.weight q4_K blk.55.ffn_gate.weight iq3_xxs blk.55.ffn_up.weight iq3_xxs +blk.56.attn_qkv.weight q4_K +blk.57.attn_qkv.weight q4_K +blk.58.attn_qkv.weight q4_K blk.59.attn_k.weight iq3_xxs blk.59.attn_q.weight iq3_xxs blk.59.attn_v.weight q4_K +blk.60.attn_qkv.weight q4_K +blk.61.attn_qkv.weight q4_K +blk.62.attn_qkv.weight q4_K blk.63.attn_k.weight iq3_xxs blk.63.attn_q.weight iq3_xxs blk.63.attn_v.weight q4_K @@ -856,128 +1044,176 @@ blk.63.attn_v.weight q4_K [IQ3_XXS] iq3_xxs output.weight q5_K token_embd.weight iq3_s +blk.0.attn_qkv.weight q4_K blk.0.ffn_down.weight q4_K +blk.1.attn_qkv.weight q4_K blk.1.ffn_down.weight q4_K +blk.2.attn_qkv.weight q4_K blk.2.ffn_down.weight q4_K blk.3.attn_k.weight iq2_s blk.3.attn_output.weight iq3_s blk.3.attn_q.weight iq2_s blk.3.attn_v.weight q4_K blk.3.ffn_down.weight q4_K +blk.4.attn_qkv.weight q4_K blk.4.ffn_down.weight q4_K +blk.5.attn_qkv.weight q4_K blk.5.ffn_down.weight q4_K +blk.6.attn_qkv.weight q4_K blk.6.ffn_down.weight q4_K blk.7.attn_k.weight iq2_s blk.7.attn_output.weight iq3_s blk.7.attn_q.weight iq2_s blk.7.attn_v.weight q4_K blk.7.ffn_down.weight q4_K +blk.8.attn_qkv.weight q4_K blk.8.ffn_down.weight q3_K +blk.9.attn_qkv.weight q4_K blk.9.ffn_down.weight q3_K +blk.10.attn_qkv.weight q4_K blk.10.ffn_down.weight q3_K blk.11.attn_k.weight iq2_s blk.11.attn_output.weight iq3_s blk.11.attn_q.weight iq2_s blk.11.attn_v.weight q4_K blk.11.ffn_down.weight q3_K +blk.12.attn_qkv.weight q4_K blk.12.ffn_down.weight q3_K +blk.13.attn_qkv.weight q4_K blk.13.ffn_down.weight q3_K +blk.14.attn_qkv.weight q4_K blk.14.ffn_down.weight q3_K blk.15.attn_k.weight iq2_s blk.15.attn_output.weight iq3_s blk.15.attn_q.weight iq2_s blk.15.attn_v.weight q4_K blk.15.ffn_down.weight q3_K +blk.16.attn_qkv.weight q4_K blk.16.ffn_down.weight q3_K +blk.17.attn_qkv.weight q4_K blk.17.ffn_down.weight q3_K +blk.18.attn_qkv.weight q4_K blk.18.ffn_down.weight q3_K blk.19.attn_k.weight iq2_s blk.19.attn_output.weight iq3_s blk.19.attn_q.weight iq2_s blk.19.attn_v.weight q4_K blk.19.ffn_down.weight q3_K +blk.20.attn_qkv.weight q4_K blk.20.ffn_down.weight q3_K +blk.21.attn_qkv.weight q4_K blk.21.ffn_down.weight q3_K +blk.22.attn_qkv.weight q4_K blk.22.ffn_down.weight q3_K blk.23.attn_k.weight iq2_s blk.23.attn_output.weight iq3_s blk.23.attn_q.weight iq2_s blk.23.attn_v.weight q4_K blk.23.ffn_down.weight q3_K +blk.24.attn_qkv.weight q4_K blk.24.ffn_down.weight q3_K +blk.25.attn_qkv.weight q4_K blk.25.ffn_down.weight q3_K +blk.26.attn_qkv.weight q4_K blk.26.ffn_down.weight q3_K blk.27.attn_k.weight iq2_s blk.27.attn_output.weight iq3_s blk.27.attn_q.weight iq2_s blk.27.attn_v.weight q4_K blk.27.ffn_down.weight q3_K +blk.28.attn_qkv.weight q4_K blk.28.ffn_down.weight q3_K +blk.29.attn_qkv.weight q4_K blk.29.ffn_down.weight q3_K +blk.30.attn_qkv.weight q4_K blk.30.ffn_down.weight q3_K blk.31.attn_k.weight iq2_s blk.31.attn_output.weight iq3_s blk.31.attn_q.weight iq2_s blk.31.attn_v.weight q4_K blk.31.ffn_down.weight q3_K +blk.32.attn_qkv.weight q4_K blk.32.ffn_down.weight q3_K +blk.33.attn_qkv.weight q4_K blk.33.ffn_down.weight q3_K +blk.34.attn_qkv.weight q4_K blk.34.ffn_down.weight q3_K blk.35.attn_k.weight iq2_s blk.35.attn_output.weight iq3_s blk.35.attn_q.weight iq2_s blk.35.attn_v.weight q4_K blk.35.ffn_down.weight q3_K +blk.36.attn_qkv.weight q4_K blk.36.ffn_down.weight q3_K +blk.37.attn_qkv.weight q4_K blk.37.ffn_down.weight q3_K +blk.38.attn_qkv.weight q4_K blk.38.ffn_down.weight q3_K blk.39.attn_k.weight iq2_s blk.39.attn_output.weight iq3_s blk.39.attn_q.weight iq2_s blk.39.attn_v.weight q4_K blk.39.ffn_down.weight q3_K +blk.40.attn_qkv.weight q4_K blk.40.ffn_down.weight q3_K +blk.41.attn_qkv.weight q4_K blk.41.ffn_down.weight q3_K +blk.42.attn_qkv.weight q4_K blk.42.ffn_down.weight q3_K blk.43.attn_k.weight iq2_s blk.43.attn_output.weight iq3_s blk.43.attn_q.weight iq2_s blk.43.attn_v.weight q4_K blk.43.ffn_down.weight q3_K +blk.44.attn_qkv.weight q4_K blk.44.ffn_down.weight q3_K +blk.45.attn_qkv.weight q4_K blk.45.ffn_down.weight q3_K +blk.46.attn_qkv.weight q4_K blk.46.ffn_down.weight q3_K blk.47.attn_k.weight iq2_s blk.47.attn_output.weight iq3_s blk.47.attn_q.weight iq2_s blk.47.attn_v.weight q4_K blk.47.ffn_down.weight q3_K +blk.48.attn_qkv.weight q4_K blk.48.ffn_down.weight q3_K +blk.49.attn_qkv.weight q4_K blk.49.ffn_down.weight q3_K +blk.50.attn_qkv.weight q4_K blk.50.ffn_down.weight q3_K blk.51.attn_k.weight iq2_s blk.51.attn_output.weight iq3_s blk.51.attn_q.weight iq2_s blk.51.attn_v.weight q4_K blk.51.ffn_down.weight q3_K +blk.52.attn_qkv.weight q4_K blk.52.ffn_down.weight q3_K +blk.53.attn_qkv.weight q4_K blk.53.ffn_down.weight q3_K +blk.54.attn_qkv.weight q4_K blk.54.ffn_down.weight q3_K blk.55.attn_k.weight iq2_s blk.55.attn_output.weight iq3_s blk.55.attn_q.weight iq2_s blk.55.attn_v.weight q4_K blk.55.ffn_down.weight q3_K +blk.56.attn_qkv.weight q4_K blk.56.ffn_down.weight q3_K +blk.57.attn_qkv.weight q4_K blk.57.ffn_down.weight q3_K +blk.58.attn_qkv.weight q4_K blk.58.ffn_down.weight q3_K blk.59.attn_k.weight iq2_s blk.59.attn_output.weight iq3_s blk.59.attn_q.weight iq2_s blk.59.attn_v.weight q4_K blk.59.ffn_down.weight q3_K +blk.60.attn_qkv.weight q4_K blk.60.ffn_down.weight q3_K +blk.61.attn_qkv.weight q4_K blk.61.ffn_down.weight q3_K +blk.62.attn_qkv.weight q4_K blk.62.ffn_down.weight q3_K blk.63.attn_k.weight iq2_s blk.63.attn_output.weight iq3_s @@ -988,91 +1224,235 @@ blk.63.ffn_down.weight q3_K [IQ1_S] iq1_s output.weight q5_K token_embd.weight q2_K +blk.0.attn_qkv.weight q4_K blk.0.ffn_down.weight q2_K +blk.1.attn_qkv.weight q4_K blk.1.ffn_down.weight q2_K +blk.2.attn_qkv.weight q4_K blk.2.ffn_down.weight q2_K blk.3.attn_output.weight iq2_xxs blk.3.attn_v.weight q4_K blk.3.ffn_down.weight q2_K +blk.4.attn_qkv.weight q4_K blk.4.ffn_down.weight q2_K +blk.5.attn_qkv.weight q4_K blk.5.ffn_down.weight q2_K +blk.6.attn_qkv.weight q4_K blk.6.ffn_down.weight q2_K blk.7.attn_output.weight iq2_xxs blk.7.attn_v.weight q4_K blk.7.ffn_down.weight q2_K +blk.8.attn_qkv.weight q4_K +blk.9.attn_qkv.weight q4_K +blk.10.attn_qkv.weight q4_K blk.11.attn_output.weight iq2_xxs blk.11.attn_v.weight q4_K +blk.12.attn_qkv.weight q4_K +blk.13.attn_qkv.weight q4_K +blk.14.attn_qkv.weight q4_K blk.15.attn_output.weight iq2_xxs blk.15.attn_v.weight q4_K +blk.16.attn_qkv.weight q4_K +blk.17.attn_qkv.weight q4_K +blk.18.attn_qkv.weight q4_K blk.19.attn_output.weight iq2_xxs blk.19.attn_v.weight q4_K +blk.20.attn_qkv.weight q4_K +blk.21.attn_qkv.weight q4_K +blk.22.attn_qkv.weight q4_K blk.23.attn_output.weight iq2_xxs blk.23.attn_v.weight q4_K +blk.24.attn_qkv.weight q4_K +blk.25.attn_qkv.weight q4_K +blk.26.attn_qkv.weight q4_K blk.27.attn_output.weight iq2_xxs blk.27.attn_v.weight q4_K +blk.28.attn_qkv.weight q4_K +blk.29.attn_qkv.weight q4_K +blk.30.attn_qkv.weight q4_K blk.31.attn_output.weight iq2_xxs blk.31.attn_v.weight q4_K +blk.32.attn_qkv.weight q4_K +blk.33.attn_qkv.weight q4_K +blk.34.attn_qkv.weight q4_K blk.35.attn_output.weight iq2_xxs blk.35.attn_v.weight q4_K +blk.36.attn_qkv.weight q4_K +blk.37.attn_qkv.weight q4_K +blk.38.attn_qkv.weight q4_K blk.39.attn_output.weight iq2_xxs blk.39.attn_v.weight q4_K +blk.40.attn_qkv.weight q4_K +blk.41.attn_qkv.weight q4_K +blk.42.attn_qkv.weight q4_K blk.43.attn_output.weight iq2_xxs blk.43.attn_v.weight q4_K +blk.44.attn_qkv.weight q4_K +blk.45.attn_qkv.weight q4_K +blk.46.attn_qkv.weight q4_K blk.47.attn_output.weight iq2_xxs blk.47.attn_v.weight q4_K +blk.48.attn_qkv.weight q4_K +blk.49.attn_qkv.weight q4_K +blk.50.attn_qkv.weight q4_K blk.51.attn_output.weight iq2_xxs blk.51.attn_v.weight q4_K +blk.52.attn_qkv.weight q4_K +blk.53.attn_qkv.weight q4_K +blk.54.attn_qkv.weight q4_K blk.55.attn_output.weight iq2_xxs blk.55.attn_v.weight q4_K +blk.56.attn_qkv.weight q4_K +blk.57.attn_qkv.weight q4_K +blk.58.attn_qkv.weight q4_K blk.59.attn_output.weight iq2_xxs blk.59.attn_v.weight q4_K +blk.60.attn_qkv.weight q4_K +blk.61.attn_qkv.weight q4_K +blk.62.attn_qkv.weight q4_K blk.63.attn_output.weight iq2_xxs blk.63.attn_v.weight q4_K [IQ4_NL] iq4_nl output.weight q6_K +blk.0.attn_qkv.weight q5_K blk.0.ffn_down.weight q5_K +blk.1.attn_qkv.weight q5_K blk.1.ffn_down.weight q5_K +blk.2.attn_qkv.weight q5_K blk.2.ffn_down.weight q5_K blk.3.attn_v.weight q5_K blk.3.ffn_down.weight q5_K +blk.4.attn_qkv.weight q5_K blk.4.ffn_down.weight q5_K +blk.5.attn_qkv.weight q5_K blk.5.ffn_down.weight q5_K +blk.6.attn_qkv.weight q5_K blk.6.ffn_down.weight q5_K blk.7.attn_v.weight q5_K blk.7.ffn_down.weight q5_K +blk.8.attn_qkv.weight q5_K +blk.9.attn_qkv.weight q5_K +blk.10.attn_qkv.weight q5_K blk.11.attn_v.weight q5_K +blk.12.attn_qkv.weight q5_K +blk.13.attn_qkv.weight q5_K +blk.14.attn_qkv.weight q5_K blk.15.attn_v.weight q5_K +blk.16.attn_qkv.weight q5_K +blk.17.attn_qkv.weight q5_K +blk.18.attn_qkv.weight q5_K blk.19.attn_v.weight q5_K +blk.20.attn_qkv.weight q5_K +blk.21.attn_qkv.weight q5_K +blk.22.attn_qkv.weight q5_K blk.23.attn_v.weight q5_K +blk.24.attn_qkv.weight q5_K +blk.25.attn_qkv.weight q5_K +blk.26.attn_qkv.weight q5_K blk.27.attn_v.weight q5_K +blk.28.attn_qkv.weight q5_K +blk.29.attn_qkv.weight q5_K +blk.30.attn_qkv.weight q5_K blk.31.attn_v.weight q5_K +blk.32.attn_qkv.weight q5_K +blk.33.attn_qkv.weight q5_K +blk.34.attn_qkv.weight q5_K blk.35.attn_v.weight q5_K +blk.36.attn_qkv.weight q5_K +blk.37.attn_qkv.weight q5_K +blk.38.attn_qkv.weight q5_K blk.39.attn_v.weight q5_K +blk.40.attn_qkv.weight q5_K +blk.41.attn_qkv.weight q5_K +blk.42.attn_qkv.weight q5_K blk.43.attn_v.weight q5_K +blk.44.attn_qkv.weight q5_K +blk.45.attn_qkv.weight q5_K +blk.46.attn_qkv.weight q5_K blk.47.attn_v.weight q5_K +blk.48.attn_qkv.weight q5_K +blk.49.attn_qkv.weight q5_K +blk.50.attn_qkv.weight q5_K blk.51.attn_v.weight q5_K +blk.52.attn_qkv.weight q5_K +blk.53.attn_qkv.weight q5_K +blk.54.attn_qkv.weight q5_K blk.55.attn_v.weight q5_K +blk.56.attn_qkv.weight q5_K +blk.57.attn_qkv.weight q5_K +blk.58.attn_qkv.weight q5_K blk.59.attn_v.weight q5_K +blk.60.attn_qkv.weight q5_K +blk.61.attn_qkv.weight q5_K +blk.62.attn_qkv.weight q5_K blk.63.attn_v.weight q5_K [IQ3_S] iq3_s output.weight q6_K +blk.0.attn_qkv.weight q4_K +blk.1.attn_qkv.weight q4_K +blk.2.attn_qkv.weight q4_K blk.3.attn_v.weight q4_K +blk.4.attn_qkv.weight q4_K +blk.5.attn_qkv.weight q4_K +blk.6.attn_qkv.weight q4_K blk.7.attn_v.weight q4_K +blk.8.attn_qkv.weight q4_K +blk.9.attn_qkv.weight q4_K +blk.10.attn_qkv.weight q4_K blk.11.attn_v.weight q4_K +blk.12.attn_qkv.weight q4_K +blk.13.attn_qkv.weight q4_K +blk.14.attn_qkv.weight q4_K blk.15.attn_v.weight q4_K +blk.16.attn_qkv.weight q4_K +blk.17.attn_qkv.weight q4_K +blk.18.attn_qkv.weight q4_K blk.19.attn_v.weight q4_K +blk.20.attn_qkv.weight q4_K +blk.21.attn_qkv.weight q4_K +blk.22.attn_qkv.weight q4_K blk.23.attn_v.weight q4_K +blk.24.attn_qkv.weight q4_K +blk.25.attn_qkv.weight q4_K +blk.26.attn_qkv.weight q4_K blk.27.attn_v.weight q4_K +blk.28.attn_qkv.weight q4_K +blk.29.attn_qkv.weight q4_K +blk.30.attn_qkv.weight q4_K blk.31.attn_v.weight q4_K +blk.32.attn_qkv.weight q4_K +blk.33.attn_qkv.weight q4_K +blk.34.attn_qkv.weight q4_K blk.35.attn_v.weight q4_K +blk.36.attn_qkv.weight q4_K +blk.37.attn_qkv.weight q4_K +blk.38.attn_qkv.weight q4_K blk.39.attn_v.weight q4_K +blk.40.attn_qkv.weight q4_K +blk.41.attn_qkv.weight q4_K +blk.42.attn_qkv.weight q4_K blk.43.attn_v.weight q4_K +blk.44.attn_qkv.weight q4_K +blk.45.attn_qkv.weight q4_K +blk.46.attn_qkv.weight q4_K blk.47.attn_v.weight q4_K +blk.48.attn_qkv.weight q4_K +blk.49.attn_qkv.weight q4_K +blk.50.attn_qkv.weight q4_K blk.51.attn_v.weight q4_K +blk.52.attn_qkv.weight q4_K +blk.53.attn_qkv.weight q4_K +blk.54.attn_qkv.weight q4_K blk.55.attn_v.weight q4_K +blk.56.attn_qkv.weight q4_K +blk.57.attn_qkv.weight q4_K +blk.58.attn_qkv.weight q4_K blk.59.attn_v.weight q4_K +blk.60.attn_qkv.weight q4_K +blk.61.attn_qkv.weight q4_K +blk.62.attn_qkv.weight q4_K blk.63.attn_v.weight q4_K [IQ3_M] iq3_s @@ -1169,159 +1549,351 @@ blk.63.attn_v.weight q4_K [IQ2_S] iq2_xs output.weight q5_K token_embd.weight iq3_s +blk.0.attn_qkv.weight q4_K blk.0.ffn_down.weight iq3_s +blk.1.attn_qkv.weight q4_K blk.1.ffn_down.weight iq3_s +blk.2.attn_qkv.weight q4_K blk.2.ffn_down.weight iq3_s blk.3.attn_output.weight iq3_s blk.3.attn_v.weight q4_K blk.3.ffn_down.weight iq3_s +blk.4.attn_qkv.weight q4_K blk.4.ffn_down.weight iq3_s +blk.5.attn_qkv.weight q4_K blk.5.ffn_down.weight iq3_s +blk.6.attn_qkv.weight q4_K blk.6.ffn_down.weight iq3_s blk.7.attn_output.weight iq3_s blk.7.attn_v.weight q4_K blk.7.ffn_down.weight iq3_s +blk.8.attn_qkv.weight q4_K +blk.9.attn_qkv.weight q4_K +blk.10.attn_qkv.weight q4_K blk.11.attn_output.weight iq3_s blk.11.attn_v.weight q4_K +blk.12.attn_qkv.weight q4_K +blk.13.attn_qkv.weight q4_K +blk.14.attn_qkv.weight q4_K blk.15.attn_output.weight iq3_s blk.15.attn_v.weight q4_K +blk.16.attn_qkv.weight q4_K +blk.17.attn_qkv.weight q4_K +blk.18.attn_qkv.weight q4_K blk.19.attn_output.weight iq3_s blk.19.attn_v.weight q4_K +blk.20.attn_qkv.weight q4_K +blk.21.attn_qkv.weight q4_K +blk.22.attn_qkv.weight q4_K blk.23.attn_output.weight iq3_s blk.23.attn_v.weight q4_K +blk.24.attn_qkv.weight q4_K +blk.25.attn_qkv.weight q4_K +blk.26.attn_qkv.weight q4_K blk.27.attn_output.weight iq3_s blk.27.attn_v.weight q4_K +blk.28.attn_qkv.weight q4_K +blk.29.attn_qkv.weight q4_K +blk.30.attn_qkv.weight q4_K blk.31.attn_output.weight iq3_s blk.31.attn_v.weight q4_K +blk.32.attn_qkv.weight q4_K +blk.33.attn_qkv.weight q4_K +blk.34.attn_qkv.weight q4_K blk.35.attn_output.weight iq3_s blk.35.attn_v.weight q4_K +blk.36.attn_qkv.weight q4_K +blk.37.attn_qkv.weight q4_K +blk.38.attn_qkv.weight q4_K blk.39.attn_output.weight iq3_s blk.39.attn_v.weight q4_K +blk.40.attn_qkv.weight q4_K +blk.41.attn_qkv.weight q4_K +blk.42.attn_qkv.weight q4_K blk.43.attn_output.weight iq3_s blk.43.attn_v.weight q4_K +blk.44.attn_qkv.weight q4_K +blk.45.attn_qkv.weight q4_K +blk.46.attn_qkv.weight q4_K blk.47.attn_output.weight iq3_s blk.47.attn_v.weight q4_K +blk.48.attn_qkv.weight q4_K +blk.49.attn_qkv.weight q4_K +blk.50.attn_qkv.weight q4_K blk.51.attn_output.weight iq3_s blk.51.attn_v.weight q4_K +blk.52.attn_qkv.weight q4_K +blk.53.attn_qkv.weight q4_K +blk.54.attn_qkv.weight q4_K blk.55.attn_output.weight iq3_s blk.55.attn_v.weight q4_K +blk.56.attn_qkv.weight q4_K +blk.57.attn_qkv.weight q4_K +blk.58.attn_qkv.weight q4_K blk.59.attn_output.weight iq3_s blk.59.attn_v.weight q4_K +blk.60.attn_qkv.weight q4_K +blk.61.attn_qkv.weight q4_K +blk.62.attn_qkv.weight q4_K blk.63.attn_output.weight iq3_s blk.63.attn_v.weight q4_K [IQ2_M] iq2_s output.weight q5_K token_embd.weight iq3_s +blk.0.attn_qkv.weight q4_K blk.0.ffn_down.weight iq3_s +blk.1.attn_qkv.weight q4_K blk.1.ffn_down.weight iq3_s +blk.2.attn_qkv.weight q4_K blk.2.ffn_down.weight iq3_s blk.3.attn_output.weight iq3_s blk.3.attn_v.weight q4_K blk.3.ffn_down.weight iq3_s +blk.4.attn_qkv.weight q4_K blk.4.ffn_down.weight iq3_s +blk.5.attn_qkv.weight q4_K blk.5.ffn_down.weight iq3_s +blk.6.attn_qkv.weight q4_K blk.6.ffn_down.weight iq3_s blk.7.attn_output.weight iq3_s blk.7.attn_v.weight q4_K blk.7.ffn_down.weight iq3_s +blk.8.attn_qkv.weight q4_K +blk.9.attn_qkv.weight q4_K +blk.10.attn_qkv.weight q4_K blk.11.attn_output.weight iq3_s blk.11.attn_v.weight q4_K +blk.12.attn_qkv.weight q4_K +blk.13.attn_qkv.weight q4_K +blk.14.attn_qkv.weight q4_K blk.15.attn_output.weight iq3_s blk.15.attn_v.weight q4_K +blk.16.attn_qkv.weight q4_K +blk.17.attn_qkv.weight q4_K +blk.18.attn_qkv.weight q4_K blk.19.attn_output.weight iq3_s blk.19.attn_v.weight q4_K +blk.20.attn_qkv.weight q4_K +blk.21.attn_qkv.weight q4_K +blk.22.attn_qkv.weight q4_K blk.23.attn_output.weight iq3_s blk.23.attn_v.weight q4_K +blk.24.attn_qkv.weight q4_K +blk.25.attn_qkv.weight q4_K +blk.26.attn_qkv.weight q4_K blk.27.attn_output.weight iq3_s blk.27.attn_v.weight q4_K +blk.28.attn_qkv.weight q4_K +blk.29.attn_qkv.weight q4_K +blk.30.attn_qkv.weight q4_K blk.31.attn_output.weight iq3_s blk.31.attn_v.weight q4_K +blk.32.attn_qkv.weight q4_K +blk.33.attn_qkv.weight q4_K +blk.34.attn_qkv.weight q4_K blk.35.attn_output.weight iq3_s blk.35.attn_v.weight q4_K +blk.36.attn_qkv.weight q4_K +blk.37.attn_qkv.weight q4_K +blk.38.attn_qkv.weight q4_K blk.39.attn_output.weight iq3_s blk.39.attn_v.weight q4_K +blk.40.attn_qkv.weight q4_K +blk.41.attn_qkv.weight q4_K +blk.42.attn_qkv.weight q4_K blk.43.attn_output.weight iq3_s blk.43.attn_v.weight q4_K +blk.44.attn_qkv.weight q4_K +blk.45.attn_qkv.weight q4_K +blk.46.attn_qkv.weight q4_K blk.47.attn_output.weight iq3_s blk.47.attn_v.weight q4_K +blk.48.attn_qkv.weight q4_K +blk.49.attn_qkv.weight q4_K +blk.50.attn_qkv.weight q4_K blk.51.attn_output.weight iq3_s blk.51.attn_v.weight q4_K +blk.52.attn_qkv.weight q4_K +blk.53.attn_qkv.weight q4_K +blk.54.attn_qkv.weight q4_K blk.55.attn_output.weight iq3_s blk.55.attn_v.weight q4_K +blk.56.attn_qkv.weight q4_K +blk.57.attn_qkv.weight q4_K +blk.58.attn_qkv.weight q4_K blk.59.attn_output.weight iq3_s blk.59.attn_v.weight q4_K +blk.60.attn_qkv.weight q4_K +blk.61.attn_qkv.weight q4_K +blk.62.attn_qkv.weight q4_K blk.63.attn_output.weight iq3_s blk.63.attn_v.weight q4_K [IQ4_XS] iq4_xs output.weight q6_K +blk.0.attn_qkv.weight q5_K blk.0.ffn_down.weight q5_K +blk.1.attn_qkv.weight q5_K blk.1.ffn_down.weight q5_K +blk.2.attn_qkv.weight q5_K blk.2.ffn_down.weight q5_K blk.3.attn_v.weight q5_K blk.3.ffn_down.weight q5_K +blk.4.attn_qkv.weight q5_K blk.4.ffn_down.weight q5_K +blk.5.attn_qkv.weight q5_K blk.5.ffn_down.weight q5_K +blk.6.attn_qkv.weight q5_K blk.6.ffn_down.weight q5_K blk.7.attn_v.weight q5_K blk.7.ffn_down.weight q5_K +blk.8.attn_qkv.weight q5_K +blk.9.attn_qkv.weight q5_K +blk.10.attn_qkv.weight q5_K blk.11.attn_v.weight q5_K +blk.12.attn_qkv.weight q5_K +blk.13.attn_qkv.weight q5_K +blk.14.attn_qkv.weight q5_K blk.15.attn_v.weight q5_K +blk.16.attn_qkv.weight q5_K +blk.17.attn_qkv.weight q5_K +blk.18.attn_qkv.weight q5_K blk.19.attn_v.weight q5_K +blk.20.attn_qkv.weight q5_K +blk.21.attn_qkv.weight q5_K +blk.22.attn_qkv.weight q5_K blk.23.attn_v.weight q5_K +blk.24.attn_qkv.weight q5_K +blk.25.attn_qkv.weight q5_K +blk.26.attn_qkv.weight q5_K blk.27.attn_v.weight q5_K +blk.28.attn_qkv.weight q5_K +blk.29.attn_qkv.weight q5_K +blk.30.attn_qkv.weight q5_K blk.31.attn_v.weight q5_K +blk.32.attn_qkv.weight q5_K +blk.33.attn_qkv.weight q5_K +blk.34.attn_qkv.weight q5_K blk.35.attn_v.weight q5_K +blk.36.attn_qkv.weight q5_K +blk.37.attn_qkv.weight q5_K +blk.38.attn_qkv.weight q5_K blk.39.attn_v.weight q5_K +blk.40.attn_qkv.weight q5_K +blk.41.attn_qkv.weight q5_K +blk.42.attn_qkv.weight q5_K blk.43.attn_v.weight q5_K +blk.44.attn_qkv.weight q5_K +blk.45.attn_qkv.weight q5_K +blk.46.attn_qkv.weight q5_K blk.47.attn_v.weight q5_K +blk.48.attn_qkv.weight q5_K +blk.49.attn_qkv.weight q5_K +blk.50.attn_qkv.weight q5_K blk.51.attn_v.weight q5_K +blk.52.attn_qkv.weight q5_K +blk.53.attn_qkv.weight q5_K +blk.54.attn_qkv.weight q5_K blk.55.attn_v.weight q5_K +blk.56.attn_qkv.weight q5_K +blk.57.attn_qkv.weight q5_K +blk.58.attn_qkv.weight q5_K blk.59.attn_v.weight q5_K +blk.60.attn_qkv.weight q5_K +blk.61.attn_qkv.weight q5_K +blk.62.attn_qkv.weight q5_K blk.63.attn_v.weight q5_K [IQ1_M] iq1_m output.weight q5_K token_embd.weight q2_K +blk.0.attn_qkv.weight q4_K blk.0.ffn_down.weight q2_K +blk.1.attn_qkv.weight q4_K blk.1.ffn_down.weight q2_K +blk.2.attn_qkv.weight q4_K blk.2.ffn_down.weight q2_K blk.3.attn_output.weight iq2_xxs blk.3.attn_v.weight q4_K blk.3.ffn_down.weight q2_K +blk.4.attn_qkv.weight q4_K blk.4.ffn_down.weight q2_K +blk.5.attn_qkv.weight q4_K blk.5.ffn_down.weight q2_K +blk.6.attn_qkv.weight q4_K blk.6.ffn_down.weight q2_K blk.7.attn_output.weight iq2_xxs blk.7.attn_v.weight q4_K blk.7.ffn_down.weight q2_K +blk.8.attn_qkv.weight q4_K +blk.9.attn_qkv.weight q4_K +blk.10.attn_qkv.weight q4_K blk.11.attn_output.weight iq2_xxs blk.11.attn_v.weight q4_K +blk.12.attn_qkv.weight q4_K +blk.13.attn_qkv.weight q4_K +blk.14.attn_qkv.weight q4_K blk.15.attn_output.weight iq2_xxs blk.15.attn_v.weight q4_K +blk.16.attn_qkv.weight q4_K +blk.17.attn_qkv.weight q4_K +blk.18.attn_qkv.weight q4_K blk.19.attn_output.weight iq2_xxs blk.19.attn_v.weight q4_K +blk.20.attn_qkv.weight q4_K +blk.21.attn_qkv.weight q4_K +blk.22.attn_qkv.weight q4_K blk.23.attn_output.weight iq2_xxs blk.23.attn_v.weight q4_K +blk.24.attn_qkv.weight q4_K +blk.25.attn_qkv.weight q4_K +blk.26.attn_qkv.weight q4_K blk.27.attn_output.weight iq2_xxs blk.27.attn_v.weight q4_K +blk.28.attn_qkv.weight q4_K +blk.29.attn_qkv.weight q4_K +blk.30.attn_qkv.weight q4_K blk.31.attn_output.weight iq2_xxs blk.31.attn_v.weight q4_K +blk.32.attn_qkv.weight q4_K +blk.33.attn_qkv.weight q4_K +blk.34.attn_qkv.weight q4_K blk.35.attn_output.weight iq2_xxs blk.35.attn_v.weight q4_K +blk.36.attn_qkv.weight q4_K +blk.37.attn_qkv.weight q4_K +blk.38.attn_qkv.weight q4_K blk.39.attn_output.weight iq2_xxs blk.39.attn_v.weight q4_K +blk.40.attn_qkv.weight q4_K +blk.41.attn_qkv.weight q4_K +blk.42.attn_qkv.weight q4_K blk.43.attn_output.weight iq2_xxs blk.43.attn_v.weight q4_K +blk.44.attn_qkv.weight q4_K +blk.45.attn_qkv.weight q4_K +blk.46.attn_qkv.weight q4_K blk.47.attn_output.weight iq2_xxs blk.47.attn_v.weight q4_K +blk.48.attn_qkv.weight q4_K +blk.49.attn_qkv.weight q4_K +blk.50.attn_qkv.weight q4_K blk.51.attn_output.weight iq2_xxs blk.51.attn_v.weight q4_K +blk.52.attn_qkv.weight q4_K +blk.53.attn_qkv.weight q4_K +blk.54.attn_qkv.weight q4_K blk.55.attn_output.weight iq2_xxs blk.55.attn_v.weight q4_K +blk.56.attn_qkv.weight q4_K +blk.57.attn_qkv.weight q4_K +blk.58.attn_qkv.weight q4_K blk.59.attn_output.weight iq2_xxs blk.59.attn_v.weight q4_K +blk.60.attn_qkv.weight q4_K +blk.61.attn_qkv.weight q4_K +blk.62.attn_qkv.weight q4_K blk.63.attn_output.weight iq2_xxs blk.63.attn_v.weight q4_K diff --git a/tests/snapshots/qwen3.5-397b-a17b.schema b/tests/snapshots/qwen3.5-397b-a17b.schema index 801c9b4f16..e62948dac8 100644 --- a/tests/snapshots/qwen3.5-397b-a17b.schema +++ b/tests/snapshots/qwen3.5-397b-a17b.schema @@ -23,150 +23,195 @@ output.weight q6_K [Q2_K] q2_K output.weight q6_K +blk.0.attn_qkv.weight q4_K blk.0.ffn_down_exps.weight q3_K blk.0.ffn_down_shexp.weight q3_K +blk.1.attn_qkv.weight q4_K blk.1.ffn_down_exps.weight q3_K blk.1.ffn_down_shexp.weight q3_K +blk.2.attn_qkv.weight q4_K blk.2.ffn_down_exps.weight q3_K blk.2.ffn_down_shexp.weight q3_K blk.3.attn_output.weight q3_K blk.3.attn_v.weight q4_K blk.3.ffn_down_exps.weight q3_K blk.3.ffn_down_shexp.weight q3_K +blk.4.attn_qkv.weight q4_K blk.4.ffn_down_exps.weight q3_K blk.4.ffn_down_shexp.weight q3_K +blk.5.attn_qkv.weight q4_K blk.5.ffn_down_exps.weight q3_K blk.5.ffn_down_shexp.weight q3_K +blk.6.attn_qkv.weight q4_K blk.6.ffn_down_exps.weight q3_K blk.6.ffn_down_shexp.weight q3_K blk.7.attn_output.weight q3_K blk.7.attn_v.weight q4_K blk.7.ffn_down_exps.weight q3_K blk.7.ffn_down_shexp.weight q3_K +blk.8.attn_qkv.weight q4_K blk.8.ffn_down_exps.weight q3_K blk.8.ffn_down_shexp.weight q3_K +blk.9.attn_qkv.weight q4_K blk.9.ffn_down_exps.weight q3_K blk.9.ffn_down_shexp.weight q3_K +blk.10.attn_qkv.weight q4_K blk.10.ffn_down_exps.weight q3_K blk.10.ffn_down_shexp.weight q3_K blk.11.attn_output.weight q3_K blk.11.attn_v.weight q4_K blk.11.ffn_down_exps.weight q3_K blk.11.ffn_down_shexp.weight q3_K +blk.12.attn_qkv.weight q4_K blk.12.ffn_down_exps.weight q3_K blk.12.ffn_down_shexp.weight q3_K +blk.13.attn_qkv.weight q4_K blk.13.ffn_down_exps.weight q3_K blk.13.ffn_down_shexp.weight q3_K +blk.14.attn_qkv.weight q4_K blk.14.ffn_down_exps.weight q3_K blk.14.ffn_down_shexp.weight q3_K blk.15.attn_output.weight q3_K blk.15.attn_v.weight q4_K blk.15.ffn_down_exps.weight q3_K blk.15.ffn_down_shexp.weight q3_K +blk.16.attn_qkv.weight q4_K blk.16.ffn_down_exps.weight q3_K blk.16.ffn_down_shexp.weight q3_K +blk.17.attn_qkv.weight q4_K blk.17.ffn_down_exps.weight q3_K blk.17.ffn_down_shexp.weight q3_K +blk.18.attn_qkv.weight q4_K blk.18.ffn_down_exps.weight q3_K blk.18.ffn_down_shexp.weight q3_K blk.19.attn_output.weight q3_K blk.19.attn_v.weight q4_K blk.19.ffn_down_exps.weight q3_K blk.19.ffn_down_shexp.weight q3_K +blk.20.attn_qkv.weight q4_K blk.20.ffn_down_exps.weight q3_K blk.20.ffn_down_shexp.weight q3_K +blk.21.attn_qkv.weight q4_K blk.21.ffn_down_exps.weight q3_K blk.21.ffn_down_shexp.weight q3_K +blk.22.attn_qkv.weight q4_K blk.22.ffn_down_exps.weight q3_K blk.22.ffn_down_shexp.weight q3_K blk.23.attn_output.weight q3_K blk.23.attn_v.weight q4_K blk.23.ffn_down_exps.weight q3_K blk.23.ffn_down_shexp.weight q3_K +blk.24.attn_qkv.weight q4_K blk.24.ffn_down_exps.weight q3_K blk.24.ffn_down_shexp.weight q3_K +blk.25.attn_qkv.weight q4_K blk.25.ffn_down_exps.weight q3_K blk.25.ffn_down_shexp.weight q3_K +blk.26.attn_qkv.weight q4_K blk.26.ffn_down_exps.weight q3_K blk.26.ffn_down_shexp.weight q3_K blk.27.attn_output.weight q3_K blk.27.attn_v.weight q4_K blk.27.ffn_down_exps.weight q3_K blk.27.ffn_down_shexp.weight q3_K +blk.28.attn_qkv.weight q4_K blk.28.ffn_down_exps.weight q3_K blk.28.ffn_down_shexp.weight q3_K +blk.29.attn_qkv.weight q4_K blk.29.ffn_down_exps.weight q3_K blk.29.ffn_down_shexp.weight q3_K +blk.30.attn_qkv.weight q4_K blk.30.ffn_down_exps.weight q3_K blk.30.ffn_down_shexp.weight q3_K blk.31.attn_output.weight q3_K blk.31.attn_v.weight q4_K blk.31.ffn_down_exps.weight q3_K blk.31.ffn_down_shexp.weight q3_K +blk.32.attn_qkv.weight q4_K blk.32.ffn_down_exps.weight q3_K blk.32.ffn_down_shexp.weight q3_K +blk.33.attn_qkv.weight q4_K blk.33.ffn_down_exps.weight q3_K blk.33.ffn_down_shexp.weight q3_K +blk.34.attn_qkv.weight q4_K blk.34.ffn_down_exps.weight q3_K blk.34.ffn_down_shexp.weight q3_K blk.35.attn_output.weight q3_K blk.35.attn_v.weight q4_K blk.35.ffn_down_exps.weight q3_K blk.35.ffn_down_shexp.weight q3_K +blk.36.attn_qkv.weight q4_K blk.36.ffn_down_exps.weight q3_K blk.36.ffn_down_shexp.weight q3_K +blk.37.attn_qkv.weight q4_K blk.37.ffn_down_exps.weight q3_K blk.37.ffn_down_shexp.weight q3_K +blk.38.attn_qkv.weight q4_K blk.38.ffn_down_exps.weight q3_K blk.38.ffn_down_shexp.weight q3_K blk.39.attn_output.weight q3_K blk.39.attn_v.weight q4_K blk.39.ffn_down_exps.weight q3_K blk.39.ffn_down_shexp.weight q3_K +blk.40.attn_qkv.weight q4_K blk.40.ffn_down_exps.weight q3_K blk.40.ffn_down_shexp.weight q3_K +blk.41.attn_qkv.weight q4_K blk.41.ffn_down_exps.weight q3_K blk.41.ffn_down_shexp.weight q3_K +blk.42.attn_qkv.weight q4_K blk.42.ffn_down_exps.weight q3_K blk.42.ffn_down_shexp.weight q3_K blk.43.attn_output.weight q3_K blk.43.attn_v.weight q4_K blk.43.ffn_down_exps.weight q3_K blk.43.ffn_down_shexp.weight q3_K +blk.44.attn_qkv.weight q4_K blk.44.ffn_down_exps.weight q3_K blk.44.ffn_down_shexp.weight q3_K +blk.45.attn_qkv.weight q4_K blk.45.ffn_down_exps.weight q3_K blk.45.ffn_down_shexp.weight q3_K +blk.46.attn_qkv.weight q4_K blk.46.ffn_down_exps.weight q3_K blk.46.ffn_down_shexp.weight q3_K blk.47.attn_output.weight q3_K blk.47.attn_v.weight q4_K blk.47.ffn_down_exps.weight q3_K blk.47.ffn_down_shexp.weight q3_K +blk.48.attn_qkv.weight q4_K blk.48.ffn_down_exps.weight q3_K blk.48.ffn_down_shexp.weight q3_K +blk.49.attn_qkv.weight q4_K blk.49.ffn_down_exps.weight q3_K blk.49.ffn_down_shexp.weight q3_K +blk.50.attn_qkv.weight q4_K blk.50.ffn_down_exps.weight q3_K blk.50.ffn_down_shexp.weight q3_K blk.51.attn_output.weight q3_K blk.51.attn_v.weight q4_K blk.51.ffn_down_exps.weight q3_K blk.51.ffn_down_shexp.weight q3_K +blk.52.attn_qkv.weight q4_K blk.52.ffn_down_exps.weight q3_K blk.52.ffn_down_shexp.weight q3_K +blk.53.attn_qkv.weight q4_K blk.53.ffn_down_exps.weight q3_K blk.53.ffn_down_shexp.weight q3_K +blk.54.attn_qkv.weight q4_K blk.54.ffn_down_exps.weight q3_K blk.54.ffn_down_shexp.weight q3_K blk.55.attn_output.weight q3_K blk.55.attn_v.weight q4_K blk.55.ffn_down_exps.weight q3_K blk.55.ffn_down_shexp.weight q3_K +blk.56.attn_qkv.weight q4_K blk.56.ffn_down_exps.weight q3_K blk.56.ffn_down_shexp.weight q3_K +blk.57.attn_qkv.weight q4_K blk.57.ffn_down_exps.weight q3_K blk.57.ffn_down_shexp.weight q3_K +blk.58.attn_qkv.weight q4_K blk.58.ffn_down_exps.weight q3_K blk.58.ffn_down_shexp.weight q3_K blk.59.attn_output.weight q3_K @@ -179,17 +224,17 @@ output.weight q6_K [Q3_K_M] q3_K output.weight q6_K -blk.0.attn_qkv.weight q4_K +blk.0.attn_qkv.weight q5_K blk.0.ffn_down_exps.weight q5_K blk.0.ffn_down_shexp.weight q5_K -blk.1.attn_qkv.weight q4_K +blk.1.attn_qkv.weight q5_K blk.1.ffn_down_exps.weight q5_K blk.1.ffn_down_shexp.weight q5_K blk.2.attn_qkv.weight q4_K blk.2.ffn_down_exps.weight q5_K blk.2.ffn_down_shexp.weight q5_K blk.3.attn_output.weight q4_K -blk.3.attn_v.weight q5_K +blk.3.attn_v.weight q4_K blk.3.ffn_down_exps.weight q4_K blk.3.ffn_down_shexp.weight q4_K blk.4.attn_qkv.weight q4_K @@ -202,7 +247,7 @@ blk.6.attn_qkv.weight q4_K blk.6.ffn_down_exps.weight q4_K blk.6.ffn_down_shexp.weight q4_K blk.7.attn_output.weight q4_K -blk.7.attn_v.weight q5_K +blk.7.attn_v.weight q4_K blk.7.ffn_down_exps.weight q4_K blk.7.ffn_down_shexp.weight q4_K blk.8.attn_qkv.weight q4_K @@ -377,195 +422,195 @@ blk.59.ffn_down_shexp.weight q4_K [Q3_K_L] q3_K output.weight q6_K -blk.0.attn_qkv.weight q4_K +blk.0.attn_qkv.weight q5_K blk.0.ffn_down_exps.weight q5_K blk.0.ffn_down_shexp.weight q5_K -blk.1.attn_qkv.weight q4_K +blk.1.attn_qkv.weight q5_K blk.1.ffn_down_exps.weight q5_K blk.1.ffn_down_shexp.weight q5_K -blk.2.attn_qkv.weight q4_K +blk.2.attn_qkv.weight q5_K blk.2.ffn_down_exps.weight q5_K blk.2.ffn_down_shexp.weight q5_K blk.3.attn_output.weight q5_K blk.3.attn_v.weight q5_K blk.3.ffn_down_exps.weight q5_K blk.3.ffn_down_shexp.weight q5_K -blk.4.attn_qkv.weight q4_K +blk.4.attn_qkv.weight q5_K blk.4.ffn_down_exps.weight q5_K blk.4.ffn_down_shexp.weight q5_K -blk.5.attn_qkv.weight q4_K +blk.5.attn_qkv.weight q5_K blk.5.ffn_down_exps.weight q5_K blk.5.ffn_down_shexp.weight q5_K -blk.6.attn_qkv.weight q4_K +blk.6.attn_qkv.weight q5_K blk.6.ffn_down_exps.weight q5_K blk.6.ffn_down_shexp.weight q5_K blk.7.attn_output.weight q5_K blk.7.attn_v.weight q5_K blk.7.ffn_down_exps.weight q5_K blk.7.ffn_down_shexp.weight q5_K -blk.8.attn_qkv.weight q4_K +blk.8.attn_qkv.weight q5_K blk.8.ffn_down_exps.weight q5_K blk.8.ffn_down_shexp.weight q5_K -blk.9.attn_qkv.weight q4_K +blk.9.attn_qkv.weight q5_K blk.9.ffn_down_exps.weight q5_K blk.9.ffn_down_shexp.weight q5_K -blk.10.attn_qkv.weight q4_K +blk.10.attn_qkv.weight q5_K blk.10.ffn_down_exps.weight q5_K blk.10.ffn_down_shexp.weight q5_K blk.11.attn_output.weight q5_K blk.11.attn_v.weight q5_K blk.11.ffn_down_exps.weight q5_K blk.11.ffn_down_shexp.weight q5_K -blk.12.attn_qkv.weight q4_K +blk.12.attn_qkv.weight q5_K blk.12.ffn_down_exps.weight q5_K blk.12.ffn_down_shexp.weight q5_K -blk.13.attn_qkv.weight q4_K +blk.13.attn_qkv.weight q5_K blk.13.ffn_down_exps.weight q5_K blk.13.ffn_down_shexp.weight q5_K -blk.14.attn_qkv.weight q4_K +blk.14.attn_qkv.weight q5_K blk.14.ffn_down_exps.weight q5_K blk.14.ffn_down_shexp.weight q5_K blk.15.attn_output.weight q5_K blk.15.attn_v.weight q5_K blk.15.ffn_down_exps.weight q5_K blk.15.ffn_down_shexp.weight q5_K -blk.16.attn_qkv.weight q4_K +blk.16.attn_qkv.weight q5_K blk.16.ffn_down_exps.weight q5_K blk.16.ffn_down_shexp.weight q5_K -blk.17.attn_qkv.weight q4_K +blk.17.attn_qkv.weight q5_K blk.17.ffn_down_exps.weight q5_K blk.17.ffn_down_shexp.weight q5_K -blk.18.attn_qkv.weight q4_K +blk.18.attn_qkv.weight q5_K blk.18.ffn_down_exps.weight q5_K blk.18.ffn_down_shexp.weight q5_K blk.19.attn_output.weight q5_K blk.19.attn_v.weight q5_K blk.19.ffn_down_exps.weight q5_K blk.19.ffn_down_shexp.weight q5_K -blk.20.attn_qkv.weight q4_K +blk.20.attn_qkv.weight q5_K blk.20.ffn_down_exps.weight q5_K blk.20.ffn_down_shexp.weight q5_K -blk.21.attn_qkv.weight q4_K +blk.21.attn_qkv.weight q5_K blk.21.ffn_down_exps.weight q5_K blk.21.ffn_down_shexp.weight q5_K -blk.22.attn_qkv.weight q4_K +blk.22.attn_qkv.weight q5_K blk.22.ffn_down_exps.weight q5_K blk.22.ffn_down_shexp.weight q5_K blk.23.attn_output.weight q5_K blk.23.attn_v.weight q5_K blk.23.ffn_down_exps.weight q5_K blk.23.ffn_down_shexp.weight q5_K -blk.24.attn_qkv.weight q4_K +blk.24.attn_qkv.weight q5_K blk.24.ffn_down_exps.weight q5_K blk.24.ffn_down_shexp.weight q5_K -blk.25.attn_qkv.weight q4_K +blk.25.attn_qkv.weight q5_K blk.25.ffn_down_exps.weight q5_K blk.25.ffn_down_shexp.weight q5_K -blk.26.attn_qkv.weight q4_K +blk.26.attn_qkv.weight q5_K blk.26.ffn_down_exps.weight q5_K blk.26.ffn_down_shexp.weight q5_K blk.27.attn_output.weight q5_K blk.27.attn_v.weight q5_K blk.27.ffn_down_exps.weight q5_K blk.27.ffn_down_shexp.weight q5_K -blk.28.attn_qkv.weight q4_K +blk.28.attn_qkv.weight q5_K blk.28.ffn_down_exps.weight q5_K blk.28.ffn_down_shexp.weight q5_K -blk.29.attn_qkv.weight q4_K +blk.29.attn_qkv.weight q5_K blk.29.ffn_down_exps.weight q5_K blk.29.ffn_down_shexp.weight q5_K -blk.30.attn_qkv.weight q4_K +blk.30.attn_qkv.weight q5_K blk.30.ffn_down_exps.weight q5_K blk.30.ffn_down_shexp.weight q5_K blk.31.attn_output.weight q5_K blk.31.attn_v.weight q5_K blk.31.ffn_down_exps.weight q5_K blk.31.ffn_down_shexp.weight q5_K -blk.32.attn_qkv.weight q4_K +blk.32.attn_qkv.weight q5_K blk.32.ffn_down_exps.weight q5_K blk.32.ffn_down_shexp.weight q5_K -blk.33.attn_qkv.weight q4_K +blk.33.attn_qkv.weight q5_K blk.33.ffn_down_exps.weight q5_K blk.33.ffn_down_shexp.weight q5_K -blk.34.attn_qkv.weight q4_K +blk.34.attn_qkv.weight q5_K blk.34.ffn_down_exps.weight q5_K blk.34.ffn_down_shexp.weight q5_K blk.35.attn_output.weight q5_K blk.35.attn_v.weight q5_K blk.35.ffn_down_exps.weight q5_K blk.35.ffn_down_shexp.weight q5_K -blk.36.attn_qkv.weight q4_K +blk.36.attn_qkv.weight q5_K blk.36.ffn_down_exps.weight q5_K blk.36.ffn_down_shexp.weight q5_K -blk.37.attn_qkv.weight q4_K +blk.37.attn_qkv.weight q5_K blk.37.ffn_down_exps.weight q5_K blk.37.ffn_down_shexp.weight q5_K -blk.38.attn_qkv.weight q4_K +blk.38.attn_qkv.weight q5_K blk.38.ffn_down_exps.weight q5_K blk.38.ffn_down_shexp.weight q5_K blk.39.attn_output.weight q5_K blk.39.attn_v.weight q5_K blk.39.ffn_down_exps.weight q5_K blk.39.ffn_down_shexp.weight q5_K -blk.40.attn_qkv.weight q4_K +blk.40.attn_qkv.weight q5_K blk.40.ffn_down_exps.weight q5_K blk.40.ffn_down_shexp.weight q5_K -blk.41.attn_qkv.weight q4_K +blk.41.attn_qkv.weight q5_K blk.41.ffn_down_exps.weight q5_K blk.41.ffn_down_shexp.weight q5_K -blk.42.attn_qkv.weight q4_K +blk.42.attn_qkv.weight q5_K blk.42.ffn_down_exps.weight q5_K blk.42.ffn_down_shexp.weight q5_K blk.43.attn_output.weight q5_K blk.43.attn_v.weight q5_K blk.43.ffn_down_exps.weight q5_K blk.43.ffn_down_shexp.weight q5_K -blk.44.attn_qkv.weight q4_K +blk.44.attn_qkv.weight q5_K blk.44.ffn_down_exps.weight q5_K blk.44.ffn_down_shexp.weight q5_K -blk.45.attn_qkv.weight q4_K +blk.45.attn_qkv.weight q5_K blk.45.ffn_down_exps.weight q5_K blk.45.ffn_down_shexp.weight q5_K -blk.46.attn_qkv.weight q4_K +blk.46.attn_qkv.weight q5_K blk.46.ffn_down_exps.weight q5_K blk.46.ffn_down_shexp.weight q5_K blk.47.attn_output.weight q5_K blk.47.attn_v.weight q5_K blk.47.ffn_down_exps.weight q5_K blk.47.ffn_down_shexp.weight q5_K -blk.48.attn_qkv.weight q4_K +blk.48.attn_qkv.weight q5_K blk.48.ffn_down_exps.weight q5_K blk.48.ffn_down_shexp.weight q5_K -blk.49.attn_qkv.weight q4_K +blk.49.attn_qkv.weight q5_K blk.49.ffn_down_exps.weight q5_K blk.49.ffn_down_shexp.weight q5_K -blk.50.attn_qkv.weight q4_K +blk.50.attn_qkv.weight q5_K blk.50.ffn_down_exps.weight q5_K blk.50.ffn_down_shexp.weight q5_K blk.51.attn_output.weight q5_K blk.51.attn_v.weight q5_K blk.51.ffn_down_exps.weight q5_K blk.51.ffn_down_shexp.weight q5_K -blk.52.attn_qkv.weight q4_K +blk.52.attn_qkv.weight q5_K blk.52.ffn_down_exps.weight q5_K blk.52.ffn_down_shexp.weight q5_K -blk.53.attn_qkv.weight q4_K +blk.53.attn_qkv.weight q5_K blk.53.ffn_down_exps.weight q5_K blk.53.ffn_down_shexp.weight q5_K -blk.54.attn_qkv.weight q4_K +blk.54.attn_qkv.weight q5_K blk.54.ffn_down_exps.weight q5_K blk.54.ffn_down_shexp.weight q5_K blk.55.attn_output.weight q5_K blk.55.attn_v.weight q5_K blk.55.ffn_down_exps.weight q5_K blk.55.ffn_down_shexp.weight q5_K -blk.56.attn_qkv.weight q4_K +blk.56.attn_qkv.weight q5_K blk.56.ffn_down_exps.weight q5_K blk.56.ffn_down_shexp.weight q5_K -blk.57.attn_qkv.weight q4_K +blk.57.attn_qkv.weight q5_K blk.57.ffn_down_exps.weight q5_K blk.57.ffn_down_shexp.weight q5_K -blk.58.attn_qkv.weight q4_K +blk.58.attn_qkv.weight q5_K blk.58.ffn_down_exps.weight q5_K blk.58.ffn_down_shexp.weight q5_K blk.59.attn_output.weight q5_K @@ -575,10 +620,13 @@ blk.59.ffn_down_shexp.weight q5_K [Q4_K_S] q4_K output.weight q6_K +blk.0.attn_qkv.weight q5_K blk.0.ffn_down_exps.weight q5_K blk.0.ffn_down_shexp.weight q5_K +blk.1.attn_qkv.weight q5_K blk.1.ffn_down_exps.weight q5_K blk.1.ffn_down_shexp.weight q5_K +blk.2.attn_qkv.weight q5_K blk.2.ffn_down_exps.weight q5_K blk.2.ffn_down_shexp.weight q5_K blk.3.attn_v.weight q5_K @@ -590,124 +638,97 @@ blk.5.ffn_down_exps.weight q5_K blk.5.ffn_down_shexp.weight q5_K blk.6.ffn_down_exps.weight q5_K blk.6.ffn_down_shexp.weight q5_K -blk.7.attn_v.weight q5_K -blk.11.attn_v.weight q5_K -blk.15.attn_v.weight q5_K [Q4_K_M] q4_K output.weight q6_K -blk.0.attn_qkv.weight q5_K +blk.0.attn_qkv.weight q6_K blk.0.ffn_down_exps.weight q6_K blk.0.ffn_down_shexp.weight q6_K -blk.1.attn_qkv.weight q5_K +blk.1.attn_qkv.weight q6_K blk.1.ffn_down_exps.weight q6_K blk.1.ffn_down_shexp.weight q6_K -blk.2.attn_qkv.weight q5_K +blk.2.attn_qkv.weight q6_K blk.2.ffn_down_exps.weight q6_K blk.2.ffn_down_shexp.weight q6_K blk.3.attn_v.weight q6_K blk.3.ffn_down_exps.weight q6_K blk.3.ffn_down_shexp.weight q6_K -blk.4.attn_qkv.weight q5_K +blk.4.attn_qkv.weight q6_K blk.4.ffn_down_exps.weight q6_K blk.4.ffn_down_shexp.weight q6_K -blk.5.attn_qkv.weight q5_K +blk.5.attn_qkv.weight q6_K blk.5.ffn_down_exps.weight q6_K blk.5.ffn_down_shexp.weight q6_K -blk.6.attn_qkv.weight q5_K +blk.6.attn_qkv.weight q6_K blk.6.ffn_down_exps.weight q6_K blk.6.ffn_down_shexp.weight q6_K -blk.7.attn_v.weight q6_K -blk.8.attn_qkv.weight q5_K -blk.9.attn_qkv.weight q5_K +blk.9.attn_qkv.weight q6_K blk.9.ffn_down_exps.weight q6_K blk.9.ffn_down_shexp.weight q6_K -blk.10.attn_qkv.weight q5_K -blk.11.attn_v.weight q6_K -blk.12.attn_qkv.weight q5_K +blk.12.attn_qkv.weight q6_K blk.12.ffn_down_exps.weight q6_K blk.12.ffn_down_shexp.weight q6_K -blk.13.attn_qkv.weight q5_K -blk.14.attn_qkv.weight q5_K blk.15.attn_v.weight q6_K blk.15.ffn_down_exps.weight q6_K blk.15.ffn_down_shexp.weight q6_K -blk.16.attn_qkv.weight q5_K -blk.17.attn_qkv.weight q5_K -blk.18.attn_qkv.weight q5_K +blk.18.attn_qkv.weight q6_K blk.18.ffn_down_exps.weight q6_K blk.18.ffn_down_shexp.weight q6_K -blk.19.attn_v.weight q6_K -blk.20.attn_qkv.weight q5_K -blk.21.attn_qkv.weight q5_K +blk.21.attn_qkv.weight q6_K blk.21.ffn_down_exps.weight q6_K blk.21.ffn_down_shexp.weight q6_K -blk.22.attn_qkv.weight q5_K -blk.23.attn_v.weight q6_K -blk.24.attn_qkv.weight q5_K +blk.24.attn_qkv.weight q6_K blk.24.ffn_down_exps.weight q6_K blk.24.ffn_down_shexp.weight q6_K -blk.25.attn_qkv.weight q5_K -blk.26.attn_qkv.weight q5_K blk.27.attn_v.weight q6_K blk.27.ffn_down_exps.weight q6_K blk.27.ffn_down_shexp.weight q6_K -blk.28.attn_qkv.weight q5_K -blk.29.attn_qkv.weight q5_K -blk.30.attn_qkv.weight q5_K +blk.30.attn_qkv.weight q6_K blk.30.ffn_down_exps.weight q6_K blk.30.ffn_down_shexp.weight q6_K -blk.32.attn_qkv.weight q5_K -blk.33.attn_qkv.weight q5_K +blk.33.attn_qkv.weight q6_K blk.33.ffn_down_exps.weight q6_K blk.33.ffn_down_shexp.weight q6_K -blk.34.attn_qkv.weight q5_K -blk.36.attn_qkv.weight q5_K +blk.36.attn_qkv.weight q6_K blk.36.ffn_down_exps.weight q6_K blk.36.ffn_down_shexp.weight q6_K -blk.37.attn_qkv.weight q5_K -blk.38.attn_qkv.weight q5_K blk.39.attn_v.weight q6_K blk.39.ffn_down_exps.weight q6_K blk.39.ffn_down_shexp.weight q6_K -blk.40.attn_qkv.weight q5_K -blk.41.attn_qkv.weight q5_K -blk.42.attn_qkv.weight q5_K +blk.42.attn_qkv.weight q6_K blk.42.ffn_down_exps.weight q6_K blk.42.ffn_down_shexp.weight q6_K -blk.44.attn_qkv.weight q5_K -blk.45.attn_qkv.weight q5_K +blk.45.attn_qkv.weight q6_K blk.45.ffn_down_exps.weight q6_K blk.45.ffn_down_shexp.weight q6_K -blk.46.attn_qkv.weight q5_K -blk.48.attn_qkv.weight q5_K +blk.48.attn_qkv.weight q6_K blk.48.ffn_down_exps.weight q6_K blk.48.ffn_down_shexp.weight q6_K -blk.49.attn_qkv.weight q5_K -blk.50.attn_qkv.weight q5_K blk.51.attn_v.weight q6_K blk.51.ffn_down_exps.weight q6_K blk.51.ffn_down_shexp.weight q6_K -blk.52.attn_qkv.weight q5_K +blk.52.attn_qkv.weight q6_K blk.52.ffn_down_exps.weight q6_K blk.52.ffn_down_shexp.weight q6_K -blk.53.attn_qkv.weight q5_K +blk.53.attn_qkv.weight q6_K blk.53.ffn_down_exps.weight q6_K blk.53.ffn_down_shexp.weight q6_K -blk.54.attn_qkv.weight q5_K +blk.54.attn_qkv.weight q6_K blk.54.ffn_down_exps.weight q6_K blk.54.ffn_down_shexp.weight q6_K +blk.55.attn_v.weight q6_K blk.55.ffn_down_exps.weight q6_K blk.55.ffn_down_shexp.weight q6_K -blk.56.attn_qkv.weight q5_K +blk.56.attn_qkv.weight q6_K blk.56.ffn_down_exps.weight q6_K blk.56.ffn_down_shexp.weight q6_K -blk.57.attn_qkv.weight q5_K +blk.57.attn_qkv.weight q6_K blk.57.ffn_down_exps.weight q6_K blk.57.ffn_down_shexp.weight q6_K -blk.58.attn_qkv.weight q5_K +blk.58.attn_qkv.weight q6_K blk.58.ffn_down_exps.weight q6_K blk.58.ffn_down_shexp.weight q6_K +blk.59.attn_v.weight q6_K blk.59.ffn_down_exps.weight q6_K blk.59.ffn_down_shexp.weight q6_K @@ -737,74 +758,48 @@ blk.5.ffn_down_shexp.weight q6_K blk.6.attn_qkv.weight q6_K blk.6.ffn_down_exps.weight q6_K blk.6.ffn_down_shexp.weight q6_K -blk.7.attn_v.weight q6_K -blk.8.attn_qkv.weight q6_K blk.9.attn_qkv.weight q6_K blk.9.ffn_down_exps.weight q6_K blk.9.ffn_down_shexp.weight q6_K -blk.10.attn_qkv.weight q6_K -blk.11.attn_v.weight q6_K blk.12.attn_qkv.weight q6_K blk.12.ffn_down_exps.weight q6_K blk.12.ffn_down_shexp.weight q6_K -blk.13.attn_qkv.weight q6_K -blk.14.attn_qkv.weight q6_K blk.15.attn_v.weight q6_K blk.15.ffn_down_exps.weight q6_K blk.15.ffn_down_shexp.weight q6_K -blk.16.attn_qkv.weight q6_K -blk.17.attn_qkv.weight q6_K blk.18.attn_qkv.weight q6_K blk.18.ffn_down_exps.weight q6_K blk.18.ffn_down_shexp.weight q6_K -blk.19.attn_v.weight q6_K -blk.20.attn_qkv.weight q6_K blk.21.attn_qkv.weight q6_K blk.21.ffn_down_exps.weight q6_K blk.21.ffn_down_shexp.weight q6_K -blk.22.attn_qkv.weight q6_K -blk.23.attn_v.weight q6_K blk.24.attn_qkv.weight q6_K blk.24.ffn_down_exps.weight q6_K blk.24.ffn_down_shexp.weight q6_K -blk.25.attn_qkv.weight q6_K -blk.26.attn_qkv.weight q6_K blk.27.attn_v.weight q6_K blk.27.ffn_down_exps.weight q6_K blk.27.ffn_down_shexp.weight q6_K -blk.28.attn_qkv.weight q6_K -blk.29.attn_qkv.weight q6_K blk.30.attn_qkv.weight q6_K blk.30.ffn_down_exps.weight q6_K blk.30.ffn_down_shexp.weight q6_K -blk.32.attn_qkv.weight q6_K blk.33.attn_qkv.weight q6_K blk.33.ffn_down_exps.weight q6_K blk.33.ffn_down_shexp.weight q6_K -blk.34.attn_qkv.weight q6_K blk.36.attn_qkv.weight q6_K blk.36.ffn_down_exps.weight q6_K blk.36.ffn_down_shexp.weight q6_K -blk.37.attn_qkv.weight q6_K -blk.38.attn_qkv.weight q6_K blk.39.attn_v.weight q6_K blk.39.ffn_down_exps.weight q6_K blk.39.ffn_down_shexp.weight q6_K -blk.40.attn_qkv.weight q6_K -blk.41.attn_qkv.weight q6_K blk.42.attn_qkv.weight q6_K blk.42.ffn_down_exps.weight q6_K blk.42.ffn_down_shexp.weight q6_K -blk.44.attn_qkv.weight q6_K blk.45.attn_qkv.weight q6_K blk.45.ffn_down_exps.weight q6_K blk.45.ffn_down_shexp.weight q6_K -blk.46.attn_qkv.weight q6_K blk.48.attn_qkv.weight q6_K blk.48.ffn_down_exps.weight q6_K blk.48.ffn_down_shexp.weight q6_K -blk.49.attn_qkv.weight q6_K -blk.50.attn_qkv.weight q6_K blk.51.attn_v.weight q6_K blk.51.ffn_down_exps.weight q6_K blk.51.ffn_down_shexp.weight q6_K @@ -817,6 +812,7 @@ blk.53.ffn_down_shexp.weight q6_K blk.54.attn_qkv.weight q6_K blk.54.ffn_down_exps.weight q6_K blk.54.ffn_down_shexp.weight q6_K +blk.55.attn_v.weight q6_K blk.55.ffn_down_exps.weight q6_K blk.55.ffn_down_shexp.weight q6_K blk.56.attn_qkv.weight q6_K @@ -828,6 +824,7 @@ blk.57.ffn_down_shexp.weight q6_K blk.58.attn_qkv.weight q6_K blk.58.ffn_down_exps.weight q6_K blk.58.ffn_down_shexp.weight q6_K +blk.59.attn_v.weight q6_K blk.59.ffn_down_exps.weight q6_K blk.59.ffn_down_shexp.weight q6_K @@ -836,92 +833,233 @@ blk.59.ffn_down_shexp.weight q6_K [IQ2_XXS] iq2_xxs output.weight q5_K token_embd.weight q2_K +blk.0.attn_qkv.weight q4_K blk.0.ffn_down_exps.weight q2_K blk.0.ffn_down_shexp.weight q2_K +blk.1.attn_qkv.weight q4_K blk.1.ffn_down_exps.weight q2_K blk.1.ffn_down_shexp.weight q2_K +blk.2.attn_qkv.weight q4_K blk.2.ffn_down_exps.weight q2_K blk.2.ffn_down_shexp.weight q2_K blk.3.attn_v.weight q4_K blk.3.ffn_down_exps.weight q2_K +blk.4.attn_qkv.weight q4_K +blk.5.attn_qkv.weight q4_K +blk.6.attn_qkv.weight q4_K blk.7.attn_v.weight q4_K +blk.8.attn_qkv.weight q4_K +blk.9.attn_qkv.weight q4_K +blk.10.attn_qkv.weight q4_K blk.11.attn_v.weight q4_K +blk.12.attn_qkv.weight q4_K +blk.13.attn_qkv.weight q4_K +blk.14.attn_qkv.weight q4_K blk.15.attn_v.weight q4_K +blk.16.attn_qkv.weight q4_K +blk.17.attn_qkv.weight q4_K +blk.18.attn_qkv.weight q4_K blk.19.attn_v.weight q4_K +blk.20.attn_qkv.weight q4_K +blk.21.attn_qkv.weight q4_K +blk.22.attn_qkv.weight q4_K blk.23.attn_v.weight q4_K +blk.24.attn_qkv.weight q4_K +blk.25.attn_qkv.weight q4_K +blk.26.attn_qkv.weight q4_K blk.27.attn_v.weight q4_K +blk.28.attn_qkv.weight q4_K +blk.29.attn_qkv.weight q4_K +blk.30.attn_qkv.weight q4_K blk.31.attn_v.weight q4_K +blk.32.attn_qkv.weight q4_K +blk.33.attn_qkv.weight q4_K +blk.34.attn_qkv.weight q4_K blk.35.attn_v.weight q4_K +blk.36.attn_qkv.weight q4_K +blk.37.attn_qkv.weight q4_K +blk.38.attn_qkv.weight q4_K blk.39.attn_v.weight q4_K +blk.40.attn_qkv.weight q4_K +blk.41.attn_qkv.weight q4_K +blk.42.attn_qkv.weight q4_K blk.43.attn_v.weight q4_K +blk.44.attn_qkv.weight q4_K +blk.45.attn_qkv.weight q4_K +blk.46.attn_qkv.weight q4_K blk.47.attn_v.weight q4_K +blk.48.attn_qkv.weight q4_K +blk.49.attn_qkv.weight q4_K +blk.50.attn_qkv.weight q4_K blk.51.attn_v.weight q4_K +blk.52.attn_qkv.weight q4_K +blk.53.attn_qkv.weight q4_K +blk.54.attn_qkv.weight q4_K blk.55.attn_v.weight q4_K +blk.56.attn_qkv.weight q4_K +blk.57.attn_qkv.weight q4_K +blk.58.attn_qkv.weight q4_K blk.59.attn_v.weight q4_K [IQ2_XS] iq2_xs output.weight q5_K token_embd.weight q2_K +blk.0.attn_qkv.weight q4_K blk.0.ffn_down_exps.weight q2_K blk.0.ffn_down_shexp.weight q2_K +blk.1.attn_qkv.weight q4_K blk.1.ffn_down_exps.weight q2_K blk.1.ffn_down_shexp.weight q2_K +blk.2.attn_qkv.weight q4_K blk.2.ffn_down_exps.weight q2_K blk.2.ffn_down_shexp.weight q2_K blk.3.attn_v.weight q4_K blk.3.ffn_down_exps.weight q2_K +blk.4.attn_qkv.weight q4_K +blk.5.attn_qkv.weight q4_K +blk.6.attn_qkv.weight q4_K blk.7.attn_v.weight q4_K +blk.8.attn_qkv.weight q4_K +blk.9.attn_qkv.weight q4_K +blk.10.attn_qkv.weight q4_K blk.11.attn_v.weight q4_K +blk.12.attn_qkv.weight q4_K +blk.13.attn_qkv.weight q4_K +blk.14.attn_qkv.weight q4_K blk.15.attn_v.weight q4_K +blk.16.attn_qkv.weight q4_K +blk.17.attn_qkv.weight q4_K +blk.18.attn_qkv.weight q4_K blk.19.attn_v.weight q4_K +blk.20.attn_qkv.weight q4_K +blk.21.attn_qkv.weight q4_K +blk.22.attn_qkv.weight q4_K blk.23.attn_v.weight q4_K +blk.24.attn_qkv.weight q4_K +blk.25.attn_qkv.weight q4_K +blk.26.attn_qkv.weight q4_K blk.27.attn_v.weight q4_K +blk.28.attn_qkv.weight q4_K +blk.29.attn_qkv.weight q4_K +blk.30.attn_qkv.weight q4_K blk.31.attn_v.weight q4_K +blk.32.attn_qkv.weight q4_K +blk.33.attn_qkv.weight q4_K +blk.34.attn_qkv.weight q4_K blk.35.attn_v.weight q4_K +blk.36.attn_qkv.weight q4_K +blk.37.attn_qkv.weight q4_K +blk.38.attn_qkv.weight q4_K blk.39.attn_v.weight q4_K +blk.40.attn_qkv.weight q4_K +blk.41.attn_qkv.weight q4_K +blk.42.attn_qkv.weight q4_K blk.43.attn_v.weight q4_K +blk.44.attn_qkv.weight q4_K +blk.45.attn_qkv.weight q4_K +blk.46.attn_qkv.weight q4_K blk.47.attn_v.weight q4_K +blk.48.attn_qkv.weight q4_K +blk.49.attn_qkv.weight q4_K +blk.50.attn_qkv.weight q4_K blk.51.attn_v.weight q4_K +blk.52.attn_qkv.weight q4_K +blk.53.attn_qkv.weight q4_K +blk.54.attn_qkv.weight q4_K blk.55.attn_v.weight q4_K +blk.56.attn_qkv.weight q4_K +blk.57.attn_qkv.weight q4_K +blk.58.attn_qkv.weight q4_K blk.59.attn_v.weight q4_K [Q2_K_S] q2_K output.weight q6_K +blk.0.attn_qkv.weight q4_K blk.0.ffn_down_exps.weight q4_K blk.0.ffn_down_shexp.weight q4_K +blk.1.attn_qkv.weight q4_K blk.1.ffn_down_exps.weight q4_K blk.1.ffn_down_shexp.weight q4_K +blk.2.attn_qkv.weight q4_K blk.2.ffn_down_exps.weight q4_K blk.2.ffn_down_shexp.weight q4_K blk.3.attn_v.weight q4_K blk.3.ffn_down_exps.weight q4_K blk.3.ffn_down_shexp.weight q4_K +blk.4.attn_qkv.weight q4_K blk.4.ffn_down_exps.weight q4_K blk.4.ffn_down_shexp.weight q4_K +blk.5.attn_qkv.weight q4_K blk.5.ffn_down_exps.weight q4_K blk.5.ffn_down_shexp.weight q4_K +blk.6.attn_qkv.weight q4_K blk.6.ffn_down_exps.weight q4_K blk.6.ffn_down_shexp.weight q4_K blk.7.attn_v.weight q4_K +blk.8.attn_qkv.weight q4_K +blk.9.attn_qkv.weight q4_K +blk.10.attn_qkv.weight q4_K blk.11.attn_v.weight q4_K +blk.12.attn_qkv.weight q4_K +blk.13.attn_qkv.weight q4_K +blk.14.attn_qkv.weight q4_K blk.15.attn_v.weight q4_K +blk.16.attn_qkv.weight q4_K +blk.17.attn_qkv.weight q4_K +blk.18.attn_qkv.weight q4_K blk.19.attn_v.weight q4_K +blk.20.attn_qkv.weight q4_K +blk.21.attn_qkv.weight q4_K +blk.22.attn_qkv.weight q4_K blk.23.attn_v.weight q4_K +blk.24.attn_qkv.weight q4_K +blk.25.attn_qkv.weight q4_K +blk.26.attn_qkv.weight q4_K blk.27.attn_v.weight q4_K +blk.28.attn_qkv.weight q4_K +blk.29.attn_qkv.weight q4_K +blk.30.attn_qkv.weight q4_K blk.31.attn_v.weight q4_K +blk.32.attn_qkv.weight q4_K +blk.33.attn_qkv.weight q4_K +blk.34.attn_qkv.weight q4_K blk.35.attn_v.weight q4_K +blk.36.attn_qkv.weight q4_K +blk.37.attn_qkv.weight q4_K +blk.38.attn_qkv.weight q4_K blk.39.attn_v.weight q4_K +blk.40.attn_qkv.weight q4_K +blk.41.attn_qkv.weight q4_K +blk.42.attn_qkv.weight q4_K blk.43.attn_v.weight q4_K +blk.44.attn_qkv.weight q4_K +blk.45.attn_qkv.weight q4_K +blk.46.attn_qkv.weight q4_K blk.47.attn_v.weight q4_K +blk.48.attn_qkv.weight q4_K +blk.49.attn_qkv.weight q4_K +blk.50.attn_qkv.weight q4_K blk.51.attn_v.weight q4_K +blk.52.attn_qkv.weight q4_K +blk.53.attn_qkv.weight q4_K +blk.54.attn_qkv.weight q4_K blk.55.attn_v.weight q4_K +blk.56.attn_qkv.weight q4_K +blk.57.attn_qkv.weight q4_K +blk.58.attn_qkv.weight q4_K blk.59.attn_v.weight q4_K [IQ3_XS] iq3_s output.weight q6_K +blk.0.attn_qkv.weight q4_K +blk.1.attn_qkv.weight q4_K +blk.2.attn_qkv.weight q4_K blk.3.attn_k.weight iq3_xxs blk.3.attn_q.weight iq3_xxs blk.3.attn_v.weight q4_K +blk.4.attn_qkv.weight q4_K +blk.5.attn_qkv.weight q4_K +blk.6.attn_qkv.weight q4_K blk.7.attn_k.weight iq3_xxs blk.7.attn_q.weight iq3_xxs blk.7.attn_v.weight q4_K @@ -929,14 +1067,17 @@ blk.7.ffn_gate_exps.weight iq3_xxs blk.7.ffn_gate_shexp.weight iq3_xxs blk.7.ffn_up_exps.weight iq3_xxs blk.7.ffn_up_shexp.weight iq3_xxs +blk.8.attn_qkv.weight q4_K blk.8.ffn_gate_exps.weight iq3_xxs blk.8.ffn_gate_shexp.weight iq3_xxs blk.8.ffn_up_exps.weight iq3_xxs blk.8.ffn_up_shexp.weight iq3_xxs +blk.9.attn_qkv.weight q4_K blk.9.ffn_gate_exps.weight iq3_xxs blk.9.ffn_gate_shexp.weight iq3_xxs blk.9.ffn_up_exps.weight iq3_xxs blk.9.ffn_up_shexp.weight iq3_xxs +blk.10.attn_qkv.weight q4_K blk.10.ffn_gate_exps.weight iq3_xxs blk.10.ffn_gate_shexp.weight iq3_xxs blk.10.ffn_up_exps.weight iq3_xxs @@ -948,14 +1089,17 @@ blk.11.ffn_gate_exps.weight iq3_xxs blk.11.ffn_gate_shexp.weight iq3_xxs blk.11.ffn_up_exps.weight iq3_xxs blk.11.ffn_up_shexp.weight iq3_xxs +blk.12.attn_qkv.weight q4_K blk.12.ffn_gate_exps.weight iq3_xxs blk.12.ffn_gate_shexp.weight iq3_xxs blk.12.ffn_up_exps.weight iq3_xxs blk.12.ffn_up_shexp.weight iq3_xxs +blk.13.attn_qkv.weight q4_K blk.13.ffn_gate_exps.weight iq3_xxs blk.13.ffn_gate_shexp.weight iq3_xxs blk.13.ffn_up_exps.weight iq3_xxs blk.13.ffn_up_shexp.weight iq3_xxs +blk.14.attn_qkv.weight q4_K blk.14.ffn_gate_exps.weight iq3_xxs blk.14.ffn_gate_shexp.weight iq3_xxs blk.14.ffn_up_exps.weight iq3_xxs @@ -967,14 +1111,17 @@ blk.15.ffn_gate_exps.weight iq3_xxs blk.15.ffn_gate_shexp.weight iq3_xxs blk.15.ffn_up_exps.weight iq3_xxs blk.15.ffn_up_shexp.weight iq3_xxs +blk.16.attn_qkv.weight q4_K blk.16.ffn_gate_exps.weight iq3_xxs blk.16.ffn_gate_shexp.weight iq3_xxs blk.16.ffn_up_exps.weight iq3_xxs blk.16.ffn_up_shexp.weight iq3_xxs +blk.17.attn_qkv.weight q4_K blk.17.ffn_gate_exps.weight iq3_xxs blk.17.ffn_gate_shexp.weight iq3_xxs blk.17.ffn_up_exps.weight iq3_xxs blk.17.ffn_up_shexp.weight iq3_xxs +blk.18.attn_qkv.weight q4_K blk.18.ffn_gate_exps.weight iq3_xxs blk.18.ffn_gate_shexp.weight iq3_xxs blk.18.ffn_up_exps.weight iq3_xxs @@ -986,14 +1133,17 @@ blk.19.ffn_gate_exps.weight iq3_xxs blk.19.ffn_gate_shexp.weight iq3_xxs blk.19.ffn_up_exps.weight iq3_xxs blk.19.ffn_up_shexp.weight iq3_xxs +blk.20.attn_qkv.weight q4_K blk.20.ffn_gate_exps.weight iq3_xxs blk.20.ffn_gate_shexp.weight iq3_xxs blk.20.ffn_up_exps.weight iq3_xxs blk.20.ffn_up_shexp.weight iq3_xxs +blk.21.attn_qkv.weight q4_K blk.21.ffn_gate_exps.weight iq3_xxs blk.21.ffn_gate_shexp.weight iq3_xxs blk.21.ffn_up_exps.weight iq3_xxs blk.21.ffn_up_shexp.weight iq3_xxs +blk.22.attn_qkv.weight q4_K blk.22.ffn_gate_exps.weight iq3_xxs blk.22.ffn_gate_shexp.weight iq3_xxs blk.22.ffn_up_exps.weight iq3_xxs @@ -1005,14 +1155,17 @@ blk.23.ffn_gate_exps.weight iq3_xxs blk.23.ffn_gate_shexp.weight iq3_xxs blk.23.ffn_up_exps.weight iq3_xxs blk.23.ffn_up_shexp.weight iq3_xxs +blk.24.attn_qkv.weight q4_K blk.24.ffn_gate_exps.weight iq3_xxs blk.24.ffn_gate_shexp.weight iq3_xxs blk.24.ffn_up_exps.weight iq3_xxs blk.24.ffn_up_shexp.weight iq3_xxs +blk.25.attn_qkv.weight q4_K blk.25.ffn_gate_exps.weight iq3_xxs blk.25.ffn_gate_shexp.weight iq3_xxs blk.25.ffn_up_exps.weight iq3_xxs blk.25.ffn_up_shexp.weight iq3_xxs +blk.26.attn_qkv.weight q4_K blk.26.ffn_gate_exps.weight iq3_xxs blk.26.ffn_gate_shexp.weight iq3_xxs blk.26.ffn_up_exps.weight iq3_xxs @@ -1024,14 +1177,17 @@ blk.27.ffn_gate_exps.weight iq3_xxs blk.27.ffn_gate_shexp.weight iq3_xxs blk.27.ffn_up_exps.weight iq3_xxs blk.27.ffn_up_shexp.weight iq3_xxs +blk.28.attn_qkv.weight q4_K blk.28.ffn_gate_exps.weight iq3_xxs blk.28.ffn_gate_shexp.weight iq3_xxs blk.28.ffn_up_exps.weight iq3_xxs blk.28.ffn_up_shexp.weight iq3_xxs +blk.29.attn_qkv.weight q4_K blk.29.ffn_gate_exps.weight iq3_xxs blk.29.ffn_gate_shexp.weight iq3_xxs blk.29.ffn_up_exps.weight iq3_xxs blk.29.ffn_up_shexp.weight iq3_xxs +blk.30.attn_qkv.weight q4_K blk.30.ffn_gate_exps.weight iq3_xxs blk.30.ffn_gate_shexp.weight iq3_xxs blk.30.ffn_up_exps.weight iq3_xxs @@ -1043,14 +1199,17 @@ blk.31.ffn_gate_exps.weight iq3_xxs blk.31.ffn_gate_shexp.weight iq3_xxs blk.31.ffn_up_exps.weight iq3_xxs blk.31.ffn_up_shexp.weight iq3_xxs +blk.32.attn_qkv.weight q4_K blk.32.ffn_gate_exps.weight iq3_xxs blk.32.ffn_gate_shexp.weight iq3_xxs blk.32.ffn_up_exps.weight iq3_xxs blk.32.ffn_up_shexp.weight iq3_xxs +blk.33.attn_qkv.weight q4_K blk.33.ffn_gate_exps.weight iq3_xxs blk.33.ffn_gate_shexp.weight iq3_xxs blk.33.ffn_up_exps.weight iq3_xxs blk.33.ffn_up_shexp.weight iq3_xxs +blk.34.attn_qkv.weight q4_K blk.34.ffn_gate_exps.weight iq3_xxs blk.34.ffn_gate_shexp.weight iq3_xxs blk.34.ffn_up_exps.weight iq3_xxs @@ -1062,14 +1221,17 @@ blk.35.ffn_gate_exps.weight iq3_xxs blk.35.ffn_gate_shexp.weight iq3_xxs blk.35.ffn_up_exps.weight iq3_xxs blk.35.ffn_up_shexp.weight iq3_xxs +blk.36.attn_qkv.weight q4_K blk.36.ffn_gate_exps.weight iq3_xxs blk.36.ffn_gate_shexp.weight iq3_xxs blk.36.ffn_up_exps.weight iq3_xxs blk.36.ffn_up_shexp.weight iq3_xxs +blk.37.attn_qkv.weight q4_K blk.37.ffn_gate_exps.weight iq3_xxs blk.37.ffn_gate_shexp.weight iq3_xxs blk.37.ffn_up_exps.weight iq3_xxs blk.37.ffn_up_shexp.weight iq3_xxs +blk.38.attn_qkv.weight q4_K blk.38.ffn_gate_exps.weight iq3_xxs blk.38.ffn_gate_shexp.weight iq3_xxs blk.38.ffn_up_exps.weight iq3_xxs @@ -1081,14 +1243,17 @@ blk.39.ffn_gate_exps.weight iq3_xxs blk.39.ffn_gate_shexp.weight iq3_xxs blk.39.ffn_up_exps.weight iq3_xxs blk.39.ffn_up_shexp.weight iq3_xxs +blk.40.attn_qkv.weight q4_K blk.40.ffn_gate_exps.weight iq3_xxs blk.40.ffn_gate_shexp.weight iq3_xxs blk.40.ffn_up_exps.weight iq3_xxs blk.40.ffn_up_shexp.weight iq3_xxs +blk.41.attn_qkv.weight q4_K blk.41.ffn_gate_exps.weight iq3_xxs blk.41.ffn_gate_shexp.weight iq3_xxs blk.41.ffn_up_exps.weight iq3_xxs blk.41.ffn_up_shexp.weight iq3_xxs +blk.42.attn_qkv.weight q4_K blk.42.ffn_gate_exps.weight iq3_xxs blk.42.ffn_gate_shexp.weight iq3_xxs blk.42.ffn_up_exps.weight iq3_xxs @@ -1100,14 +1265,17 @@ blk.43.ffn_gate_exps.weight iq3_xxs blk.43.ffn_gate_shexp.weight iq3_xxs blk.43.ffn_up_exps.weight iq3_xxs blk.43.ffn_up_shexp.weight iq3_xxs +blk.44.attn_qkv.weight q4_K blk.44.ffn_gate_exps.weight iq3_xxs blk.44.ffn_gate_shexp.weight iq3_xxs blk.44.ffn_up_exps.weight iq3_xxs blk.44.ffn_up_shexp.weight iq3_xxs +blk.45.attn_qkv.weight q4_K blk.45.ffn_gate_exps.weight iq3_xxs blk.45.ffn_gate_shexp.weight iq3_xxs blk.45.ffn_up_exps.weight iq3_xxs blk.45.ffn_up_shexp.weight iq3_xxs +blk.46.attn_qkv.weight q4_K blk.46.ffn_gate_exps.weight iq3_xxs blk.46.ffn_gate_shexp.weight iq3_xxs blk.46.ffn_up_exps.weight iq3_xxs @@ -1119,14 +1287,17 @@ blk.47.ffn_gate_exps.weight iq3_xxs blk.47.ffn_gate_shexp.weight iq3_xxs blk.47.ffn_up_exps.weight iq3_xxs blk.47.ffn_up_shexp.weight iq3_xxs +blk.48.attn_qkv.weight q4_K blk.48.ffn_gate_exps.weight iq3_xxs blk.48.ffn_gate_shexp.weight iq3_xxs blk.48.ffn_up_exps.weight iq3_xxs blk.48.ffn_up_shexp.weight iq3_xxs +blk.49.attn_qkv.weight q4_K blk.49.ffn_gate_exps.weight iq3_xxs blk.49.ffn_gate_shexp.weight iq3_xxs blk.49.ffn_up_exps.weight iq3_xxs blk.49.ffn_up_shexp.weight iq3_xxs +blk.50.attn_qkv.weight q4_K blk.50.ffn_gate_exps.weight iq3_xxs blk.50.ffn_gate_shexp.weight iq3_xxs blk.50.ffn_up_exps.weight iq3_xxs @@ -1138,9 +1309,15 @@ blk.51.ffn_gate_exps.weight iq3_xxs blk.51.ffn_gate_shexp.weight iq3_xxs blk.51.ffn_up_exps.weight iq3_xxs blk.51.ffn_up_shexp.weight iq3_xxs +blk.52.attn_qkv.weight q4_K +blk.53.attn_qkv.weight q4_K +blk.54.attn_qkv.weight q4_K blk.55.attn_k.weight iq3_xxs blk.55.attn_q.weight iq3_xxs blk.55.attn_v.weight q4_K +blk.56.attn_qkv.weight q4_K +blk.57.attn_qkv.weight q4_K +blk.58.attn_qkv.weight q4_K blk.59.attn_k.weight iq3_xxs blk.59.attn_q.weight iq3_xxs blk.59.attn_v.weight q4_K @@ -1148,10 +1325,13 @@ blk.59.attn_v.weight q4_K [IQ3_XXS] iq3_xxs output.weight q5_K token_embd.weight iq3_s +blk.0.attn_qkv.weight q4_K blk.0.ffn_down_exps.weight q4_K blk.0.ffn_down_shexp.weight q4_K +blk.1.attn_qkv.weight q4_K blk.1.ffn_down_exps.weight q4_K blk.1.ffn_down_shexp.weight q4_K +blk.2.attn_qkv.weight q4_K blk.2.ffn_down_exps.weight q4_K blk.2.ffn_down_shexp.weight q4_K blk.3.attn_k.weight iq2_s @@ -1160,10 +1340,13 @@ blk.3.attn_q.weight iq2_s blk.3.attn_v.weight q4_K blk.3.ffn_down_exps.weight q4_K blk.3.ffn_down_shexp.weight q4_K +blk.4.attn_qkv.weight q4_K blk.4.ffn_down_exps.weight q4_K blk.4.ffn_down_shexp.weight q4_K +blk.5.attn_qkv.weight q4_K blk.5.ffn_down_exps.weight q4_K blk.5.ffn_down_shexp.weight q4_K +blk.6.attn_qkv.weight q4_K blk.6.ffn_down_exps.weight q4_K blk.6.ffn_down_shexp.weight q4_K blk.7.attn_k.weight iq2_s @@ -1172,10 +1355,13 @@ blk.7.attn_q.weight iq2_s blk.7.attn_v.weight q4_K blk.7.ffn_down_exps.weight q3_K blk.7.ffn_down_shexp.weight q3_K +blk.8.attn_qkv.weight q4_K blk.8.ffn_down_exps.weight q3_K blk.8.ffn_down_shexp.weight q3_K +blk.9.attn_qkv.weight q4_K blk.9.ffn_down_exps.weight q3_K blk.9.ffn_down_shexp.weight q3_K +blk.10.attn_qkv.weight q4_K blk.10.ffn_down_exps.weight q3_K blk.10.ffn_down_shexp.weight q3_K blk.11.attn_k.weight iq2_s @@ -1184,10 +1370,13 @@ blk.11.attn_q.weight iq2_s blk.11.attn_v.weight q4_K blk.11.ffn_down_exps.weight q3_K blk.11.ffn_down_shexp.weight q3_K +blk.12.attn_qkv.weight q4_K blk.12.ffn_down_exps.weight q3_K blk.12.ffn_down_shexp.weight q3_K +blk.13.attn_qkv.weight q4_K blk.13.ffn_down_exps.weight q3_K blk.13.ffn_down_shexp.weight q3_K +blk.14.attn_qkv.weight q4_K blk.14.ffn_down_exps.weight q3_K blk.14.ffn_down_shexp.weight q3_K blk.15.attn_k.weight iq2_s @@ -1196,10 +1385,13 @@ blk.15.attn_q.weight iq2_s blk.15.attn_v.weight q4_K blk.15.ffn_down_exps.weight q3_K blk.15.ffn_down_shexp.weight q3_K +blk.16.attn_qkv.weight q4_K blk.16.ffn_down_exps.weight q3_K blk.16.ffn_down_shexp.weight q3_K +blk.17.attn_qkv.weight q4_K blk.17.ffn_down_exps.weight q3_K blk.17.ffn_down_shexp.weight q3_K +blk.18.attn_qkv.weight q4_K blk.18.ffn_down_exps.weight q3_K blk.18.ffn_down_shexp.weight q3_K blk.19.attn_k.weight iq2_s @@ -1208,10 +1400,13 @@ blk.19.attn_q.weight iq2_s blk.19.attn_v.weight q4_K blk.19.ffn_down_exps.weight q3_K blk.19.ffn_down_shexp.weight q3_K +blk.20.attn_qkv.weight q4_K blk.20.ffn_down_exps.weight q3_K blk.20.ffn_down_shexp.weight q3_K +blk.21.attn_qkv.weight q4_K blk.21.ffn_down_exps.weight q3_K blk.21.ffn_down_shexp.weight q3_K +blk.22.attn_qkv.weight q4_K blk.22.ffn_down_exps.weight q3_K blk.22.ffn_down_shexp.weight q3_K blk.23.attn_k.weight iq2_s @@ -1220,10 +1415,13 @@ blk.23.attn_q.weight iq2_s blk.23.attn_v.weight q4_K blk.23.ffn_down_exps.weight q3_K blk.23.ffn_down_shexp.weight q3_K +blk.24.attn_qkv.weight q4_K blk.24.ffn_down_exps.weight q3_K blk.24.ffn_down_shexp.weight q3_K +blk.25.attn_qkv.weight q4_K blk.25.ffn_down_exps.weight q3_K blk.25.ffn_down_shexp.weight q3_K +blk.26.attn_qkv.weight q4_K blk.26.ffn_down_exps.weight q3_K blk.26.ffn_down_shexp.weight q3_K blk.27.attn_k.weight iq2_s @@ -1232,10 +1430,13 @@ blk.27.attn_q.weight iq2_s blk.27.attn_v.weight q4_K blk.27.ffn_down_exps.weight q3_K blk.27.ffn_down_shexp.weight q3_K +blk.28.attn_qkv.weight q4_K blk.28.ffn_down_exps.weight q3_K blk.28.ffn_down_shexp.weight q3_K +blk.29.attn_qkv.weight q4_K blk.29.ffn_down_exps.weight q3_K blk.29.ffn_down_shexp.weight q3_K +blk.30.attn_qkv.weight q4_K blk.30.ffn_down_exps.weight q3_K blk.30.ffn_down_shexp.weight q3_K blk.31.attn_k.weight iq2_s @@ -1244,10 +1445,13 @@ blk.31.attn_q.weight iq2_s blk.31.attn_v.weight q4_K blk.31.ffn_down_exps.weight q3_K blk.31.ffn_down_shexp.weight q3_K +blk.32.attn_qkv.weight q4_K blk.32.ffn_down_exps.weight q3_K blk.32.ffn_down_shexp.weight q3_K +blk.33.attn_qkv.weight q4_K blk.33.ffn_down_exps.weight q3_K blk.33.ffn_down_shexp.weight q3_K +blk.34.attn_qkv.weight q4_K blk.34.ffn_down_exps.weight q3_K blk.34.ffn_down_shexp.weight q3_K blk.35.attn_k.weight iq2_s @@ -1256,10 +1460,13 @@ blk.35.attn_q.weight iq2_s blk.35.attn_v.weight q4_K blk.35.ffn_down_exps.weight q3_K blk.35.ffn_down_shexp.weight q3_K +blk.36.attn_qkv.weight q4_K blk.36.ffn_down_exps.weight q3_K blk.36.ffn_down_shexp.weight q3_K +blk.37.attn_qkv.weight q4_K blk.37.ffn_down_exps.weight q3_K blk.37.ffn_down_shexp.weight q3_K +blk.38.attn_qkv.weight q4_K blk.38.ffn_down_exps.weight q3_K blk.38.ffn_down_shexp.weight q3_K blk.39.attn_k.weight iq2_s @@ -1268,10 +1475,13 @@ blk.39.attn_q.weight iq2_s blk.39.attn_v.weight q4_K blk.39.ffn_down_exps.weight q3_K blk.39.ffn_down_shexp.weight q3_K +blk.40.attn_qkv.weight q4_K blk.40.ffn_down_exps.weight q3_K blk.40.ffn_down_shexp.weight q3_K +blk.41.attn_qkv.weight q4_K blk.41.ffn_down_exps.weight q3_K blk.41.ffn_down_shexp.weight q3_K +blk.42.attn_qkv.weight q4_K blk.42.ffn_down_exps.weight q3_K blk.42.ffn_down_shexp.weight q3_K blk.43.attn_k.weight iq2_s @@ -1280,10 +1490,13 @@ blk.43.attn_q.weight iq2_s blk.43.attn_v.weight q4_K blk.43.ffn_down_exps.weight q3_K blk.43.ffn_down_shexp.weight q3_K +blk.44.attn_qkv.weight q4_K blk.44.ffn_down_exps.weight q3_K blk.44.ffn_down_shexp.weight q3_K +blk.45.attn_qkv.weight q4_K blk.45.ffn_down_exps.weight q3_K blk.45.ffn_down_shexp.weight q3_K +blk.46.attn_qkv.weight q4_K blk.46.ffn_down_exps.weight q3_K blk.46.ffn_down_shexp.weight q3_K blk.47.attn_k.weight iq2_s @@ -1292,10 +1505,13 @@ blk.47.attn_q.weight iq2_s blk.47.attn_v.weight q4_K blk.47.ffn_down_exps.weight q3_K blk.47.ffn_down_shexp.weight q3_K +blk.48.attn_qkv.weight q4_K blk.48.ffn_down_exps.weight q3_K blk.48.ffn_down_shexp.weight q3_K +blk.49.attn_qkv.weight q4_K blk.49.ffn_down_exps.weight q3_K blk.49.ffn_down_shexp.weight q3_K +blk.50.attn_qkv.weight q4_K blk.50.ffn_down_exps.weight q3_K blk.50.ffn_down_shexp.weight q3_K blk.51.attn_k.weight iq2_s @@ -1304,10 +1520,13 @@ blk.51.attn_q.weight iq2_s blk.51.attn_v.weight q4_K blk.51.ffn_down_exps.weight q3_K blk.51.ffn_down_shexp.weight q3_K +blk.52.attn_qkv.weight q4_K blk.52.ffn_down_exps.weight q3_K blk.52.ffn_down_shexp.weight q3_K +blk.53.attn_qkv.weight q4_K blk.53.ffn_down_exps.weight q3_K blk.53.ffn_down_shexp.weight q3_K +blk.54.attn_qkv.weight q4_K blk.54.ffn_down_exps.weight q3_K blk.54.ffn_down_shexp.weight q3_K blk.55.attn_k.weight iq2_s @@ -1316,10 +1535,13 @@ blk.55.attn_q.weight iq2_s blk.55.attn_v.weight q4_K blk.55.ffn_down_exps.weight q3_K blk.55.ffn_down_shexp.weight q3_K +blk.56.attn_qkv.weight q4_K blk.56.ffn_down_exps.weight q3_K blk.56.ffn_down_shexp.weight q3_K +blk.57.attn_qkv.weight q4_K blk.57.ffn_down_exps.weight q3_K blk.57.ffn_down_shexp.weight q3_K +blk.58.attn_qkv.weight q4_K blk.58.ffn_down_exps.weight q3_K blk.58.ffn_down_shexp.weight q3_K blk.59.attn_k.weight iq2_s @@ -1332,92 +1554,227 @@ blk.59.ffn_down_shexp.weight q3_K [IQ1_S] iq1_s output.weight q5_K token_embd.weight q2_K +blk.0.attn_qkv.weight q4_K blk.0.ffn_down_exps.weight q2_K blk.0.ffn_down_shexp.weight q2_K +blk.1.attn_qkv.weight q4_K blk.1.ffn_down_exps.weight q2_K blk.1.ffn_down_shexp.weight q2_K +blk.2.attn_qkv.weight q4_K blk.2.ffn_down_exps.weight q2_K blk.2.ffn_down_shexp.weight q2_K blk.3.attn_output.weight iq2_xxs blk.3.attn_v.weight q4_K blk.3.ffn_down_exps.weight q2_K +blk.4.attn_qkv.weight q4_K +blk.5.attn_qkv.weight q4_K +blk.6.attn_qkv.weight q4_K blk.7.attn_output.weight iq2_xxs blk.7.attn_v.weight q4_K +blk.8.attn_qkv.weight q4_K +blk.9.attn_qkv.weight q4_K +blk.10.attn_qkv.weight q4_K blk.11.attn_output.weight iq2_xxs blk.11.attn_v.weight q4_K +blk.12.attn_qkv.weight q4_K +blk.13.attn_qkv.weight q4_K +blk.14.attn_qkv.weight q4_K blk.15.attn_output.weight iq2_xxs blk.15.attn_v.weight q4_K +blk.16.attn_qkv.weight q4_K +blk.17.attn_qkv.weight q4_K +blk.18.attn_qkv.weight q4_K blk.19.attn_output.weight iq2_xxs blk.19.attn_v.weight q4_K +blk.20.attn_qkv.weight q4_K +blk.21.attn_qkv.weight q4_K +blk.22.attn_qkv.weight q4_K blk.23.attn_output.weight iq2_xxs blk.23.attn_v.weight q4_K +blk.24.attn_qkv.weight q4_K +blk.25.attn_qkv.weight q4_K +blk.26.attn_qkv.weight q4_K blk.27.attn_output.weight iq2_xxs blk.27.attn_v.weight q4_K +blk.28.attn_qkv.weight q4_K +blk.29.attn_qkv.weight q4_K +blk.30.attn_qkv.weight q4_K blk.31.attn_output.weight iq2_xxs blk.31.attn_v.weight q4_K +blk.32.attn_qkv.weight q4_K +blk.33.attn_qkv.weight q4_K +blk.34.attn_qkv.weight q4_K blk.35.attn_output.weight iq2_xxs blk.35.attn_v.weight q4_K +blk.36.attn_qkv.weight q4_K +blk.37.attn_qkv.weight q4_K +blk.38.attn_qkv.weight q4_K blk.39.attn_output.weight iq2_xxs blk.39.attn_v.weight q4_K +blk.40.attn_qkv.weight q4_K +blk.41.attn_qkv.weight q4_K +blk.42.attn_qkv.weight q4_K blk.43.attn_output.weight iq2_xxs blk.43.attn_v.weight q4_K +blk.44.attn_qkv.weight q4_K +blk.45.attn_qkv.weight q4_K +blk.46.attn_qkv.weight q4_K blk.47.attn_output.weight iq2_xxs blk.47.attn_v.weight q4_K +blk.48.attn_qkv.weight q4_K +blk.49.attn_qkv.weight q4_K +blk.50.attn_qkv.weight q4_K blk.51.attn_output.weight iq2_xxs blk.51.attn_v.weight q4_K +blk.52.attn_qkv.weight q4_K +blk.53.attn_qkv.weight q4_K +blk.54.attn_qkv.weight q4_K blk.55.attn_output.weight iq2_xxs blk.55.attn_v.weight q4_K +blk.56.attn_qkv.weight q4_K +blk.57.attn_qkv.weight q4_K +blk.58.attn_qkv.weight q4_K blk.59.attn_output.weight iq2_xxs blk.59.attn_v.weight q4_K [IQ4_NL] iq4_nl output.weight q6_K +blk.0.attn_qkv.weight q5_K blk.0.ffn_down_exps.weight q5_K blk.0.ffn_down_shexp.weight q5_K +blk.1.attn_qkv.weight q5_K blk.1.ffn_down_exps.weight q5_K blk.1.ffn_down_shexp.weight q5_K +blk.2.attn_qkv.weight q5_K blk.2.ffn_down_exps.weight q5_K blk.2.ffn_down_shexp.weight q5_K blk.3.attn_v.weight q5_K blk.3.ffn_down_exps.weight q5_K blk.3.ffn_down_shexp.weight q5_K +blk.4.attn_qkv.weight q5_K blk.4.ffn_down_exps.weight q5_K blk.4.ffn_down_shexp.weight q5_K +blk.5.attn_qkv.weight q5_K blk.5.ffn_down_exps.weight q5_K blk.5.ffn_down_shexp.weight q5_K +blk.6.attn_qkv.weight q5_K blk.6.ffn_down_exps.weight q5_K blk.6.ffn_down_shexp.weight q5_K blk.7.attn_v.weight q5_K +blk.8.attn_qkv.weight q5_K +blk.9.attn_qkv.weight q5_K +blk.10.attn_qkv.weight q5_K blk.11.attn_v.weight q5_K +blk.12.attn_qkv.weight q5_K +blk.13.attn_qkv.weight q5_K +blk.14.attn_qkv.weight q5_K blk.15.attn_v.weight q5_K +blk.16.attn_qkv.weight q5_K +blk.17.attn_qkv.weight q5_K +blk.18.attn_qkv.weight q5_K blk.19.attn_v.weight q5_K +blk.20.attn_qkv.weight q5_K +blk.21.attn_qkv.weight q5_K +blk.22.attn_qkv.weight q5_K blk.23.attn_v.weight q5_K +blk.24.attn_qkv.weight q5_K +blk.25.attn_qkv.weight q5_K +blk.26.attn_qkv.weight q5_K blk.27.attn_v.weight q5_K +blk.28.attn_qkv.weight q5_K +blk.29.attn_qkv.weight q5_K +blk.30.attn_qkv.weight q5_K blk.31.attn_v.weight q5_K +blk.32.attn_qkv.weight q5_K +blk.33.attn_qkv.weight q5_K +blk.34.attn_qkv.weight q5_K blk.35.attn_v.weight q5_K +blk.36.attn_qkv.weight q5_K +blk.37.attn_qkv.weight q5_K +blk.38.attn_qkv.weight q5_K blk.39.attn_v.weight q5_K +blk.40.attn_qkv.weight q5_K +blk.41.attn_qkv.weight q5_K +blk.42.attn_qkv.weight q5_K blk.43.attn_v.weight q5_K +blk.44.attn_qkv.weight q5_K +blk.45.attn_qkv.weight q5_K +blk.46.attn_qkv.weight q5_K blk.47.attn_v.weight q5_K +blk.48.attn_qkv.weight q5_K +blk.49.attn_qkv.weight q5_K +blk.50.attn_qkv.weight q5_K blk.51.attn_v.weight q5_K +blk.52.attn_qkv.weight q5_K +blk.53.attn_qkv.weight q5_K +blk.54.attn_qkv.weight q5_K blk.55.attn_v.weight q5_K +blk.56.attn_qkv.weight q5_K +blk.57.attn_qkv.weight q5_K +blk.58.attn_qkv.weight q5_K blk.59.attn_v.weight q5_K [IQ3_S] iq3_s output.weight q6_K +blk.0.attn_qkv.weight q4_K +blk.1.attn_qkv.weight q4_K +blk.2.attn_qkv.weight q4_K blk.3.attn_v.weight q4_K +blk.4.attn_qkv.weight q4_K +blk.5.attn_qkv.weight q4_K +blk.6.attn_qkv.weight q4_K blk.7.attn_v.weight q4_K +blk.8.attn_qkv.weight q4_K +blk.9.attn_qkv.weight q4_K +blk.10.attn_qkv.weight q4_K blk.11.attn_v.weight q4_K +blk.12.attn_qkv.weight q4_K +blk.13.attn_qkv.weight q4_K +blk.14.attn_qkv.weight q4_K blk.15.attn_v.weight q4_K +blk.16.attn_qkv.weight q4_K +blk.17.attn_qkv.weight q4_K +blk.18.attn_qkv.weight q4_K blk.19.attn_v.weight q4_K +blk.20.attn_qkv.weight q4_K +blk.21.attn_qkv.weight q4_K +blk.22.attn_qkv.weight q4_K blk.23.attn_v.weight q4_K +blk.24.attn_qkv.weight q4_K +blk.25.attn_qkv.weight q4_K +blk.26.attn_qkv.weight q4_K blk.27.attn_v.weight q4_K +blk.28.attn_qkv.weight q4_K +blk.29.attn_qkv.weight q4_K +blk.30.attn_qkv.weight q4_K blk.31.attn_v.weight q4_K +blk.32.attn_qkv.weight q4_K +blk.33.attn_qkv.weight q4_K +blk.34.attn_qkv.weight q4_K blk.35.attn_v.weight q4_K +blk.36.attn_qkv.weight q4_K +blk.37.attn_qkv.weight q4_K +blk.38.attn_qkv.weight q4_K blk.39.attn_v.weight q4_K +blk.40.attn_qkv.weight q4_K +blk.41.attn_qkv.weight q4_K +blk.42.attn_qkv.weight q4_K blk.43.attn_v.weight q4_K +blk.44.attn_qkv.weight q4_K +blk.45.attn_qkv.weight q4_K +blk.46.attn_qkv.weight q4_K blk.47.attn_v.weight q4_K +blk.48.attn_qkv.weight q4_K +blk.49.attn_qkv.weight q4_K +blk.50.attn_qkv.weight q4_K blk.51.attn_v.weight q4_K +blk.52.attn_qkv.weight q4_K +blk.53.attn_qkv.weight q4_K +blk.54.attn_qkv.weight q4_K blk.55.attn_v.weight q4_K +blk.56.attn_qkv.weight q4_K +blk.57.attn_qkv.weight q4_K +blk.58.attn_qkv.weight q4_K blk.59.attn_v.weight q4_K [IQ3_M] iq3_s @@ -1515,155 +1872,335 @@ blk.59.attn_v.weight q4_K [IQ2_S] iq2_xs output.weight q5_K token_embd.weight iq3_s +blk.0.attn_qkv.weight q4_K blk.0.ffn_down_exps.weight iq3_s blk.0.ffn_down_shexp.weight iq3_s +blk.1.attn_qkv.weight q4_K blk.1.ffn_down_exps.weight iq3_s blk.1.ffn_down_shexp.weight iq3_s +blk.2.attn_qkv.weight q4_K blk.2.ffn_down_exps.weight iq3_s blk.2.ffn_down_shexp.weight iq3_s blk.3.attn_output.weight iq3_s blk.3.attn_v.weight q4_K blk.3.ffn_down_exps.weight iq3_s +blk.4.attn_qkv.weight q4_K +blk.5.attn_qkv.weight q4_K +blk.6.attn_qkv.weight q4_K blk.7.attn_output.weight iq3_s blk.7.attn_v.weight q4_K +blk.8.attn_qkv.weight q4_K +blk.9.attn_qkv.weight q4_K +blk.10.attn_qkv.weight q4_K blk.11.attn_output.weight iq3_s blk.11.attn_v.weight q4_K +blk.12.attn_qkv.weight q4_K +blk.13.attn_qkv.weight q4_K +blk.14.attn_qkv.weight q4_K blk.15.attn_output.weight iq3_s blk.15.attn_v.weight q4_K +blk.16.attn_qkv.weight q4_K +blk.17.attn_qkv.weight q4_K +blk.18.attn_qkv.weight q4_K blk.19.attn_output.weight iq3_s blk.19.attn_v.weight q4_K +blk.20.attn_qkv.weight q4_K +blk.21.attn_qkv.weight q4_K +blk.22.attn_qkv.weight q4_K blk.23.attn_output.weight iq3_s blk.23.attn_v.weight q4_K +blk.24.attn_qkv.weight q4_K +blk.25.attn_qkv.weight q4_K +blk.26.attn_qkv.weight q4_K blk.27.attn_output.weight iq3_s blk.27.attn_v.weight q4_K +blk.28.attn_qkv.weight q4_K +blk.29.attn_qkv.weight q4_K +blk.30.attn_qkv.weight q4_K blk.31.attn_output.weight iq3_s blk.31.attn_v.weight q4_K +blk.32.attn_qkv.weight q4_K +blk.33.attn_qkv.weight q4_K +blk.34.attn_qkv.weight q4_K blk.35.attn_output.weight iq3_s blk.35.attn_v.weight q4_K +blk.36.attn_qkv.weight q4_K +blk.37.attn_qkv.weight q4_K +blk.38.attn_qkv.weight q4_K blk.39.attn_output.weight iq3_s blk.39.attn_v.weight q4_K +blk.40.attn_qkv.weight q4_K +blk.41.attn_qkv.weight q4_K +blk.42.attn_qkv.weight q4_K blk.43.attn_output.weight iq3_s blk.43.attn_v.weight q4_K +blk.44.attn_qkv.weight q4_K +blk.45.attn_qkv.weight q4_K +blk.46.attn_qkv.weight q4_K blk.47.attn_output.weight iq3_s blk.47.attn_v.weight q4_K +blk.48.attn_qkv.weight q4_K +blk.49.attn_qkv.weight q4_K +blk.50.attn_qkv.weight q4_K blk.51.attn_output.weight iq3_s blk.51.attn_v.weight q4_K +blk.52.attn_qkv.weight q4_K +blk.53.attn_qkv.weight q4_K +blk.54.attn_qkv.weight q4_K blk.55.attn_output.weight iq3_s blk.55.attn_v.weight q4_K +blk.56.attn_qkv.weight q4_K +blk.57.attn_qkv.weight q4_K +blk.58.attn_qkv.weight q4_K blk.59.attn_output.weight iq3_s blk.59.attn_v.weight q4_K [IQ2_M] iq2_s output.weight q5_K token_embd.weight iq3_s +blk.0.attn_qkv.weight q4_K blk.0.ffn_down_exps.weight iq3_s blk.0.ffn_down_shexp.weight iq3_s +blk.1.attn_qkv.weight q4_K blk.1.ffn_down_exps.weight iq3_s blk.1.ffn_down_shexp.weight iq3_s +blk.2.attn_qkv.weight q4_K blk.2.ffn_down_exps.weight iq3_s blk.2.ffn_down_shexp.weight iq3_s blk.3.attn_output.weight iq3_s blk.3.attn_v.weight q4_K blk.3.ffn_down_exps.weight iq3_s +blk.4.attn_qkv.weight q4_K +blk.5.attn_qkv.weight q4_K +blk.6.attn_qkv.weight q4_K blk.7.attn_output.weight iq3_s blk.7.attn_v.weight q4_K +blk.8.attn_qkv.weight q4_K +blk.9.attn_qkv.weight q4_K +blk.10.attn_qkv.weight q4_K blk.11.attn_output.weight iq3_s blk.11.attn_v.weight q4_K +blk.12.attn_qkv.weight q4_K +blk.13.attn_qkv.weight q4_K +blk.14.attn_qkv.weight q4_K blk.15.attn_output.weight iq3_s blk.15.attn_v.weight q4_K +blk.16.attn_qkv.weight q4_K +blk.17.attn_qkv.weight q4_K +blk.18.attn_qkv.weight q4_K blk.19.attn_output.weight iq3_s blk.19.attn_v.weight q4_K +blk.20.attn_qkv.weight q4_K +blk.21.attn_qkv.weight q4_K +blk.22.attn_qkv.weight q4_K blk.23.attn_output.weight iq3_s blk.23.attn_v.weight q4_K +blk.24.attn_qkv.weight q4_K +blk.25.attn_qkv.weight q4_K +blk.26.attn_qkv.weight q4_K blk.27.attn_output.weight iq3_s blk.27.attn_v.weight q4_K +blk.28.attn_qkv.weight q4_K +blk.29.attn_qkv.weight q4_K +blk.30.attn_qkv.weight q4_K blk.31.attn_output.weight iq3_s blk.31.attn_v.weight q4_K +blk.32.attn_qkv.weight q4_K +blk.33.attn_qkv.weight q4_K +blk.34.attn_qkv.weight q4_K blk.35.attn_output.weight iq3_s blk.35.attn_v.weight q4_K +blk.36.attn_qkv.weight q4_K +blk.37.attn_qkv.weight q4_K +blk.38.attn_qkv.weight q4_K blk.39.attn_output.weight iq3_s blk.39.attn_v.weight q4_K +blk.40.attn_qkv.weight q4_K +blk.41.attn_qkv.weight q4_K +blk.42.attn_qkv.weight q4_K blk.43.attn_output.weight iq3_s blk.43.attn_v.weight q4_K +blk.44.attn_qkv.weight q4_K +blk.45.attn_qkv.weight q4_K +blk.46.attn_qkv.weight q4_K blk.47.attn_output.weight iq3_s blk.47.attn_v.weight q4_K +blk.48.attn_qkv.weight q4_K +blk.49.attn_qkv.weight q4_K +blk.50.attn_qkv.weight q4_K blk.51.attn_output.weight iq3_s blk.51.attn_v.weight q4_K +blk.52.attn_qkv.weight q4_K +blk.53.attn_qkv.weight q4_K +blk.54.attn_qkv.weight q4_K blk.55.attn_output.weight iq3_s blk.55.attn_v.weight q4_K +blk.56.attn_qkv.weight q4_K +blk.57.attn_qkv.weight q4_K +blk.58.attn_qkv.weight q4_K blk.59.attn_output.weight iq3_s blk.59.attn_v.weight q4_K [IQ4_XS] iq4_xs output.weight q6_K +blk.0.attn_qkv.weight q5_K blk.0.ffn_down_exps.weight q5_K blk.0.ffn_down_shexp.weight q5_K +blk.1.attn_qkv.weight q5_K blk.1.ffn_down_exps.weight q5_K blk.1.ffn_down_shexp.weight q5_K +blk.2.attn_qkv.weight q5_K blk.2.ffn_down_exps.weight q5_K blk.2.ffn_down_shexp.weight q5_K blk.3.attn_v.weight q5_K blk.3.ffn_down_exps.weight q5_K blk.3.ffn_down_shexp.weight q5_K +blk.4.attn_qkv.weight q5_K blk.4.ffn_down_exps.weight q5_K blk.4.ffn_down_shexp.weight q5_K +blk.5.attn_qkv.weight q5_K blk.5.ffn_down_exps.weight q5_K blk.5.ffn_down_shexp.weight q5_K +blk.6.attn_qkv.weight q5_K blk.6.ffn_down_exps.weight q5_K blk.6.ffn_down_shexp.weight q5_K blk.7.attn_v.weight q5_K +blk.8.attn_qkv.weight q5_K +blk.9.attn_qkv.weight q5_K +blk.10.attn_qkv.weight q5_K blk.11.attn_v.weight q5_K +blk.12.attn_qkv.weight q5_K +blk.13.attn_qkv.weight q5_K +blk.14.attn_qkv.weight q5_K blk.15.attn_v.weight q5_K +blk.16.attn_qkv.weight q5_K +blk.17.attn_qkv.weight q5_K +blk.18.attn_qkv.weight q5_K blk.19.attn_v.weight q5_K +blk.20.attn_qkv.weight q5_K +blk.21.attn_qkv.weight q5_K +blk.22.attn_qkv.weight q5_K blk.23.attn_v.weight q5_K +blk.24.attn_qkv.weight q5_K +blk.25.attn_qkv.weight q5_K +blk.26.attn_qkv.weight q5_K blk.27.attn_v.weight q5_K +blk.28.attn_qkv.weight q5_K +blk.29.attn_qkv.weight q5_K +blk.30.attn_qkv.weight q5_K blk.31.attn_v.weight q5_K +blk.32.attn_qkv.weight q5_K +blk.33.attn_qkv.weight q5_K +blk.34.attn_qkv.weight q5_K blk.35.attn_v.weight q5_K +blk.36.attn_qkv.weight q5_K +blk.37.attn_qkv.weight q5_K +blk.38.attn_qkv.weight q5_K blk.39.attn_v.weight q5_K +blk.40.attn_qkv.weight q5_K +blk.41.attn_qkv.weight q5_K +blk.42.attn_qkv.weight q5_K blk.43.attn_v.weight q5_K +blk.44.attn_qkv.weight q5_K +blk.45.attn_qkv.weight q5_K +blk.46.attn_qkv.weight q5_K blk.47.attn_v.weight q5_K +blk.48.attn_qkv.weight q5_K +blk.49.attn_qkv.weight q5_K +blk.50.attn_qkv.weight q5_K blk.51.attn_v.weight q5_K +blk.52.attn_qkv.weight q5_K +blk.53.attn_qkv.weight q5_K +blk.54.attn_qkv.weight q5_K blk.55.attn_v.weight q5_K +blk.56.attn_qkv.weight q5_K +blk.57.attn_qkv.weight q5_K +blk.58.attn_qkv.weight q5_K blk.59.attn_v.weight q5_K [IQ1_M] iq1_m output.weight q5_K token_embd.weight q2_K +blk.0.attn_qkv.weight q4_K blk.0.ffn_down_exps.weight q2_K blk.0.ffn_down_shexp.weight q2_K +blk.1.attn_qkv.weight q4_K blk.1.ffn_down_exps.weight q2_K blk.1.ffn_down_shexp.weight q2_K +blk.2.attn_qkv.weight q4_K blk.2.ffn_down_exps.weight q2_K blk.2.ffn_down_shexp.weight q2_K blk.3.attn_output.weight iq2_xxs blk.3.attn_v.weight q4_K blk.3.ffn_down_exps.weight q2_K +blk.4.attn_qkv.weight q4_K +blk.5.attn_qkv.weight q4_K +blk.6.attn_qkv.weight q4_K blk.7.attn_output.weight iq2_xxs blk.7.attn_v.weight q4_K +blk.8.attn_qkv.weight q4_K +blk.9.attn_qkv.weight q4_K +blk.10.attn_qkv.weight q4_K blk.11.attn_output.weight iq2_xxs blk.11.attn_v.weight q4_K +blk.12.attn_qkv.weight q4_K +blk.13.attn_qkv.weight q4_K +blk.14.attn_qkv.weight q4_K blk.15.attn_output.weight iq2_xxs blk.15.attn_v.weight q4_K +blk.16.attn_qkv.weight q4_K +blk.17.attn_qkv.weight q4_K +blk.18.attn_qkv.weight q4_K blk.19.attn_output.weight iq2_xxs blk.19.attn_v.weight q4_K +blk.20.attn_qkv.weight q4_K +blk.21.attn_qkv.weight q4_K +blk.22.attn_qkv.weight q4_K blk.23.attn_output.weight iq2_xxs blk.23.attn_v.weight q4_K +blk.24.attn_qkv.weight q4_K +blk.25.attn_qkv.weight q4_K +blk.26.attn_qkv.weight q4_K blk.27.attn_output.weight iq2_xxs blk.27.attn_v.weight q4_K +blk.28.attn_qkv.weight q4_K +blk.29.attn_qkv.weight q4_K +blk.30.attn_qkv.weight q4_K blk.31.attn_output.weight iq2_xxs blk.31.attn_v.weight q4_K +blk.32.attn_qkv.weight q4_K +blk.33.attn_qkv.weight q4_K +blk.34.attn_qkv.weight q4_K blk.35.attn_output.weight iq2_xxs blk.35.attn_v.weight q4_K +blk.36.attn_qkv.weight q4_K +blk.37.attn_qkv.weight q4_K +blk.38.attn_qkv.weight q4_K blk.39.attn_output.weight iq2_xxs blk.39.attn_v.weight q4_K +blk.40.attn_qkv.weight q4_K +blk.41.attn_qkv.weight q4_K +blk.42.attn_qkv.weight q4_K blk.43.attn_output.weight iq2_xxs blk.43.attn_v.weight q4_K +blk.44.attn_qkv.weight q4_K +blk.45.attn_qkv.weight q4_K +blk.46.attn_qkv.weight q4_K blk.47.attn_output.weight iq2_xxs blk.47.attn_v.weight q4_K +blk.48.attn_qkv.weight q4_K +blk.49.attn_qkv.weight q4_K +blk.50.attn_qkv.weight q4_K blk.51.attn_output.weight iq2_xxs blk.51.attn_v.weight q4_K +blk.52.attn_qkv.weight q4_K +blk.53.attn_qkv.weight q4_K +blk.54.attn_qkv.weight q4_K blk.55.attn_output.weight iq2_xxs blk.55.attn_v.weight q4_K +blk.56.attn_qkv.weight q4_K +blk.57.attn_qkv.weight q4_K +blk.58.attn_qkv.weight q4_K blk.59.attn_output.weight iq2_xxs blk.59.attn_v.weight q4_K