From dd4f4bd0b88c4d59613033ba941d85e7ce1d9547 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sat, 27 Sep 2025 17:23:48 +0100 Subject: [PATCH] Reduce bpw range --- src/llama-quant.cpp | 7 +------ tools/quantize/quantize.cpp | 8 ++++---- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp index f36b9202d5..0386352014 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -655,12 +655,7 @@ static std::unordered_map target_bpw_type( GGML_TYPE_Q5_1, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, - GGML_TYPE_Q8_0, -#ifdef GGML_USE_METAL - GGML_TYPE_F16 -#else - GGML_TYPE_BF16 -#endif + GGML_TYPE_Q8_0 }; constexpr double epsilon = 1e-12; diff --git a/tools/quantize/quantize.cpp b/tools/quantize/quantize.cpp index 03018cc301..69e03179b3 100644 --- a/tools/quantize/quantize.cpp +++ b/tools/quantize/quantize.cpp @@ -132,7 +132,7 @@ static void usage(const char * executable) { printf(" Advanced option to selectively quantize tensors. May be specified multiple times.\n"); printf(" --prune-layers L0,L1,L2...comma-separated list of layer numbers to prune from the model\n"); printf(" Advanced option to remove all tensors from the given layers\n"); - printf(" --target-bpw: target bits per weight (bpw). Must be a positive number between 0.0 and 16.0\n"); + printf(" --target-bpw: target bits per weight (bpw). Must be a positive number between 0.0 and 8.0\n"); printf(" Advanced option to automatically select quantization types to achieve a total bits per weight (bpw) target\n"); printf(" --no-bias: use mean square error estimation only (no aligment bias)\n"); printf(" Advanced option use MSE only and disable aligment bias error estimation\n"); @@ -484,13 +484,13 @@ static bool parse_target_bpw(const char * data, float & target_bpw) { try { target_bpw = std::stof(data); - if (target_bpw < 0.0f || target_bpw > 16.0f) { - printf("\n%s: target bits per weight (bpw) must be a positive number between 0.0 and 16.0\n\n", __func__); + if (target_bpw < 0.0f || target_bpw > 8.0f) { + printf("\n%s: target bits per weight (bpw) must be a positive number between 0.0 and 8.0\n\n", __func__); return false; } } catch (const std::exception & e) { - printf("\n%s: '%s' is not valid. Target bits per weight (bpw) must be a positive number between 0.0 and 16.0\n\n", __func__, data); + printf("\n%s: '%s' is not valid. Target bits per weight (bpw) must be a positive number between 0.0 and 8.0\n\n", __func__, data); return false; }