happyz
/
llama.cpp
mirror of https://github.com/ggerganov/llama.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Reduce bpw range

This commit is contained in:
Ed Addario 2025-09-27 17:23:48 +01:00
parent 29bb30c4ed
commit dd4f4bd0b8
No known key found for this signature in database
GPG Key ID: E7875815A3230993
2 changed files with 5 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/llama-quant.cpp
View File

@ -655,12 +655,7 @@ static std::unordered_map<std::string, ggml_type> target_bpw_type(
GGML_TYPE_Q5_1, GGML_TYPE_Q5_1,
GGML_TYPE_Q5_K, GGML_TYPE_Q5_K,
GGML_TYPE_Q6_K, GGML_TYPE_Q6_K,
GGML_TYPE_Q8_0, GGML_TYPE_Q8_0
#ifdef GGML_USE_METAL
GGML_TYPE_F16
#else
GGML_TYPE_BF16
#endif
}; };
constexpr double epsilon = 1e-12; constexpr double epsilon = 1e-12;

8
tools/quantize/quantize.cpp
View File

@ -132,7 +132,7 @@ static void usage(const char * executable) {
printf(" Advanced option to selectively quantize tensors. May be specified multiple times.\n"); printf(" Advanced option to selectively quantize tensors. May be specified multiple times.\n");
printf(" --prune-layers L0,L1,L2...comma-separated list of layer numbers to prune from the model\n"); printf(" --prune-layers L0,L1,L2...comma-separated list of layer numbers to prune from the model\n");
printf(" Advanced option to remove all tensors from the given layers\n"); printf(" Advanced option to remove all tensors from the given layers\n");
printf(" --target-bpw: target bits per weight (bpw). Must be a positive number between 0.0 and 16.0\n"); printf(" --target-bpw: target bits per weight (bpw). Must be a positive number between 0.0 and 8.0\n");
printf(" Advanced option to automatically select quantization types to achieve a total bits per weight (bpw) target\n"); printf(" Advanced option to automatically select quantization types to achieve a total bits per weight (bpw) target\n");
printf(" --no-bias: use mean square error estimation only (no aligment bias)\n"); printf(" --no-bias: use mean square error estimation only (no aligment bias)\n");
printf(" Advanced option use MSE only and disable aligment bias error estimation\n"); printf(" Advanced option use MSE only and disable aligment bias error estimation\n");
@ -484,13 +484,13 @@ static bool parse_target_bpw(const char * data, float & target_bpw) {
try { try {
target_bpw = std::stof(data); target_bpw = std::stof(data);
if (target_bpw < 0.0f || target_bpw > 16.0f) { if (target_bpw < 0.0f || target_bpw > 8.0f) {
printf("\n%s: target bits per weight (bpw) must be a positive number between 0.0 and 16.0\n\n", __func__); printf("\n%s: target bits per weight (bpw) must be a positive number between 0.0 and 8.0\n\n", __func__);
return false; return false;
} }
} }
catch (const std::exception & e) { catch (const std::exception & e) {
printf("\n%s: '%s' is not valid. Target bits per weight (bpw) must be a positive number between 0.0 and 16.0\n\n", __func__, data); printf("\n%s: '%s' is not valid. Target bits per weight (bpw) must be a positive number between 0.0 and 8.0\n\n", __func__, data);
return false; return false;
} }