happyz
/
llama.cpp
mirror of https://github.com/ggerganov/llama.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Avoid division by zero on zero-count matrices

This commit is contained in:
Ed Addario 2025-10-28 18:33:37 +00:00
parent c9a0874f35
commit 637e674da6
No known key found for this signature in database
GPG Key ID: E7875815A3230993
1 changed files with 32 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
tools/imatrix/imatrix.cpp
View File

@ -179,26 +179,38 @@ static bool compute_vector_statistics(std::vector<tensor_statistics> & tstats, c
activations.reserve(e.values.size()); activations.reserve(e.values.size());
for (int i = 0; i < n_mat; ++i) { for (int i = 0; i < n_mat; ++i) {
const float c = (float)e.counts[i];
const size_t off = i * row_size;
for (int j = 0; j < row_size; ++j) { for (int j = 0; j < row_size; ++j) {
activations.push_back(e.values[i*row_size + j] / e.counts[i]); if (c <= 0.0f) {
activations.push_back(1.0f); // same as legacy
} else {
activations.push_back(e.values[off + j] / c);
}
} }
} }
} else { } else {
activations.reserve(e.activations.size()); activations.reserve(e.activations.size());
for (int i = 0; i < n_mat; ++i) { for (int i = 0; i < n_mat; ++i) {
const float c = (float)e.counts[i];
const size_t off = i * row_size;
for (int j = 0; j < row_size; ++j) { for (int j = 0; j < row_size; ++j) {
activations.push_back(e.activations[i*row_size + j] / e.counts[i]); if (c <= 0.0f) {
activations.push_back(1.0f); // same as legacy
} else {
activations.push_back(e.activations[off + j] / c);
}
} }
} }
} }
const float sum = std::accumulate(activations.begin(), activations.end(), 0.0f); const float sum = std::accumulate(activations.begin(), activations.end(), 0.0f);
const float max = *std::max_element(activations.begin(), activations.end()); const float max = * std::max_element(activations.begin(), activations.end());
const float min = *std::min_element(activations.begin(), activations.end()); const float min = * std::min_element(activations.begin(), activations.end());
const float mean = sum / activations.size(); const float mean = sum / activations.size();
const float sqr_sum = std::inner_product(activations.begin(), activations.end(), activations.begin(), 0.0f); const float sqr_sum = std::inner_product(activations.begin(), activations.end(), activations.begin(), 0.0f);
const float variance = (sqr_sum / activations.size()) - (mean * mean); const float variance = sqr_sum / activations.size() - mean * mean;
const float std_deviation = std::sqrt(std::max(0.0f, variance)); const float std_deviation = std::sqrt(std::max(0.0f, variance));
float entropy = 0; float entropy = 0;
@ -226,7 +238,7 @@ static bool compute_vector_statistics(std::vector<tensor_statistics> & tstats, c
} }
} }
int zd_score = 0; float zd_score = 0.0f;
if (std_deviation > 0.0f) { if (std_deviation > 0.0f) {
for (const auto act : activations) { for (const auto act : activations) {
const float z = (act - mean) / std_deviation; const float z = (act - mean) / std_deviation;
@ -244,7 +256,7 @@ static bool compute_vector_statistics(std::vector<tensor_statistics> & tstats, c
ts.elements = static_cast<int>(activations.size()); ts.elements = static_cast<int>(activations.size());
ts.std_deviation = std_deviation; ts.std_deviation = std_deviation;
ts.entropy = entropy; ts.entropy = entropy;
ts.zd_score = static_cast<float>(zd_score) / ts.elements; ts.zd_score = zd_score / ts.elements;
return e.activations.empty(); return e.activations.empty();
} }