happyz
/
llama.cpp
mirror of https://github.com/ggerganov/llama.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Compute entropy for activations

This commit is contained in:
Ed Addario 2025-08-02 16:40:40 +01:00
parent 9744a4a1c6
commit cce514a392
No known key found for this signature in database
GPG Key ID: E7875815A3230993
1 changed files with 24 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
tools/imatrix/imatrix.cpp
View File

@ -173,6 +173,23 @@ static int compute_tensor_statistics(std::vector<tensor_statistics> & tstats, co
const float active_ratio = 1 - static_cast<float>(inactive_count) / activations.size(); const float active_ratio = 1 - static_cast<float>(inactive_count) / activations.size();
float entropy = 0; float entropy = 0;
if (calc_mode == 1) {
float div = 0.0;
std::vector<float> weights(activations.size());
for (size_t i = 0; i < activations.size(); ++i) {
const float w = activations[i] * activations[i];
weights[i] = w;
div += w;
}
if (div > 0.0) {
for (float w : weights) {
const float p = w / div;
if (p > 0.0) entropy -= p * std::log2(p);
}
}
} else {
if (sum > 0) { if (sum > 0) {
for (const auto act : activations) { for (const auto act : activations) {
if (const float p = act / sum; p > 0) { if (const float p = act / sum; p > 0) {
@ -180,6 +197,7 @@ static int compute_tensor_statistics(std::vector<tensor_statistics> & tstats, co
} }
} }
} }
}
int z_score = 0; int z_score = 0;
if (std_deviation > 0.0f) { if (std_deviation > 0.0f) {
@ -202,6 +220,8 @@ static int compute_tensor_statistics(std::vector<tensor_statistics> & tstats, co
ts.active = active_ratio; ts.active = active_ratio;
ts.entropy = entropy; ts.entropy = entropy;
ts.zd_score = static_cast<float>(z_score) / ts.elements; ts.zd_score = static_cast<float>(z_score) / ts.elements;
return calc_mode;
} }
static void compute_cossim(std::vector<tensor_statistics> & tstats) { static void compute_cossim(std::vector<tensor_statistics> & tstats) {