happyz
/
llama.cpp
mirror of https://github.com/ggerganov/llama.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Compute cosine similarity based on activations

This commit is contained in:
Ed Addario 2025-08-02 16:43:49 +01:00
parent cce514a392
commit b7fb362d8e
No known key found for this signature in database
GPG Key ID: E7875815A3230993
1 changed files with 45 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
tools/imatrix/imatrix.cpp
View File

@ -224,27 +224,60 @@ static int compute_tensor_statistics(std::vector<tensor_statistics> & tstats, co
return calc_mode;
}
static void compute_cossim(std::vector<tensor_statistics> & tstats) {
static void compute_layer_statistics(std::vector<tensor_statistics> & tstats) {
static const std::regex pattern(R"(blk\.(\d+)\.)");
auto build_avg = [](const Stats & s) -> std::vector<float> {
if (s.counts.empty()) return {};
const size_t n_mat = s.counts.size();
const size_t len = !s.in_sum.empty() ? s.in_sum.size()
: s.in_sum2.size();
if (len == 0 || len % n_mat != 0) return {};
const size_t row = len / n_mat;
std::vector<float> v;
v.reserve(len);
if (!s.in_sum.empty()) {
for (size_t m = 0; m < n_mat; ++m) {
const float c = (float)s.counts[m];
if (c <= 0) return {};
const size_t off = m*row;
for (size_t j = 0; j < row; ++j) v.push_back(s.in_sum[off+j]/c);
}
} else {
for (size_t m = 0; m < n_mat; ++m) {
const float c = (float)s.counts[m];
if (c <= 0) return {};
const size_t off = m*row;
for (size_t j = 0; j < row; ++j) v.push_back(s.in_sum2[off+j]/c);
}
}
return v;
};
// compute the cosine similarity between the same tensors in consecutive layers
for (auto & ts : tstats) {
ts.cossim = 0;
if (std::smatch match; std::regex_search(ts.tensor, match, pattern)) {
const int blk = std::stoi(match[1]);
if (blk <= 0) continue;
std::string tname(ts.tensor);
tname.replace(match.position(1), match.length(1), std::to_string(blk-1));
auto prev = std::find_if(tstats.begin(), tstats.end(),
[tname](const tensor_statistics & t) { return t.tensor == tname; });
if (prev != tstats.end()) {
const float dot_product = std::inner_product(ts.stats.in_sum2.begin(), ts.stats.in_sum2.end(),
prev->stats.in_sum2.begin(), 0.0f);
const float magnitude = std::sqrt(std::inner_product(ts.stats.in_sum2.begin(), ts.stats.in_sum2.end(),
ts.stats.in_sum2.begin(), 0.0f));
const float prev_magnitude = std::sqrt(std::inner_product(prev->stats.in_sum2.begin(), prev->stats.in_sum2.end(),
prev->stats.in_sum2.begin(), 0.0f));
const float cos_sim = dot_product / (magnitude * prev_magnitude);
ts.cossim = cos_sim;
if (prev == tstats.end()) continue;
const auto curr_avg = build_avg(ts.stats);
const auto prev_avg = build_avg(prev->stats);
if (curr_avg.size() == prev_avg.size() && !curr_avg.empty()) {
float dot_prod = 0.0f, vec1 = 0.0f, vec2 = 0.0f;
for (size_t i = 0; i < curr_avg.size(); ++i) {
dot_prod += curr_avg[i]*prev_avg[i];
vec1 += curr_avg[i]*curr_avg[i];
vec2 += prev_avg[i]*prev_avg[i];
}
if (vec1 > 0 && vec2 > 0) ts.cossim = dot_prod / (std::sqrt(vec1)*std::sqrt(vec2));
}
} else {
ts.cossim = 0;
}
}
}
}
}