happyz
/
llama.cpp
mirror of https://github.com/ggerganov/llama.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

server: add /v1/metrics endpoint

This commit is contained in:
Kritavya 2025-12-13 21:51:25 +05:30
parent 254098a279
commit aec7bdb42b
3 changed files with 32 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
tools/server/server-context.cpp
View File

@ -2912,6 +2912,27 @@ void server_routes::init_routes() {
return res;
};
this->get_v1_metrics = [this](const server_http_req &) {
auto res = std::make_unique<server_res_generator>(ctx_server);
// Calculate uptime in seconds
int64_t uptime_sec = (llama_time_us() - t_server_start) / 1000000;
json data = {
{"status", "online"},
{"uptime_sec", uptime_sec}
};
// Include system_info if available
if (!system_info_str.empty()) {
data["system_info"] = system_info_str;
}
res->ok(data);
return res;
};
this->get_metrics = [this](const server_http_req &) {
auto res = std::make_unique<server_res_generator>(ctx_server);
if (!params.endpoint_metrics) {

8
tools/server/server-context.h
View File

@ -51,8 +51,9 @@ struct server_context {
struct server_res_generator;
struct server_routes {
server_routes(const common_params & params, server_context & ctx_server, std::function<bool()> is_ready = []() { return true; })
: params(params), ctx_server(*ctx_server.impl), is_ready(is_ready) {
server_routes(const common_params & params, server_context & ctx_server, std::function<bool()> is_ready = []() { return true; }, int64_t t_start = 0)
: params(params), ctx_server(*ctx_server.impl), is_ready(is_ready), t_server_start(t_start),
system_info_str(common_params_get_system_info(params)) {
init_routes();
}
@ -60,6 +61,7 @@ struct server_routes {
// handlers using lambda function, so that they can capture `this` without `std::bind`
server_http_context::handler_t get_health;
server_http_context::handler_t get_metrics;
server_http_context::handler_t get_v1_metrics;
server_http_context::handler_t get_slots;
server_http_context::handler_t post_slots;
server_http_context::handler_t get_props;
@ -90,4 +92,6 @@ private:
const common_params & params;
server_context_impl & ctx_server;
std::function<bool()> is_ready;
int64_t t_server_start;
std::string system_info_str;
};

6
tools/server/server.cpp
View File

@ -108,12 +108,15 @@ int main(int argc, char ** argv, char ** envp) {
return 1;
}
// Capture server start time for metrics
int64_t t_server_start = llama_time_us();
//
// Router
//
// register API routes
server_routes routes(params, ctx_server, [&ctx_http]() { return ctx_http.is_ready.load(); });
server_routes routes(params, ctx_server, [&ctx_http]() { return ctx_http.is_ready.load(); }, t_server_start);
bool is_router_server = params.model.path.empty();
std::optional<server_models_routes> models_routes{};
@ -153,6 +156,7 @@ int main(int argc, char ** argv, char ** envp) {
ctx_http.get ("/health", ex_wrapper(routes.get_health)); // public endpoint (no API key check)
ctx_http.get ("/v1/health", ex_wrapper(routes.get_health)); // public endpoint (no API key check)
ctx_http.get ("/v1/metrics", ex_wrapper(routes.get_v1_metrics)); // public endpoint (no API key check)
ctx_http.get ("/metrics", ex_wrapper(routes.get_metrics));
ctx_http.get ("/props", ex_wrapper(routes.get_props));
ctx_http.post("/props", ex_wrapper(routes.post_props));