happyz
/
llama.cpp
mirror of https://github.com/ggerganov/llama.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

server : Cmdline arg -to changes http read timeout from current 600sec default (#18279) 

* Prevent crash if TTFT >300sec, boosted to 90 days

* server : allow configurable HTTP timeouts for child models

* server : pass needed timeouts from params only

---------

Co-authored-by: Greg Slocum <fromgit@wbtek.slocum.net>
This commit is contained in:
wbtek 2025-12-30 01:12:48 +09:00 committed by GitHub
parent 3595ae5963
commit 5b1248c9af
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 14 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
tools/server/server-models.cpp
View File

@ -662,7 +662,10 @@ server_http_res_ptr server_models::proxy_request(const server_http_req & req, co
req.path, req.path,
req.headers, req.headers,
req.body, req.body,
req.should_stop); req.should_stop,
base_params.timeout_read,
base_params.timeout_write
);
return proxy; return proxy;
} }
@ -950,13 +953,18 @@ server_http_proxy::server_http_proxy(
const std::string & path, const std::string & path,
const std::map<std::string, std::string> & headers, const std::map<std::string, std::string> & headers,
const std::string & body, const std::string & body,
const std::function<bool()> should_stop) { const std::function<bool()> should_stop,
int32_t timeout_read,
int32_t timeout_write
) {
// shared between reader and writer threads // shared between reader and writer threads
auto cli = std::make_shared<httplib::Client>(host, port); auto cli = std::make_shared<httplib::Client>(host, port);
auto pipe = std::make_shared<pipe_t<msg_t>>(); auto pipe = std::make_shared<pipe_t<msg_t>>();
// setup Client // setup Client
cli->set_connection_timeout(0, 200000); // 200 milliseconds cli->set_connection_timeout(0, 200000); // 200 milliseconds
cli->set_write_timeout(timeout_read, 0); // reversed for cli (client) vs srv (server)
cli->set_read_timeout(timeout_write, 0);
this->status = 500; // to be overwritten upon response this->status = 500; // to be overwritten upon response
this->cleanup = [pipe]() { this->cleanup = [pipe]() {
pipe->close_read(); pipe->close_read();

5
tools/server/server-models.h
View File

@ -183,7 +183,10 @@ public:
const std::string & path, const std::string & path,
const std::map<std::string, std::string> & headers, const std::map<std::string, std::string> & headers,
const std::string & body, const std::string & body,
const std::function<bool()> should_stop); const std::function<bool()> should_stop,
int32_t timeout_read,
int32_t timeout_write
);
~server_http_proxy() { ~server_http_proxy() {
if (cleanup) { if (cleanup) {
cleanup(); cleanup();