happyz
/
llama.cpp
mirror of https://github.com/ggerganov/llama.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'ngxson/xsn/cors_proxy_demo' into allozaur/mcp-mvp

This commit is contained in:
Aleksander Grygier 2026-01-29 13:34:20 +01:00
parent 9d6e210a5e 2b612ad9ae
commit 406cb1dd99
7 changed files with 88 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
.github/workflows/build.yml vendored
View File

@ -79,7 +79,8 @@ jobs:
cmake -B build \
-DCMAKE_BUILD_RPATH="@loader_path" \
-DLLAMA_FATAL_WARNINGS=ON \
-DLLAMA_BUILD_BORINGSSL=ON \
-DLLAMA_CURL=OFF \
-DLLAMA_BORINGSSL=ON \
-DGGML_METAL_USE_BF16=ON \
-DGGML_METAL_EMBED_LIBRARY=OFF \
-DGGML_METAL_SHADER_DEBUG=ON \
@ -117,7 +118,8 @@ jobs:
cmake -B build \
-DCMAKE_BUILD_RPATH="@loader_path" \
-DLLAMA_FATAL_WARNINGS=ON \
-DLLAMA_BUILD_BORINGSSL=ON \
-DLLAMA_CURL=OFF \
-DLLAMA_BORINGSSL=ON \
-DGGML_METAL=OFF \
-DGGML_RPC=ON \
-DCMAKE_OSX_DEPLOYMENT_TARGET=13.3
@ -1015,7 +1017,7 @@ jobs:
id: cmake_build
run: |
cmake -S . -B build ${{ matrix.defines }} `
-DLLAMA_BUILD_BORINGSSL=ON
-DLLAMA_CURL=OFF -DLLAMA_BORINGSSL=ON
cmake --build build --config Release -j ${env:NUMBER_OF_PROCESSORS}
- name: Add libopenblas.dll
@ -1120,7 +1122,8 @@ jobs:
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
cmake -S . -B build -G "Ninja Multi-Config" ^
-DLLAMA_BUILD_SERVER=ON ^
-DLLAMA_BUILD_BORINGSSL=ON ^
-DLLAMA_CURL=OFF ^
-DLLAMA_BORINGSSL=ON ^
-DGGML_NATIVE=OFF ^
-DGGML_BACKEND_DL=ON ^
-DGGML_CPU_ALL_VARIANTS=ON ^
@ -1227,7 +1230,8 @@ jobs:
-DCMAKE_CXX_COMPILER="${env:HIP_PATH}\bin\clang++.exe" `
-DCMAKE_CXX_FLAGS="-I$($PWD.Path.Replace('\', '/'))/opt/rocm-${{ env.ROCM_VERSION }}/include/" `
-DCMAKE_BUILD_TYPE=Release `
-DLLAMA_BUILD_BORINGSSL=ON `
-DLLAMA_CURL=OFF `
-DLLAMA_BORINGSSL=ON `
-DROCM_DIR="${env:HIP_PATH}" `
-DGGML_HIP=ON `
-DGGML_HIP_ROCWMMA_FATTN=ON `

4
.github/workflows/server.yml vendored
View File

@ -72,7 +72,7 @@ jobs:
- name: Build
id: cmake_build
run: |
cmake -B build -DLLAMA_BUILD_BORINGSSL=ON -DGGML_SCHED_NO_REALLOC=ON
cmake -B build -DLLAMA_BORINGSSL=ON -DGGML_SCHED_NO_REALLOC=ON
cmake --build build --config ${{ matrix.build_type }} -j ${env:NUMBER_OF_PROCESSORS} --target llama-server
- name: Python setup
@ -108,7 +108,7 @@ jobs:
- name: Build
id: cmake_build
run: |
cmake -B build -DLLAMA_BUILD_BORINGSSL=ON -DGGML_SCHED_NO_REALLOC=ON
cmake -B build -DLLAMA_BORINGSSL=ON -DGGML_SCHED_NO_REALLOC=ON
cmake --build build --config Release -j ${env:NUMBER_OF_PROCESSORS} --target llama-server
- name: Python setup

7
CMakeLists.txt
View File

@ -111,8 +111,11 @@ option(LLAMA_BUILD_SERVER "llama: build server example" ${LLAMA_STANDALONE})
option(LLAMA_TOOLS_INSTALL "llama: install tools" ${LLAMA_TOOLS_INSTALL_DEFAULT})
# 3rd party libs
option(LLAMA_HTTPLIB "llama: httplib for downloading functionality" ON)
option(LLAMA_OPENSSL "llama: use openssl to support HTTPS" ON)
option(LLAMA_CURL "llama: use libcurl to download model from an URL" OFF)
option(LLAMA_HTTPLIB "llama: if libcurl is disabled, use httplib to download model from an URL" ON)
option(LLAMA_BORINGSSL "llama: use boringssl to support HTTPS" ON)
option(LLAMA_LIBRESSL "llama: use libressl to support HTTPS" OFF)
option(LLAMA_OPENSSL "llama: use openssl to support HTTPS" OFF)
option(LLAMA_LLGUIDANCE "llama-common: include LLGuidance library for structured output in common utils" OFF)
# deprecated

21
tools/server/server-models.cpp
View File

@ -1000,10 +1000,19 @@ server_http_proxy::server_http_proxy(
int32_t timeout_write
) {
// shared between reader and writer threads
auto cli = std::make_shared<httplib::Client>(host, port);
auto cli = std::make_shared<httplib::ClientImpl>(host, port);
auto pipe = std::make_shared<pipe_t<msg_t>>();
if (port == 443) {
#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
cli.reset(new httplib::SSLClient(host, port));
#else
throw std::runtime_error("HTTPS requested but CPPHTTPLIB_OPENSSL_SUPPORT is not defined");
#endif
}
// setup Client
cli->set_follow_location(true);
cli->set_connection_timeout(0, 200000); // 200 milliseconds
cli->set_write_timeout(timeout_read, 0); // reversed for cli (client) vs srv (server)
cli->set_read_timeout(timeout_write, 0);
@ -1053,7 +1062,15 @@ server_http_proxy::server_http_proxy(
req.method = method;
req.path = path;
for (const auto & [key, value] : headers) {
req.set_header(key, value);
if (key == "Accept-Encoding") {
// disable Accept-Encoding to avoid compressed responses
continue;
}
if (key == "Host" || key == "host") {
req.set_header(key, host);
} else {
req.set_header(key, value);
}
}
req.body = body;
req.response_handler = response_handler;

50
tools/server/server-models.h
View File

@ -2,6 +2,7 @@
#include "common.h"
#include "preset.h"
#include "http.h"
#include "server-common.h"
#include "server-http.h"
@ -184,8 +185,8 @@ public:
const std::map<std::string, std::string> & headers,
const std::string & body,
const std::function<bool()> should_stop,
int32_t timeout_read,
int32_t timeout_write
int32_t timeout_read = 600,
int32_t timeout_write = 600
);
~server_http_proxy() {
if (cleanup) {
@ -201,3 +202,48 @@ private:
std::string content_type;
};
};
// BELOW IS DEMO CODE FOR PROXY HANDLERS
// DO NOT MERGE IT AS-IS
static server_http_res_ptr proxy_request(const server_http_req & req, std::string method) {
std::string target_url = req.get_param("url");
common_http_url parsed_url = common_http_parse_url(target_url);
if (parsed_url.host.empty()) {
throw std::runtime_error("invalid target URL: missing host");
}
if (parsed_url.path.empty()) {
parsed_url.path = "/";
}
if (!parsed_url.password.empty()) {
throw std::runtime_error("authentication in target URL is not supported");
}
if (parsed_url.scheme != "http" && parsed_url.scheme != "https") {
throw std::runtime_error("unsupported URL scheme in target URL: " + parsed_url.scheme);
}
SRV_INF("proxying %s request to %s://%s%s\n", method.c_str(), parsed_url.scheme.c_str(), parsed_url.host.c_str(), parsed_url.path.c_str());
auto proxy = std::make_unique<server_http_proxy>(
method,
parsed_url.host,
parsed_url.scheme == "http" ? 80 : 443,
parsed_url.path,
req.headers,
req.body,
req.should_stop);
return proxy;
}
static server_http_context::handler_t proxy_handler_post = [](const server_http_req & req) -> server_http_res_ptr {
return proxy_request(req, "POST");
};
static server_http_context::handler_t proxy_handler_get = [](const server_http_req & req) -> server_http_res_ptr {
return proxy_request(req, "GET");
};

3
tools/server/server.cpp
View File

@ -197,6 +197,9 @@ int main(int argc, char ** argv) {
// Save & load slots
ctx_http.get ("/slots", ex_wrapper(routes.get_slots));
ctx_http.post("/slots/:id_slot", ex_wrapper(routes.post_slots));
// CORS proxy
ctx_http.get ("/cors-proxy", ex_wrapper(proxy_handler_get));
ctx_http.post("/cors-proxy", ex_wrapper(proxy_handler_post));
//
// Start the server

4
vendor/cpp-httplib/CMakeLists.txt vendored
View File

@ -30,7 +30,7 @@ target_compile_definitions(${TARGET} PRIVATE
set(OPENSSL_NO_ASM ON CACHE BOOL "Disable OpenSSL ASM code when building BoringSSL or LibreSSL")
if (LLAMA_BUILD_BORINGSSL)
if (LLAMA_BORINGSSL)
set(FIPS OFF CACHE BOOL "Enable FIPS (BoringSSL)")
set(BORINGSSL_GIT "https://boringssl.googlesource.com/boringssl" CACHE STRING "BoringSSL git repository")
@ -73,7 +73,7 @@ if (LLAMA_BUILD_BORINGSSL)
set(CPPHTTPLIB_OPENSSL_SUPPORT TRUE)
target_link_libraries(${TARGET} PUBLIC ssl crypto)
elseif (LLAMA_BUILD_LIBRESSL)
elseif (LLAMA_LIBRESSL)
set(LIBRESSL_VERSION "4.2.1" CACHE STRING "LibreSSL version")
message(STATUS "Fetching LibreSSL version ${LIBRESSL_VERSION}")