From dd67c79a26e3cfd811ecedbf7fdf2af3421c77ef Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Tue, 6 Jan 2026 18:52:56 +0100 Subject: [PATCH 1/4] demo cors proxy --- tools/server/server-models.cpp | 7 ++++- tools/server/server-models.h | 50 ++++++++++++++++++++++++++++++++-- tools/server/server.cpp | 3 ++ 3 files changed, 57 insertions(+), 3 deletions(-) diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 803cb02e6e..ab495cdb46 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -1004,6 +1004,7 @@ server_http_proxy::server_http_proxy( auto pipe = std::make_shared>(); // setup Client + cli->set_follow_location(true); cli->set_connection_timeout(0, 200000); // 200 milliseconds cli->set_write_timeout(timeout_read, 0); // reversed for cli (client) vs srv (server) cli->set_read_timeout(timeout_write, 0); @@ -1053,7 +1054,11 @@ server_http_proxy::server_http_proxy( req.method = method; req.path = path; for (const auto & [key, value] : headers) { - req.set_header(key, value); + if (key == "Host" || key == "host") { + req.set_header(key, host + ":" + std::to_string(port)); + } else { + req.set_header(key, value); + } } req.body = body; req.response_handler = response_handler; diff --git a/tools/server/server-models.h b/tools/server/server-models.h index a397abda4a..7ffd6bef5c 100644 --- a/tools/server/server-models.h +++ b/tools/server/server-models.h @@ -2,6 +2,7 @@ #include "common.h" #include "preset.h" +#include "http.h" #include "server-common.h" #include "server-http.h" @@ -184,8 +185,8 @@ public: const std::map & headers, const std::string & body, const std::function should_stop, - int32_t timeout_read, - int32_t timeout_write + int32_t timeout_read = 600, + int32_t timeout_write = 600 ); ~server_http_proxy() { if (cleanup) { @@ -201,3 +202,48 @@ private: std::string content_type; }; }; + +// BELOW IS DEMO CODE FOR PROXY HANDLERS +// DO NOT MERGE IT AS-IS + +static server_http_res_ptr proxy_request(const server_http_req & req, std::string method) { + std::string target_url = req.get_param("url"); + common_http_url parsed_url = common_http_parse_url(target_url); + + if (parsed_url.host.empty()) { + throw std::runtime_error("invalid target URL: missing host"); + } + + if (parsed_url.path.empty()) { + parsed_url.path = "/"; + } + + if (!parsed_url.password.empty()) { + throw std::runtime_error("authentication in target URL is not supported"); + } + + if (parsed_url.scheme != "http" && parsed_url.scheme != "https") { + throw std::runtime_error("unsupported URL scheme in target URL: " + parsed_url.scheme); + } + + SRV_INF("proxying %s request to %s://%s%s\n", method.c_str(), parsed_url.scheme.c_str(), parsed_url.host.c_str(), parsed_url.path.c_str()); + + auto proxy = std::make_unique( + method, + parsed_url.host, + parsed_url.scheme == "http" ? 80 : 443, + parsed_url.path, + req.headers, + req.body, + req.should_stop); + + return proxy; +} + +static server_http_context::handler_t proxy_handler_post = [](const server_http_req & req) -> server_http_res_ptr { + return proxy_request(req, "POST"); +}; + +static server_http_context::handler_t proxy_handler_get = [](const server_http_req & req) -> server_http_res_ptr { + return proxy_request(req, "GET"); +}; diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 1d9abf6055..e8f16d5b23 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -195,6 +195,9 @@ int main(int argc, char ** argv) { // Save & load slots ctx_http.get ("/slots", ex_wrapper(routes.get_slots)); ctx_http.post("/slots/:id_slot", ex_wrapper(routes.post_slots)); + // CORS proxy + ctx_http.get ("/cors-proxy", ex_wrapper(proxy_handler_get)); + ctx_http.post("/cors-proxy", ex_wrapper(proxy_handler_post)); // // Start the server From 52ae4e80f3bb96d656da967b50ae515fd981b521 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Tue, 6 Jan 2026 21:04:15 +0100 Subject: [PATCH 2/4] fix --- tools/server/server-models.cpp | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index ab495cdb46..bd0aa9dd63 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -1003,6 +1003,14 @@ server_http_proxy::server_http_proxy( auto cli = std::make_shared(host, port); auto pipe = std::make_shared>(); + if (port == 443) { +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + cli.reset(new httplib::SSLClient(host.c_str(), port)); +#else + throw std::runtime_error("HTTPS requested but CPPHTTPLIB_OPENSSL_SUPPORT is not defined"); +#endif + } + // setup Client cli->set_follow_location(true); cli->set_connection_timeout(0, 200000); // 200 milliseconds @@ -1054,11 +1062,16 @@ server_http_proxy::server_http_proxy( req.method = method; req.path = path; for (const auto & [key, value] : headers) { + if (key == "Accept-Encoding") { + // disable Accept-Encoding to avoid compressed responses + continue; + } if (key == "Host" || key == "host") { - req.set_header(key, host + ":" + std::to_string(port)); + req.set_header(key, host); } else { req.set_header(key, value); } + SRV_INF("header %s: %s\n", key.c_str(), req.get_header_value(key).c_str()); } req.body = body; req.response_handler = response_handler; From 481b8c3efc0d02ad2247f18f1cd7da91024cfa61 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Tue, 6 Jan 2026 18:59:33 +0100 Subject: [PATCH 3/4] download: use httplib and boringssl by default --- .github/workflows/build.yml | 10 +++++----- .github/workflows/server.yml | 4 ++-- CMakeLists.txt | 6 ++++-- vendor/cpp-httplib/CMakeLists.txt | 4 ++-- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1193779d0b..4e8612781f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -80,7 +80,7 @@ jobs: -DCMAKE_BUILD_RPATH="@loader_path" \ -DLLAMA_FATAL_WARNINGS=ON \ -DLLAMA_CURL=OFF \ - -DLLAMA_BUILD_BORINGSSL=ON \ + -DLLAMA_BORINGSSL=ON \ -DGGML_METAL_USE_BF16=ON \ -DGGML_METAL_EMBED_LIBRARY=OFF \ -DGGML_METAL_SHADER_DEBUG=ON \ @@ -119,7 +119,7 @@ jobs: -DCMAKE_BUILD_RPATH="@loader_path" \ -DLLAMA_FATAL_WARNINGS=ON \ -DLLAMA_CURL=OFF \ - -DLLAMA_BUILD_BORINGSSL=ON \ + -DLLAMA_BORINGSSL=ON \ -DGGML_METAL=OFF \ -DGGML_RPC=ON \ -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3 @@ -1043,7 +1043,7 @@ jobs: id: cmake_build run: | cmake -S . -B build ${{ matrix.defines }} ` - -DLLAMA_CURL=OFF -DLLAMA_BUILD_BORINGSSL=ON + -DLLAMA_CURL=OFF -DLLAMA_BORINGSSL=ON cmake --build build --config Release -j ${env:NUMBER_OF_PROCESSORS} - name: Add libopenblas.dll @@ -1151,7 +1151,7 @@ jobs: cmake -S . -B build -G "Ninja Multi-Config" ^ -DLLAMA_BUILD_SERVER=ON ^ -DLLAMA_CURL=OFF ^ - -DLLAMA_BUILD_BORINGSSL=ON ^ + -DLLAMA_BORINGSSL=ON ^ -DGGML_NATIVE=OFF ^ -DGGML_BACKEND_DL=ON ^ -DGGML_CPU_ALL_VARIANTS=ON ^ @@ -1259,7 +1259,7 @@ jobs: -DCMAKE_CXX_FLAGS="-I$($PWD.Path.Replace('\', '/'))/opt/rocm-${{ env.ROCM_VERSION }}/include/" ` -DCMAKE_BUILD_TYPE=Release ` -DLLAMA_CURL=OFF ` - -DLLAMA_BUILD_BORINGSSL=ON ` + -DLLAMA_BORINGSSL=ON ` -DROCM_DIR="${env:HIP_PATH}" ` -DGGML_HIP=ON ` -DGGML_HIP_ROCWMMA_FATTN=ON ` diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index 5694feb2c9..4db6732fdf 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -72,7 +72,7 @@ jobs: - name: Build id: cmake_build run: | - cmake -B build -DLLAMA_CURL=OFF -DLLAMA_BUILD_BORINGSSL=ON + cmake -B build -DLLAMA_CURL=OFF -DLLAMA_BORINGSSL=ON cmake --build build --config ${{ matrix.build_type }} -j ${env:NUMBER_OF_PROCESSORS} --target llama-server - name: Python setup @@ -108,7 +108,7 @@ jobs: - name: Build id: cmake_build run: | - cmake -B build -DLLAMA_CURL=OFF -DLLAMA_BUILD_BORINGSSL=ON + cmake -B build -DLLAMA_CURL=OFF -DLLAMA_BORINGSSL=ON cmake --build build --config Release -j ${env:NUMBER_OF_PROCESSORS} --target llama-server - name: Python setup diff --git a/CMakeLists.txt b/CMakeLists.txt index c231ec0e3f..f8cb253db2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,9 +111,11 @@ option(LLAMA_BUILD_SERVER "llama: build server example" ${LLAMA_STANDALONE}) option(LLAMA_TOOLS_INSTALL "llama: install tools" ${LLAMA_TOOLS_INSTALL_DEFAULT}) # 3rd party libs -option(LLAMA_CURL "llama: use libcurl to download model from an URL" ON) +option(LLAMA_CURL "llama: use libcurl to download model from an URL" OFF) option(LLAMA_HTTPLIB "llama: if libcurl is disabled, use httplib to download model from an URL" ON) -option(LLAMA_OPENSSL "llama: use openssl to support HTTPS" OFF) +option(LLAMA_BORINGSSL "llama: use boringssl to support HTTPS" ON) +option(LLAMA_LIBRESSL "llama: use libressl to support HTTPS" OFF) +option(LLAMA_OPENSSL "llama: use openssl to support HTTPS" OFF) option(LLAMA_LLGUIDANCE "llama-common: include LLGuidance library for structured output in common utils" OFF) # Required for relocatable CMake package diff --git a/vendor/cpp-httplib/CMakeLists.txt b/vendor/cpp-httplib/CMakeLists.txt index 8f0d15d1fd..081421c2ab 100644 --- a/vendor/cpp-httplib/CMakeLists.txt +++ b/vendor/cpp-httplib/CMakeLists.txt @@ -29,7 +29,7 @@ target_compile_definitions(${TARGET} PRIVATE set(OPENSSL_NO_ASM ON CACHE BOOL "Disable OpenSSL ASM code when building BoringSSL or LibreSSL") -if (LLAMA_BUILD_BORINGSSL) +if (LLAMA_BORINGSSL) set(FIPS OFF CACHE BOOL "Enable FIPS (BoringSSL)") set(BORINGSSL_GIT "https://boringssl.googlesource.com/boringssl" CACHE STRING "BoringSSL git repository") @@ -70,7 +70,7 @@ if (LLAMA_BUILD_BORINGSSL) set(CPPHTTPLIB_OPENSSL_SUPPORT TRUE) target_link_libraries(${TARGET} PUBLIC ssl crypto) -elseif (LLAMA_BUILD_LIBRESSL) +elseif (LLAMA_LIBRESSL) set(LIBRESSL_VERSION "4.2.1" CACHE STRING "LibreSSL version") message(STATUS "Fetching LibreSSL version ${LIBRESSL_VERSION}") From 2b612ad9ae0bdcf7e2f294b2f3d2563c4816c095 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Tue, 6 Jan 2026 21:14:25 +0100 Subject: [PATCH 4/4] https working --- tools/server/server-models.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index bd0aa9dd63..51de216d46 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -1000,12 +1000,12 @@ server_http_proxy::server_http_proxy( int32_t timeout_write ) { // shared between reader and writer threads - auto cli = std::make_shared(host, port); + auto cli = std::make_shared(host, port); auto pipe = std::make_shared>(); if (port == 443) { #ifdef CPPHTTPLIB_OPENSSL_SUPPORT - cli.reset(new httplib::SSLClient(host.c_str(), port)); + cli.reset(new httplib::SSLClient(host, port)); #else throw std::runtime_error("HTTPS requested but CPPHTTPLIB_OPENSSL_SUPPORT is not defined"); #endif @@ -1071,7 +1071,6 @@ server_http_proxy::server_http_proxy( } else { req.set_header(key, value); } - SRV_INF("header %s: %s\n", key.c_str(), req.get_header_value(key).c_str()); } req.body = body; req.response_handler = response_handler;