kanshan/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-07-08 20:20:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into xsn/cli_http_based

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-07-07 21:11:05 +02:00
parent a432e6f863 bec4772f6a
commit 7cd7832297
444 changed files with 44486 additions and 18746 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server.cpp
+71 -13
View File
@@ -2,6 +2,7 @@
#include "server-http.h"
#include "server-models.h"
#include "server-cors-proxy.h"
#include "server-stream.h"
#include "server-tools.h"
#include "arg.h"
@@ -92,6 +93,10 @@ int llama_server(int argc, char ** argv) {
common_init();
// start the stream session manager GC right after common init, before any HTTP route can
// touch it. lifecycle is symmetric, stop_gc() runs in clean_up() before backend free
g_stream_sessions.start_gc();
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_SERVER)) {
return 1;
}
@@ -105,14 +110,19 @@ int llama_server(int argc, char ** argv) {
int llama_server(common_params & params, int argc, char ** argv) {
bool is_run_by_cli = (argv == nullptr);
common_models_handler models_handler;
// note: router mode also accepts -hf remote-preset, so we need to check that first
if (!is_run_by_cli && !params.model.hf_repo.empty()) {
try {
common_params_handle_models_params handle_params;
handle_params.preset_only = true;
common_params_handle_models(params, LLAMA_EXAMPLE_SERVER, handle_params);
models_handler = common_models_handler_init(params, LLAMA_EXAMPLE_SERVER);
if (common_models_handler_is_preset_repo(models_handler)) {
// apply the preset and start the server in router mode
common_models_handler_apply(models_handler, params);
}
} catch (const std::exception & e) {
// ignored for now
SRV_ERR("failed to fetch model metadata: %s\n", e.what());
return 1;
}
}
@@ -134,7 +144,7 @@ int llama_server(common_params & params, int argc, char ** argv) {
}
if (params.n_parallel < 0) {
SRV_INF("%s", "n_parallel is set to auto, using n_parallel = 4 and kv_unified = true\n");
SRV_TRC("%s", "n_parallel is set to auto, using n_parallel = 4 and kv_unified = true\n");
params.n_parallel = 4;
params.kv_unified = true;
@@ -254,9 +264,45 @@ int llama_server(common_params & params, int argc, char ** argv) {
ctx_http.get ("/slots", ex_wrapper(routes.get_slots));
ctx_http.post("/slots/:id_slot", ex_wrapper(routes.post_slots));
// resumable streaming, the conversation_id is the session identity end to end. router and
// child wire different handlers under the same paths: a child binds the local g_stream_sessions
// backed factories, the router binds proxies that resolve the owning child through the
// conv_id -> model map
server_http_context::handler_t stream_get_h;
server_http_context::handler_t streams_lookup_h;
server_http_context::handler_t stream_delete_h;
if (is_router_server) {
stream_get_h = models_routes->router_stream_get;
streams_lookup_h = models_routes->router_streams_lookup;
stream_delete_h = models_routes->router_stream_delete;
} else {
stream_get_h = make_stream_get_handler();
streams_lookup_h = make_streams_lookup_handler();
stream_delete_h = make_stream_delete_handler();
}
ctx_http.get ("/v1/stream/:conv_id", ex_wrapper(stream_get_h));
// POST /v1/streams/lookup with body {"conversation_ids": [...]}. you can only ask for ids
// you already own (the WebUI passes the convs visible in its sidebar). the server never
// lists ids it has not been asked about, so a random caller cannot enumerate live sessions
ctx_http.post("/v1/streams/lookup", ex_wrapper(streams_lookup_h));
ctx_http.del ("/v1/stream/:conv_id", ex_wrapper(stream_delete_h));
// Google Cloud Platform (Vertex AI) compat
ctx_http.register_gcp_compat();
// return 403 for disabled features
server_http_context::handler_t res_403 = [](const server_http_req &) {
auto res = std::make_unique<server_http_res>();
res->status = 403;
res->data = safe_json_to_str({
{"error", {
{"message", "this feature is disabled"},
{"type", "feature_disabled"},
}}
});
return res;
};
// CORS proxy (EXPERIMENTAL, only used by the Web UI for MCP)
if (params.ui_mcp_proxy) {
SRV_WRN("%s", "-----------------\n");
@@ -265,7 +311,11 @@ int llama_server(common_params & params, int argc, char ** argv) {
SRV_WRN("%s", "-----------------\n");
ctx_http.get ("/cors-proxy", ex_wrapper(proxy_handler_get));
ctx_http.post("/cors-proxy", ex_wrapper(proxy_handler_post));
} else {
ctx_http.get ("/cors-proxy", ex_wrapper(res_403));
ctx_http.post("/cors-proxy", ex_wrapper(res_403));
}
// EXPERIMENTAL built-in tools
if (!params.server_tools.empty()) {
try {
@@ -280,6 +330,9 @@ int llama_server(common_params & params, int argc, char ** argv) {
SRV_WRN("%s", "-----------------\n");
ctx_http.get ("/tools", ex_wrapper(tools.handle_get));
ctx_http.post("/tools", ex_wrapper(tools.handle_post));
} else {
ctx_http.get ("/tools", ex_wrapper(res_403));
ctx_http.post("/tools", ex_wrapper(res_403));
}
//
@@ -291,7 +344,12 @@ int llama_server(common_params & params, int argc, char ** argv) {
} else if (!is_router_server && !is_run_by_cli) {
// single-model mode (NOT spawned by router)
// if this is invoked by CLI, model downloading should be already handled
common_params_handle_models(params, LLAMA_EXAMPLE_SERVER, {});
try {
common_models_handler_apply(models_handler, params);
} catch (const std::exception & e) {
SRV_ERR("failed to download model: %s\n", e.what());
return 1;
}
}
//
@@ -301,10 +359,12 @@ int llama_server(common_params & params, int argc, char ** argv) {
std::function<void()> clean_up;
if (is_router_server) {
SRV_INF("%s", "starting router server, no model will be loaded in this process\n");
SRV_INF("%s", "starting server in router mode. models will be automatically loaded on-demand\n");
clean_up = [&models_routes]() {
SRV_INF("%s: cleaning up before exit...\n", __func__);
// stop the session GC first, it finalizes live sessions and wakes pending readers
g_stream_sessions.stop_gc();
if (models_routes.has_value()) {
models_routes->stopping.store(true); // maybe redundant, but just to be safe
models_routes->models.unload_all();
@@ -331,6 +391,8 @@ int llama_server(common_params & params, int argc, char ** argv) {
// setup clean up function, to be called before exit
clean_up = [&ctx_http, &ctx_server]() {
SRV_INF("%s: cleaning up before exit...\n", __func__);
// stop the session GC first, it finalizes live sessions and wakes pending readers
g_stream_sessions.stop_gc();
ctx_http.stop();
ctx_server.terminate();
llama_backend_free();
@@ -350,9 +412,6 @@ int llama_server(common_params & params, int argc, char ** argv) {
});
}
// load the model
SRV_INF("%s", "loading model\n");
if (!ctx_server.load_model(params)) {
clean_up();
if (ctx_http.thread.joinable()) {
@@ -390,8 +449,9 @@ int llama_server(common_params & params, int argc, char ** argv) {
#endif
}
SRV_INF("listening on %s\n", ctx_http.listening_address.c_str());
if (is_router_server) {
SRV_INF("router server is listening on %s\n", ctx_http.listening_address.c_str());
SRV_WRN("%s", "NOTE: router mode is experimental\n");
SRV_WRN("%s", " it is not recommended to use this mode in untrusted environments\n");
@@ -407,8 +467,6 @@ int llama_server(common_params & params, int argc, char ** argv) {
// when the HTTP server stops, clean up and exit
clean_up();
} else {
SRV_INF("server is listening on %s\n", ctx_http.listening_address.c_str());
// optionally, notify router server that this instance is ready
std::thread monitor_thread;
if (child.is_child()) {