kanshan/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-07-09 04:30:33 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
bbebeec4a87355896e3faac0c2baca8130c91b6a
llama.cpp/tools/server/server.cpp
T
Pascal bbebeec4a8 server-stream: follow-up on SSE Replay Buffer (#23226) (#25047) 
* server-stream : pimpl

* server-stream: prefix free functions with server_stream_

address review from ggerganov: scope the public stream functions under the
server_stream_ prefix, matching server_stream_session_manager_start/stop.

* server-stream: guard session and manager state with the mutex

address review from ggerganov: make done, completed_ts and the GC running flag plain members under their
mutex and set the condvar predicates under the lock. keep cancelled atomic for
the lock-free should_stop poll.

* server-stream: trim comments to the non-obvious

address review from ggerganov: drop comments that restate the code, keep the
concurrency, lifetime and ordering rationale. de-stale a few comments left by the
pimpl: g_stream_sessions is now internal and the /v1/streams listing is gone.

* server-stream: update dev docs for the pimpl and prefix

reflect server_stream_session_manager_start/stop and the server_stream_ prefix,
note the manager is now a file-static singleton hidden in the .cpp

* server-stream: move stream traces to debug level

keep the bring-up traces for diagnostics but off the default log: skip
drain, draining, drain ended, DELETE evict, attach_pipe, and the router
stream resume proxy.

* server-stream: align router stream resume proxy trace with upstream

the child-side bring-up traces are already SRV_TRC on master, move the
router stream resume proxy trace to the same level.

* server-stream: move stream_read_status enum to the cpp

it is only used by the hidden session and consumer types, so it belongs
with them behind the pimpl boundary, not on the public header surface.

---------

Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2026-07-08 12:02:50 +03:00

473 lines
20 KiB
C++
Raw Blame History

#include "server-context.h"
#include "server-http.h"
#include "server-models.h"
#include "server-cors-proxy.h"
#include "server-stream.h"
#include "server-tools.h"
#include "arg.h"
#include "build-info.h"
#include "common.h"
#include "fit.h"
#include "llama.h"
#include "log.h"
#include <atomic>
#include <clocale>
#include <exception>
#include <signal.h>
#include <thread> // for std::thread::hardware_concurrency
#if defined(_WIN32)
#include <windows.h>
#endif
static std::function<void(int)> shutdown_handler;
static std::atomic_flag is_terminating = ATOMIC_FLAG_INIT;
static inline void signal_handler(int signal) {
if (is_terminating.test_and_set()) {
// in case it hangs, we can force terminate the server by hitting Ctrl+C twice
// this is for better developer experience, we can remove when the server is stable enough
fprintf(stderr, "Received second interrupt, terminating immediately.\n");
exit(1);
}
shutdown_handler(signal);
}
// wrapper function that handles exceptions and logs errors
// this is to make sure handler_t never throws exceptions; instead, it returns an error response
static server_http_context::handler_t ex_wrapper(server_http_context::handler_t func) {
return [func = std::move(func)](const server_http_req & req) -> server_http_res_ptr {
std::string message;
error_type error;
try {
return func(req);
} catch (const std::invalid_argument & e) {
// treat invalid_argument as invalid request (400)
error = ERROR_TYPE_INVALID_REQUEST;
message = e.what();
} catch (const std::exception & e) {
// treat other exceptions as server error (500)
error = ERROR_TYPE_SERVER;
message = e.what();
} catch (...) {
error = ERROR_TYPE_SERVER;
message = "unknown error";
}
auto res = std::make_unique<server_http_res>();
res->status = 500;
try {
json error_data = format_error_response(message, error);
res->status = json_value(error_data, "code", 500);
res->data = safe_json_to_str({{ "error", error_data }});
SRV_WRN("got exception: %s\n", res->data.c_str());
} catch (const std::exception & e) {
SRV_ERR("got another exception: %s | while handling exception: %s\n", e.what(), message.c_str());
res->data = "Internal Server Error";
}
return res;
};
}
// satisfies -Wmissing-declarations
int llama_server(int argc, char ** argv);
int llama_server(int argc, char ** argv) {
std::setlocale(LC_NUMERIC, "C");
// own arguments required by this example
common_params params;
common_init();
// start the stream session manager GC right after common init, before any HTTP route can
// touch it. lifecycle is symmetric, stop_gc() runs in clean_up() before backend free
server_stream_session_manager_start();
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_SERVER)) {
return 1;
}
llama_backend_init();
llama_numa_init(params.numa);
common_models_handler models_handler;
try {
models_handler = common_models_handler_init(params, LLAMA_EXAMPLE_SERVER);
if (common_models_handler_is_preset_repo(models_handler)) {
// apply the preset and start the server in router mode
common_models_handler_apply(models_handler, params);
}
} catch (const std::exception & e) {
SRV_ERR("failed to fetch model metadata: %s\n", e.what());
return 1;
}
// router server never loads a model and must not touch the GPU
const bool is_router_server = params.model.path.empty()
&& params.model.hf_repo.empty();
// skip device enumeration so the CUDA primary context stays uncreated
common_params_print_info(params, !is_router_server);
if (!is_router_server) {
// validate batch size for embeddings
// embeddings require all tokens to be processed in a single ubatch
// see https://github.com/ggml-org/llama.cpp/issues/12836
if (params.embedding && params.n_batch > params.n_ubatch) {
SRV_WRN("embeddings enabled with n_batch (%d) > n_ubatch (%d)\n", params.n_batch, params.n_ubatch);
SRV_WRN("setting n_batch = n_ubatch = %d to avoid assertion failure\n", params.n_ubatch);
params.n_batch = params.n_ubatch;
}
if (params.n_parallel < 0) {
SRV_TRC("%s", "n_parallel is set to auto, using n_parallel = 4 and kv_unified = true\n");
params.n_parallel = 4;
params.kv_unified = true;
}
}
// for consistency between server router mode and single-model mode, we set the same model name as alias
auto model_name = params.model.get_name();
if (params.model_alias.empty() && !model_name.empty()) {
params.model_alias.insert(model_name);
}
// struct that contains llama context and inference
server_context ctx_server;
server_http_context ctx_http;
if (!ctx_http.init(params)) {
SRV_ERR("%s", "failed to initialize HTTP server\n");
return 1;
}
//
// Router
//
// register API routes
server_child child; // only used in non-router mode
server_routes routes(params, ctx_server);
server_tools tools;
std::optional<server_models_routes> models_routes{};
if (is_router_server) {
// setup server instances manager
try {
models_routes.emplace(params, argc, argv);
} catch (const std::exception & e) {
SRV_ERR("failed to initialize router models: %s\n", e.what());
return 1;
}
// proxy handlers
// note: routes.get_health stays the same
routes.get_metrics = models_routes->proxy_get;
routes.post_props = models_routes->proxy_post;
routes.post_completions = models_routes->proxy_post;
routes.post_completions_oai = models_routes->proxy_post;
routes.post_chat_completions = models_routes->proxy_post;
routes.post_control = models_routes->proxy_post;
routes.post_responses_oai = models_routes->proxy_post;
routes.post_transcriptions_oai = models_routes->proxy_post;
routes.post_anthropic_messages = models_routes->proxy_post;
routes.post_anthropic_count_tokens = models_routes->proxy_post;
routes.post_infill = models_routes->proxy_post;
routes.post_embeddings = models_routes->proxy_post;
routes.post_embeddings_oai = models_routes->proxy_post;
routes.post_rerank = models_routes->proxy_post;
routes.post_tokenize = models_routes->proxy_post;
routes.post_detokenize = models_routes->proxy_post;
routes.post_apply_template = models_routes->proxy_post;
routes.post_chat_completions_tok = models_routes->proxy_post;
routes.post_responses_tok_oai = models_routes->proxy_post;
routes.get_lora_adapters = models_routes->proxy_get;
routes.post_lora_adapters = models_routes->proxy_post;
routes.get_slots = models_routes->proxy_get;
routes.post_slots = models_routes->proxy_post;
// custom routes for router
routes.get_props = models_routes->get_router_props;
routes.get_models = models_routes->get_router_models;
ctx_http.post("/models", ex_wrapper(models_routes->post_router_models));
ctx_http.post("/models/load", ex_wrapper(models_routes->post_router_models_load));
ctx_http.post("/models/unload", ex_wrapper(models_routes->post_router_models_unload));
ctx_http.get ("/models/sse", ex_wrapper(models_routes->get_router_models_sse));
ctx_http.del ("/models", ex_wrapper(models_routes->del_router_models));
}
ctx_http.get ("/health", ex_wrapper(routes.get_health)); // public endpoint (no API key check)
ctx_http.get ("/v1/health", ex_wrapper(routes.get_health)); // public endpoint (no API key check)
ctx_http.get ("/metrics", ex_wrapper(routes.get_metrics));
ctx_http.get ("/props", ex_wrapper(routes.get_props));
ctx_http.post("/props", ex_wrapper(routes.post_props));
ctx_http.get ("/models", ex_wrapper(routes.get_models)); // public endpoint (no API key check)
ctx_http.get ("/v1/models", ex_wrapper(routes.get_models)); // public endpoint (no API key check)
ctx_http.post("/completion", ex_wrapper(routes.post_completions)); // legacy
ctx_http.post("/completions", ex_wrapper(routes.post_completions));
ctx_http.post("/v1/completions", ex_wrapper(routes.post_completions_oai));
ctx_http.post("/chat/completions", ex_wrapper(routes.post_chat_completions));
ctx_http.post("/v1/chat/completions", ex_wrapper(routes.post_chat_completions));
ctx_http.post("/v1/chat/completions/control", ex_wrapper(routes.post_control));
ctx_http.post("/v1/responses", ex_wrapper(routes.post_responses_oai));
ctx_http.post("/responses", ex_wrapper(routes.post_responses_oai));
ctx_http.post("/v1/audio/transcriptions", ex_wrapper(routes.post_transcriptions_oai));
ctx_http.post("/audio/transcriptions", ex_wrapper(routes.post_transcriptions_oai));
ctx_http.post("/v1/messages", ex_wrapper(routes.post_anthropic_messages)); // anthropic messages API
ctx_http.post("/infill", ex_wrapper(routes.post_infill));
ctx_http.post("/embedding", ex_wrapper(routes.post_embeddings)); // legacy
ctx_http.post("/embeddings", ex_wrapper(routes.post_embeddings));
ctx_http.post("/v1/embeddings", ex_wrapper(routes.post_embeddings_oai));
ctx_http.post("/rerank", ex_wrapper(routes.post_rerank));
ctx_http.post("/reranking", ex_wrapper(routes.post_rerank));
ctx_http.post("/v1/rerank", ex_wrapper(routes.post_rerank));
ctx_http.post("/v1/reranking", ex_wrapper(routes.post_rerank));
ctx_http.post("/tokenize", ex_wrapper(routes.post_tokenize));
ctx_http.post("/detokenize", ex_wrapper(routes.post_detokenize));
ctx_http.post("/apply-template", ex_wrapper(routes.post_apply_template));
// token counting
ctx_http.post("/chat/completions/input_tokens", ex_wrapper(routes.post_chat_completions_tok));
ctx_http.post("/v1/chat/completions/input_tokens", ex_wrapper(routes.post_chat_completions_tok));
ctx_http.post("/responses/input_tokens", ex_wrapper(routes.post_responses_tok_oai));
ctx_http.post("/v1/responses/input_tokens", ex_wrapper(routes.post_responses_tok_oai));
ctx_http.post("/v1/messages/count_tokens", ex_wrapper(routes.post_anthropic_count_tokens)); // anthropic token counting
// LoRA adapters hotswap
ctx_http.get ("/lora-adapters", ex_wrapper(routes.get_lora_adapters));
ctx_http.post("/lora-adapters", ex_wrapper(routes.post_lora_adapters));
// Save & load slots
ctx_http.get ("/slots", ex_wrapper(routes.get_slots));
ctx_http.post("/slots/:id_slot", ex_wrapper(routes.post_slots));
// resumable streaming, the conversation_id is the session identity end to end. router and
// child wire different handlers under the same paths: a child binds the local session
// factories, the router binds proxies that resolve the owning child through the
// conv_id -> model map
server_http_context::handler_t stream_get_h;
server_http_context::handler_t streams_lookup_h;
server_http_context::handler_t stream_delete_h;
if (is_router_server) {
stream_get_h = models_routes->router_stream_get;
streams_lookup_h = models_routes->router_streams_lookup;
stream_delete_h = models_routes->router_stream_delete;
} else {
stream_get_h = server_stream_make_get_handler();
streams_lookup_h = server_stream_make_lookup_handler();
stream_delete_h = server_stream_make_delete_handler();
}
ctx_http.get ("/v1/stream/:conv_id", ex_wrapper(stream_get_h));
// POST /v1/streams/lookup with body {"conversation_ids": [...]}. you can only ask for ids
// you already own (the WebUI passes the convs visible in its sidebar). the server never
// lists ids it has not been asked about, so a random caller cannot enumerate live sessions
ctx_http.post("/v1/streams/lookup", ex_wrapper(streams_lookup_h));
ctx_http.del ("/v1/stream/:conv_id", ex_wrapper(stream_delete_h));
// Google Cloud Platform (Vertex AI) compat
ctx_http.register_gcp_compat();
// return 403 for disabled features
server_http_context::handler_t res_403 = [](const server_http_req &) {
auto res = std::make_unique<server_http_res>();
res->status = 403;
res->data = safe_json_to_str({
{"error", {
{"message", "this feature is disabled"},
{"type", "feature_disabled"},
}}
});
return res;
};
// CORS proxy (EXPERIMENTAL, only used by the Web UI for MCP)
if (params.ui_mcp_proxy) {
SRV_WRN("%s", "-----------------\n");
SRV_WRN("%s", "CORS proxy is enabled, do not expose server to untrusted environments\n");
SRV_WRN("%s", "This feature is EXPERIMENTAL and may be removed or changed in future versions\n");
SRV_WRN("%s", "-----------------\n");
ctx_http.get ("/cors-proxy", ex_wrapper(proxy_handler_get));
ctx_http.post("/cors-proxy", ex_wrapper(proxy_handler_post));
} else {
ctx_http.get ("/cors-proxy", ex_wrapper(res_403));
ctx_http.post("/cors-proxy", ex_wrapper(res_403));
}
// EXPERIMENTAL built-in tools
if (!params.server_tools.empty()) {
try {
tools.setup(params.server_tools);
} catch (const std::exception & e) {
SRV_ERR("tools setup failed: %s\n", e.what());
return 1;
}
SRV_WRN("%s", "-----------------\n");
SRV_WRN("%s", "Built-in tools are enabled, do not expose server to untrusted environments\n");
SRV_WRN("%s", "This feature is EXPERIMENTAL and may be changed in the future\n");
SRV_WRN("%s", "-----------------\n");
ctx_http.get ("/tools", ex_wrapper(tools.handle_get));
ctx_http.post("/tools", ex_wrapper(tools.handle_post));
} else {
ctx_http.get ("/tools", ex_wrapper(res_403));
ctx_http.post("/tools", ex_wrapper(res_403));
}
//
// Handle downloading model
//
if (child.is_child() && child.get_mode() == SERVER_CHILD_MODE_DOWNLOAD) {
return child.run_download(params);
} else if (!is_router_server) {
// single-model mode (NOT spawned by router)
try {
common_models_handler_apply(models_handler, params);
} catch (const std::exception & e) {
SRV_ERR("failed to download model: %s\n", e.what());
return 1;
}
}
//
// Start the server
//
std::function<void()> clean_up;
if (is_router_server) {
SRV_INF("%s", "starting server in router mode. models will be automatically loaded on-demand\n");
clean_up = [&models_routes]() {
SRV_INF("%s: cleaning up before exit...\n", __func__);
// stop the session GC first, it finalizes live sessions and wakes pending readers
server_stream_session_manager_stop();
if (models_routes.has_value()) {
models_routes->stopping.store(true); // maybe redundant, but just to be safe
models_routes->models.unload_all();
}
llama_backend_free();
};
if (!ctx_http.start()) {
clean_up();
SRV_ERR("%s", "exiting due to HTTP server error\n");
return 1;
}
ctx_http.is_ready.store(true);
shutdown_handler = [&](int) {
if (models_routes.has_value()) {
// important to disconnect any SSE clients
models_routes->stopping.store(true);
}
ctx_http.stop();
};
} else {
// setup clean up function, to be called before exit
clean_up = [&ctx_http, &ctx_server]() {
SRV_INF("%s: cleaning up before exit...\n", __func__);
// stop the session GC first, it finalizes live sessions and wakes pending readers
server_stream_session_manager_stop();
ctx_http.stop();
ctx_server.terminate();
llama_backend_free();
};
// start the HTTP server before loading the model to be able to serve /health requests
if (!ctx_http.start()) {
clean_up();
SRV_ERR("%s", "exiting due to HTTP server error\n");
return 1;
}
// setup communication child --> router if necessary
if (child.is_child()) {
ctx_server.set_state_callback([&](server_state state, json payload) {
child.notify_to_router(server_state_to_str(state), payload);
});
}
if (!ctx_server.load_model(params)) {
clean_up();
if (ctx_http.thread.joinable()) {
ctx_http.thread.join();
}
SRV_ERR("%s", "exiting due to model loading error\n");
return 1;
}
routes.update_meta(ctx_server);
ctx_http.is_ready.store(true);
SRV_INF("%s", "model loaded\n");
shutdown_handler = [&](int) {
// this will unblock start_loop()
ctx_server.terminate();
};
}
// TODO: refactor in common/console
#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))
struct sigaction sigint_action;
sigint_action.sa_handler = signal_handler;
sigemptyset (&sigint_action.sa_mask);
sigint_action.sa_flags = 0;
sigaction(SIGINT, &sigint_action, NULL);
sigaction(SIGTERM, &sigint_action, NULL);
#elif defined (_WIN32)
auto console_ctrl_handler = +[](DWORD ctrl_type) -> BOOL {
return (ctrl_type == CTRL_C_EVENT) ? (signal_handler(SIGINT), true) : false;
};
SetConsoleCtrlHandler(reinterpret_cast<PHANDLER_ROUTINE>(console_ctrl_handler), true);
#endif
SRV_INF("listening on %s\n", ctx_http.listening_address.c_str());
if (is_router_server) {
SRV_WRN("%s", "NOTE: router mode is experimental\n");
SRV_WRN("%s", " it is not recommended to use this mode in untrusted environments\n");
if (!params.models_preset_hf.empty()) {
SRV_WRN( "NOTE: using preset.ini from HF repo '%s'\n", params.models_preset_hf.c_str());
SRV_WRN("%s", " please only use presets that you can trust! Unknown presets may be unsafe\n");
}
if (ctx_http.thread.joinable()) {
ctx_http.thread.join(); // keep the main thread alive
}
// when the HTTP server stops, clean up and exit
clean_up();
} else {
// optionally, notify router server that this instance is ready
std::thread monitor_thread;
if (child.is_child()) {
monitor_thread = child.setup(shutdown_handler);
child.notify_to_router(server_state_to_str(SERVER_STATE_READY), routes.get_model_info());
}
// this call blocks the main thread until queue_tasks.terminate() is called
ctx_server.start_loop();
clean_up();
if (ctx_http.thread.joinable()) {
ctx_http.thread.join();
}
if (monitor_thread.joinable()) {
monitor_thread.join();
}
auto * ll_ctx = ctx_server.get_llama_context();
if (ll_ctx != nullptr) {
common_memory_breakdown_print(ll_ctx);
}
}
return 0;
}
Reference in New Issue View Git Blame Copy Permalink