kanshan/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-07-09 12:40:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

server: add --models-memory-max parameter to allow dynamically unloading models when they exceed a memory size threshold

Browse Source 
estimate with to-be-loaded model size included

use no_alloc to get memory requirements for model load

only set model memory_mb if not previously calculated

use memory margin instead of total size limit, apply to each device separately

add server memory debug logging

move llama_context_device_memory function to llama-ext.h

fix model count exceeded check

improve memory_per_device map naming

improve variable naming, fix style

also strip models memory margin from child processes

cont : clean-up

replace device memory map with buft memory map. Use llama_get_memory_breakdown

extract duplicated check into helper function

move model memory estimation to subprocess

precompute name->buft map, map GPU host types to CPU buft

cleanup unused variable

remove duplicated init calls
This commit is contained in:
Ruben Ortlam
2026-03-29 10:00:49 +02:00
parent 277a105dc8
commit e4d2e198b9
6 changed files with 331 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server.cpp
+40
View File
@@ -12,6 +12,8 @@
#include "llama.h"
#include "log.h"
#include "../../src/llama-ext.h"
#include <atomic>
#include <clocale>
#include <exception>
@@ -140,6 +142,44 @@ int llama_server(int argc, char ** argv) {
// struct that contains llama context and inference
server_context ctx_server;
if (params.measure_only) {
llama_model_params mparams = common_model_params_to_llama(params);
mparams.no_alloc = true;
mparams.use_mmap = false;
mparams.use_mlock = false;
llama_model_ptr model{llama_model_load_from_file(params.model.path.c_str(), mparams)};
if (!model) {
LOG_ERR("%s: failed to load model for measurement\n", __func__);
llama_backend_free();
return 1;
}
llama_context_params cparams = common_context_params_to_llama(params);
llama_context_ptr ctx{llama_init_from_model(model.get(), cparams)};
if (!ctx) {
LOG_ERR("%s: failed to create context for measurement\n", __func__);
llama_backend_free();
return 1;
}
common_log_pause(common_log_main());
for (const auto & [buft, data] : llama_get_memory_breakdown(ctx.get())) {
size_t total = data.total();
if (total > 0) {
fprintf(stdout, "measure:%s %zu\n", ggml_backend_buft_name(buft), total);
}
}
fflush(stdout);
common_log_resume(common_log_main());
llama_backend_free();
return 0;
}
LOG_INF("build_info: %s\n", llama_build_info());
LOG_INF("%s\n", common_params_get_system_info(params).c_str());
server_http_context ctx_http;
if (!ctx_http.init(params)) {
SRV_ERR("%s", "failed to initialize HTTP server\n");