diff --git a/tools/cli/cli-context.cpp b/tools/cli/cli-context.cpp index 646c5daf3a..d432bdc7a2 100644 --- a/tools/cli/cli-context.cpp +++ b/tools/cli/cli-context.cpp @@ -12,10 +12,24 @@ #include #include -std::atomic g_cli_interrupted = false; +struct cli_context_impl { + json messages = json::array(); + json pending_media = json::array(); // staged multimodal content parts +}; + +cli_context::cli_context(const common_params & params) : params(params), impl(new cli_context_impl()) {} + +cli_context::~cli_context() { + shutdown(); +} + +std::atomic & cli_context::interrupted() { + static std::atomic flag = false; + return flag; +} static bool should_stop() { - return g_cli_interrupted.load(); + return cli_context::interrupted().load(); } static constexpr size_t FILE_GLOB_MAX_RESULTS = 100; @@ -208,7 +222,7 @@ bool cli_context::list_and_ask_models() { void cli_context::add_system_prompt() { if (!params.system_prompt.empty()) { - messages.push_back({ + impl->messages.push_back({ {"role", "system"}, {"content", params.system_prompt} }); @@ -217,18 +231,18 @@ void cli_context::add_system_prompt() { void cli_context::push_user_message(const std::string & text) { json content; - if (pending_media.empty()) { + if (impl->pending_media.empty()) { content = text; } else { // multimodal message: media parts first, then the text - content = pending_media; + content = impl->pending_media; content.push_back({ {"type", "text"}, {"text", text} }); - pending_media = json::array(); + impl->pending_media = json::array(); } - messages.push_back({ + impl->messages.push_back({ {"role", "user"}, {"content", content} }); @@ -245,7 +259,7 @@ bool cli_context::stage_media_file(const std::string & fname, const std::string if (type == "audio") { std::string ext = std::filesystem::path(fname).extension().string(); std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower); - pending_media.push_back({ + impl->pending_media.push_back({ {"type", "input_audio"}, {"input_audio", { {"data", encoded}, @@ -253,7 +267,7 @@ bool cli_context::stage_media_file(const std::string & fname, const std::string }} }); } else if (type == "video") { - pending_media.push_back({ + impl->pending_media.push_back({ {"type", "input_video"}, {"input_video", { {"data", encoded} @@ -261,7 +275,7 @@ bool cli_context::stage_media_file(const std::string & fname, const std::string }); } else { // the server detects the actual image type from the data - pending_media.push_back({ + impl->pending_media.push_back({ {"type", "image_url"}, {"image_url", { {"url", "data:image/unknown;base64," + encoded} @@ -273,7 +287,7 @@ bool cli_context::stage_media_file(const std::string & fname, const std::string bool cli_context::generate_completion(std::string & assistant_content, cli_timings & timings) { json body = { - {"messages", messages}, + {"messages", impl->messages}, {"stream", true}, // in order to get timings even when we cancel mid-way {"timings_per_token", true}, @@ -317,7 +331,7 @@ bool cli_context::generate_completion(std::string & assistant_content, cli_timin } }); - g_cli_interrupted.store(false); + cli_context::interrupted().store(false); if (!err.is_null()) { ui::show_error(format_error_message(err)); @@ -414,7 +428,7 @@ int cli_context::run() { } if (should_stop()) { - g_cli_interrupted.store(false); + cli_context::interrupted().store(false); break; } @@ -434,19 +448,19 @@ int cli_context::run() { if (string_starts_with(buffer, "/exit")) { break; } else if (string_starts_with(buffer, "/regen")) { - if (messages.size() >= 2) { - size_t last_idx = messages.size() - 1; - messages.erase(last_idx); + if (impl->messages.size() >= 2) { + size_t last_idx = impl->messages.size() - 1; + impl->messages.erase(last_idx); add_user_msg = false; } else { ui::show_error("No message to regenerate."); continue; } } else if (string_starts_with(buffer, "/clear")) { - messages.clear(); + impl->messages.clear(); add_system_prompt(); - pending_media = json::array(); + impl->pending_media = json::array(); ui::show_message("Chat history cleared."); continue; } else if ( @@ -532,7 +546,7 @@ int cli_context::run() { cli_timings timings; std::string assistant_content; generate_completion(assistant_content, timings); - messages.push_back({ + impl->messages.push_back({ {"role", "assistant"}, {"content", assistant_content} }); diff --git a/tools/cli/cli-context.h b/tools/cli/cli-context.h index a4d504227b..79694fb6c0 100644 --- a/tools/cli/cli-context.h +++ b/tools/cli/cli-context.h @@ -6,6 +6,7 @@ #include "cli-server.h" #include +#include #include #include @@ -14,8 +15,7 @@ struct cli_timings { double predicted_per_second = 0.0; }; -// set by the SIGINT handler; cleared once the interrupt has been handled -extern std::atomic g_cli_interrupted; +struct cli_context_impl; struct cli_context { common_params params; @@ -23,9 +23,6 @@ struct cli_context { cli_client client; // always initialized std::optional server; // only set when no --server-base is given - json messages = json::array(); - json pending_media = json::array(); // staged multimodal content parts - // properties of the connected server // will be populated by fetch_server_props() std::string model_name; @@ -35,10 +32,8 @@ struct cli_context { bool has_audio = false; bool has_video = false; - cli_context(const common_params & params) : params(params) {} - ~cli_context() { - shutdown(); - } + cli_context(const common_params & params); + ~cli_context(); // connect to --server-base or spawn a local llama-server child; // argc/argv are needed to forward the server-relevant args to the child @@ -50,6 +45,9 @@ struct cli_context { // stop the local server child (if any) void shutdown(); + // set by the SIGINT handler; cleared once the interrupt has been handled + static std::atomic & interrupted(); + private: bool generate_completion(std::string & assistant_content, cli_timings & timings); void fetch_server_props(); @@ -63,4 +61,6 @@ private: // read a file and stage it as a multimodal content part; type is one of // "image", "audio", "video"; returns false if the file cannot be read bool stage_media_file(const std::string & fname, const std::string & type); + + std::unique_ptr impl; }; diff --git a/tools/cli/cli.cpp b/tools/cli/cli.cpp index b361553ade..dcdb6aeac2 100644 --- a/tools/cli/cli.cpp +++ b/tools/cli/cli.cpp @@ -16,14 +16,14 @@ #if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) || defined (_WIN32) static void signal_handler(int) { - if (g_cli_interrupted.load()) { + if (cli_context::interrupted().load()) { // second Ctrl+C - exit immediately // make sure to clear colors before exiting (not using LOG or console.cpp here to avoid deadlock) fprintf(stdout, "\033[0m\n"); fflush(stdout); std::exit(130); } - g_cli_interrupted.store(true); + cli_context::interrupted().store(true); } #endif