diff --git a/common/arg.cpp b/common/arg.cpp index 202ebcb0f0..8a78291658 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -2849,7 +2849,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.out_file = value; } ).set_examples({LLAMA_EXAMPLE_IMATRIX, LLAMA_EXAMPLE_CVECTOR_GENERATOR, LLAMA_EXAMPLE_EXPORT_LORA, LLAMA_EXAMPLE_TTS, LLAMA_EXAMPLE_FINETUNE, - LLAMA_EXAMPLE_RESULTS, LLAMA_EXAMPLE_EXPORT_GRAPH_OPS})); + LLAMA_EXAMPLE_RESULTS, LLAMA_EXAMPLE_EXPORT_GRAPH_OPS, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( {"-ofreq", "--output-frequency"}, "N", string_format("output the imatrix every N iterations (default: %d)", params.n_out_freq), diff --git a/tools/cli/cli-context.cpp b/tools/cli/cli-context.cpp index 78b1b102f1..74d60eb19d 100644 --- a/tools/cli/cli-context.cpp +++ b/tools/cli/cli-context.cpp @@ -162,6 +162,14 @@ bool cli_context::init() { fetch_server_props(); + if (!params.out_file.empty()) { + output_file.emplace(params.out_file); + if (!output_file->is_open()) { + ui::show_error(string_format("failed to open output file '%s'", params.out_file.c_str())); + return false; + } + } + return true; } @@ -323,7 +331,14 @@ bool cli_context::stage_media_file(const std::string & fname, const std::string return true; } -bool cli_context::generate_completion(std::string & assistant_content, cli_timings & timings) { +void cli_context::write_output_file(const std::string & content) { + if (output_file) { + (*output_file) << content; + output_file->flush(); + } +} + +bool cli_context::generate_completion(generated_content & content_out, cli_timings & timings) { json body = { {"messages", impl->messages}, {"stream", true}, @@ -364,13 +379,14 @@ bool cli_context::generate_completion(std::string & assistant_content, cli_timin if (delta.contains("reasoning_content") && delta.at("reasoning_content").is_string()) { const std::string text = delta.at("reasoning_content").get(); if (!text.empty()) { + content_out.reasoning += text; a.push(ui::ASSISTANT_DISPLAY_MODE_REASONING, text); } } if (delta.contains("content") && delta.at("content").is_string()) { const std::string text = delta.at("content").get(); if (!text.empty()) { - assistant_content += text; + content_out.content += text; a.push(ui::ASSISTANT_DISPLAY_MODE_CONTENT, text); } } @@ -520,10 +536,12 @@ int cli_context::run() { continue; } ui::show_message(string_format("Loaded media from '%s'", fname.c_str())); + write_output_file(string_format("User: Added media: %s\n", fname.c_str())); continue; } else if (string_starts_with(buffer, "/read ")) { std::string fname = string_strip(buffer.substr(6)); add_text_file(fname); + write_output_file(string_format("User: Added text file: %s\n", fname.c_str())); continue; } else if (string_starts_with(buffer, "/glob ")) { std::error_code ec; @@ -568,9 +586,11 @@ int cli_context::run() { continue; } - if (!add_text_file((rel_path / rel).string())) { + const std::string full_path = (curdir / rel).string(); + if (!add_text_file(full_path)) { continue; } + write_output_file(string_format("User: Added text file: %s\n", full_path.c_str())); if (++count >= FILE_GLOB_MAX_RESULTS) { ui::show_error(string_format("Maximum number of globbed files allowed (%zu) reached.", FILE_GLOB_MAX_RESULTS)); @@ -586,16 +606,34 @@ int cli_context::run() { // generate response if (add_user_msg) { push_user_message(cur_msg); + write_output_file(string_format("User:\n%s\n\n", cur_msg.c_str())); cur_msg.clear(); } + cli_timings timings; - std::string assistant_content; - generate_completion(assistant_content, timings); + generated_content content; + generate_completion(content, timings); + impl->messages.push_back({ {"role", "assistant"}, - {"content", assistant_content} + {"content", content.content} }); + if (output_file) { + std::string out_content = "Assistant:\n"; + if (!content.reasoning.empty()) { + out_content += "[Start thinking]\n\n"; + out_content += content.reasoning; + out_content += "[End thinking]\n\n"; + } + out_content += content.content; + if (!out_content.empty() && out_content.back() != '\n') { + out_content += "\n"; + } + out_content += "\n"; + write_output_file(out_content); + } + if (params.show_timings) { ui::show_info(string_format( "\n[ Prompt: %.1f t/s | Generation: %.1f t/s ]", @@ -619,4 +657,8 @@ void cli_context::shutdown() { server->stop(); server.reset(); } + if (output_file) { + output_file->close(); + output_file.reset(); + } } diff --git a/tools/cli/cli-context.h b/tools/cli/cli-context.h index 79694fb6c0..15ce4efee6 100644 --- a/tools/cli/cli-context.h +++ b/tools/cli/cli-context.h @@ -9,6 +9,7 @@ #include #include #include +#include struct cli_timings { double prompt_per_second = 0.0; @@ -32,6 +33,8 @@ struct cli_context { bool has_audio = false; bool has_video = false; + std::optional output_file; + cli_context(const common_params & params); ~cli_context(); @@ -49,7 +52,11 @@ struct cli_context { static std::atomic & interrupted(); private: - bool generate_completion(std::string & assistant_content, cli_timings & timings); + struct generated_content { + std::string reasoning; + std::string content; + }; + bool generate_completion(generated_content & content_out, cli_timings & timings); void fetch_server_props(); void add_system_prompt(); void push_user_message(const std::string & text); @@ -62,5 +69,8 @@ private: // "image", "audio", "video"; returns false if the file cannot be read bool stage_media_file(const std::string & fname, const std::string & type); + // no-op if output file is not set + void write_output_file(const std::string & content); + std::unique_ptr impl; };