kanshan/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-07-08 20:20:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

server : add timings and progress to /responses API stream (#25348)

Browse Source
This commit is contained in:
Thomas LECONTE
2026-07-07 16:13:03 +02:00
committed by GitHub
parent 6c487e2f79
commit 5eca4e3cab
3 changed files with 76 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-task.cpp
+33 -1
View File
@@ -730,6 +730,10 @@ json server_task_result_cmpl_final::to_json_oaicompat_resp_stream() {
}}
});
if (timings.prompt_n >= 0) {
server_sent_events.back().at("data").push_back({"timings", timings.to_json()});
}
return server_sent_events;
}
@@ -1016,6 +1020,7 @@ void server_task_result_cmpl_partial::update(task_result_state & state) {
thinking_block_started = state.thinking_block_started;
text_block_started = state.text_block_started;
oai_resp_created = state.oai_resp_created;
oai_resp_id = state.oai_resp_id;
oai_resp_reasoning_id = state.oai_resp_reasoning_id;
oai_resp_message_id = state.oai_resp_message_id;
@@ -1024,6 +1029,10 @@ void server_task_result_cmpl_partial::update(task_result_state & state) {
// track if the accumulated message has any reasoning content
anthropic_has_reasoning = !state.chat_msg.reasoning_content.empty();
if (res_type == TASK_RESPONSE_TYPE_OAI_RESP && !state.oai_resp_created && (is_progress || n_decoded == 1)) {
state.oai_resp_created = true;
}
// Pre-compute state updates based on diffs (for next chunk)
for (const common_chat_msg_diff & diff : oaicompat_msg_diffs) {
if (!diff.reasoning_content_delta.empty() && !state.thinking_block_started) {
@@ -1181,7 +1190,7 @@ json server_task_result_cmpl_partial::to_json_oaicompat_chat() {
json server_task_result_cmpl_partial::to_json_oaicompat_resp() {
std::vector<json> events;
if (n_decoded == 1) {
if (!oai_resp_created) {
events.push_back(json {
{"event", "response.created"},
{"data", json {
@@ -1204,6 +1213,18 @@ json server_task_result_cmpl_partial::to_json_oaicompat_resp() {
}},
}},
});
} else if (is_progress) {
events.push_back(json {
{"event", "response.in_progress"},
{"data", json {
{"type", "response.in_progress"},
{"response", json {
{"id", oai_resp_id},
{"object", "response"},
{"status", "in_progress"},
}},
}},
});
}
for (const common_chat_msg_diff & diff : oaicompat_msg_diffs) {
@@ -1302,6 +1323,17 @@ json server_task_result_cmpl_partial::to_json_oaicompat_resp() {
});
}
}
if (!events.empty()) {
json & data = events.back().at("data");
if (timings.prompt_n >= 0) {
data.push_back({"timings", timings.to_json()});
}
if (is_progress) {
data.push_back({"prompt_progress", progress.to_json()});
}
}
return events;
}