kanshan/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-07-05 18:50:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

chat: trim messages sent to StepFun parser (fixes long reasoning loops) (#25238)

Browse Source 
* chat: trim messages sent to StepFun parser (fixes long reasoning loops)

* add regression test; remove duplicate template

* chat: trim StepFun content parts before rendering

The StepFun trim workaround ran on the already-rendered messages, where
typed content parts have been concatenated into a single string, so the
per-part whitespace could no longer be reached. Move the trim ahead of
rendering and apply it to content_parts text as well as the string
content and reasoning_content. Adds a content-parts regression test.

Co-Authored-By: Piotr Wilkin <ilintar@gmail.com>
Assisted-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: tarruda <tpadilha84@gmail.com>
This commit is contained in:
Piotr Wilkin (ilintar)
2026-07-03 23:12:11 +02:00
committed by GitHub
parent d4cff114c0
commit 2d973636e2
4 changed files with 80 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/chat.cpp
+27 -1
View File
@@ -2378,6 +2378,23 @@ static void func_args_not_string(json & messages) {
}
}
// Trim leading/trailing whitespace from message contents before rendering. This
// has to run on the messages (not on the rendered JSON) because templates with
// string-only content caps concatenate typed content parts into a single string
// during rendering, after which the per-part whitespace can no longer be reached.
// Both the plain string content and the text of typed content parts are trimmed.
static void trim_all_content(std::vector<common_chat_msg> & messages) {
for (auto & message : messages) {
message.content = trim_whitespace(message.content);
message.reasoning_content = trim_whitespace(message.reasoning_content);
for (auto & part : message.content_parts) {
if (part.type == "text") {
part.text = trim_whitespace(part.text);
}
}
}
}
}
// MiniCPM5 format:
@@ -2634,7 +2651,16 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_
params.tools.is_array() && tmpls->template_tool_use ? *tmpls->template_tool_use : *tmpls->template_default;
const auto & src = tmpl.source();
const auto & caps = tmpl.original_caps();
params.messages = render_message_to_json(inputs.messages, tmpl.original_caps());
std::vector<common_chat_msg> trimmed_messages;
const std::vector<common_chat_msg> * messages_to_render = &inputs.messages;
if (src.find("You have access to the following functions in JSONSchema format") != std::string::npos) {
// StepFun: trim message contents (including typed content parts) before rendering,
// otherwise leftover whitespace drives the model into reasoning loops (issue #24181)
trimmed_messages = inputs.messages;
workaround::trim_all_content(trimmed_messages);
messages_to_render = &trimmed_messages;
}
params.messages = render_message_to_json(*messages_to_render, tmpl.original_caps());
params.tool_choice = inputs.tool_choice;
params.reasoning_format = inputs.reasoning_format;
params.enable_thinking = inputs.enable_thinking;
models/templates/stepfun-ai-Step-3.5-Flash.jinja
-80
View File
@@ -1,80 +0,0 @@
{% macro render_content(content) %}{% if content is none %}{{- '' }}{% elif content is string %}{{- content }}{% elif content is mapping %}{{- content['value'] if 'value' in content else content['text'] }}{% elif content is iterable %}{% for item in content %}{% if item.type == 'text' %}{{- item['value'] if 'value' in item else item['text'] }}{% elif item.type == 'image' %}<im_patch>{% endif %}{% endfor %}{% endif %}{% endmacro %}
{{bos_token}}{%- if tools %}
{{- '<|im_start|>system\n' }}
{%- if messages[0].role == 'system' %}
{{- render_content(messages[0].content) + '\n\n' }}
{%- endif %}
{{- "# Tools\n\nYou have access to the following functions in JSONSchema format:\n\n<tools>" }}
{%- for tool in tools %}
{{- "\n" }}
{{- tool | tojson(ensure_ascii=False) }}
{%- endfor %}
{{- "\n</tools>\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...>\n...\n</function> block must be nested within <tool_call>\n...\n</tool_call> XML tags\n- Required parameters MUST be specified\n</IMPORTANT><|im_end|>\n" }}
{%- else %}
{%- if messages[0].role == 'system' %}
{{- '<|im_start|>system\n' + render_content(messages[0].content) + '<|im_end|>\n' }}
{%- endif %}
{%- endif %}
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
{%- for message in messages[::-1] %}
{%- set index = (messages|length - 1) - loop.index0 %}
{%- if ns.multi_step_tool and message.role == "user" and render_content(message.content) is string and not(render_content(message.content).startswith('<tool_response>') and render_content(message.content).endswith('</tool_response>')) %}
{%- set ns.multi_step_tool = false %}
{%- set ns.last_query_index = index %}
{%- endif %}
{%- endfor %}
{%- for message in messages %}
{%- set content = render_content(message.content) %}
{%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
{%- set role_name = 'observation' if (message.role == "system" and not loop.first and message.name == 'observation') else message.role %}
{{- '<|im_start|>' + role_name + '\n' + content + '<|im_end|>' + '\n' }}
{%- elif message.role == "assistant" %}
{%- if message.reasoning_content is string %}
{%- set reasoning_content = render_content(message.reasoning_content) %}
{%- else %}
{%- if '</think>' in content %}
{%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
{%- set content = content.split('</think>')[-1].lstrip('\n') %}
{%- else %}
{%- set reasoning_content = '' %}
{%- endif %}
{%- endif %}
{%- if loop.index0 > ns.last_query_index %}
{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n' + content }}
{%- else %}
{{- '<|im_start|>' + message.role + '\n' + content }}
{%- endif %}
{%- if message.tool_calls %}
{%- for tool_call in message.tool_calls %}
{%- if tool_call.function is defined %}
{%- set tool_call = tool_call.function %}
{%- endif %}
{{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
{%- if tool_call.arguments is defined %}
{%- set arguments = tool_call.arguments %}
{%- for args_name, args_value in arguments|items %}
{{- '<parameter=' + args_name + '>\n' }}
{%- set args_value = args_value | tojson(ensure_ascii=False) | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
{{- args_value }}
{{- '\n</parameter>\n' }}
{%- endfor %}
{%- endif %}
{{- '</function>\n</tool_call>' }}
{%- endfor %}
{%- endif %}
{{- '<|im_end|>\n' }}
{%- elif message.role == "tool" %}
{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
{{- '<|im_start|>tool_response\n' }}
{%- endif %}
{{- '<tool_response>' }}
{{- content }}
{{- '</tool_response>' }}
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
{{- '<|im_end|>\n' }}
{%- endif %}
{%- endif %}
{%- endfor %}
{%- if add_generation_prompt %}
{{- '<|im_start|>assistant\n<think>\n' }}
{%- endif %}
tests/test-chat-auto-parser.cpp
-1
View File
@@ -1887,7 +1887,6 @@ static void test_role_markers_all_templates(testing & t) {
{ "Qwen-Qwen3-0.6B.jinja", "<|im_start|>user", "<|im_start|>assistant" },
{ "Qwen-QwQ-32B.jinja", "<|im_start|>user", "<|im_start|>assistant" },
{ "StepFun3.5-Flash.jinja", "<|im_start|>user", "<|im_start|>assistant" },
{ "stepfun-ai-Step-3.5-Flash.jinja", "<|im_start|>user", "<|im_start|>assistant" },
// DeepSeek family
{ "deepseek-ai-DeepSeek-R1-Distill-Llama-8B.jinja", "<｜User｜>", "<｜Assistant｜>" },
tests/test-chat.cpp
+53
View File
@@ -3155,6 +3155,59 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
}
}
}
{
// StepFun trimming regression test (see https://github.com/ggml-org/llama.cpp/pull/25238)
auto tmpls = read_templates("models/templates/StepFun3.5-Flash.jinja");
common_chat_msg message_chatbot = simple_assist_msg("Let me check.\n\n", "I am thinking.\n\n");
{
common_chat_templates_inputs inputs;
inputs.messages = { message_chatbot };
inputs.add_generation_prompt = true;
auto params = common_chat_templates_apply(tmpls.get(), inputs);
if (params.prompt.find("Let me check.\n\n") != std::string::npos) {
throw std::runtime_error("StepFun 3.5: content not trimmed");
}
if (params.prompt.find("I am thinking.\n\n") != std::string::npos) {
throw std::runtime_error("StepFun 3.5: reasoning_content not trimmed");
}
}
{
// Trimming must also reach typed (text) content parts, not just string content
// (see https://github.com/ggml-org/llama.cpp/pull/25238)
common_chat_msg message_parts;
message_parts.role = "user";
message_parts.content_parts = {
{ /* .type = */ "text", /* .text = */ "First part.\n\n" },
{ /* .type = */ "media_marker", /* .text = */ "<__media__>" },
{ /* .type = */ "text", /* .text = */ "Second part.\n\n" },
};
common_chat_templates_inputs inputs;
inputs.messages = { message_parts };
inputs.add_generation_prompt = true;
auto params = common_chat_templates_apply(tmpls.get(), inputs);
if (params.prompt.find("First part.\n\n") != std::string::npos ||
params.prompt.find("Second part.\n\n") != std::string::npos) {
throw std::runtime_error("StepFun 3.5: text content parts not trimmed");
}
// the trimmed text itself must still be present
if (params.prompt.find("First part.") == std::string::npos ||
params.prompt.find("Second part.") == std::string::npos) {
throw std::runtime_error("StepFun 3.5: text content parts missing after trim");
}
}
}
}
{