obscura/vllm
2
0
Fork 0
mirror of https://github.com/vllm-project/vllm.git synced 2026-07-29 09:58:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

[Model] Add Apertus Tool Parser (#41154)

Browse Source 
Signed-off-by: Blanc <[email protected]>
This commit is contained in:
Blanc Swan
2026-05-18 11:20:04 -04:00
committed by GitHub
parent 78e7a7b9b0
commit 4a39b4f553
5 changed files with 1385 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/features/tool_calling.md
+11
View File
@@ -464,6 +464,17 @@ Supported models:
Flags: `--tool-call-parser gigachat3`
### Apertus Models (`apertus`)
Use the chat template from the examples folder; it fixes several OpenAI compatibility issues: `--chat-template /vllm-workspace/examples/tool_chat_template_apertus.jinja`
Supported models:
* `swiss-ai/Apertus-8B-Instruct-2509`
* `swiss-ai/Apertus-70B-Instruct-2509`
Flags: `--tool-call-parser apertus`
### Models with Pythonic Tool Calls (`pythonic`)
A growing number of models output a python list to represent tool calls instead of using JSON. This has the advantage of inherently supporting parallel tool calls and removing ambiguity around the JSON schema required for tool calls. The `pythonic` tool parser can support such models.
examples/tool_chat_template_apertus.jinja
+384
View File
@@ -0,0 +1,384 @@
{%- macro render_typescript_type(param_spec, required_params, is_nullable=false) -%}
{%- if param_spec.type == "array" -%}
{%- if param_spec['items'] -%}
{%- if param_spec['items']['type'] == "string" -%}
{{- "string[]" }}
{%- elif param_spec['items']['type'] == "number" -%}
{{- "number[]" }}
{%- elif param_spec['items']['type'] == "integer" -%}
{{- "number[]" }}
{%- elif param_spec['items']['type'] == "boolean" -%}
{{- "boolean[]" }}
{%- else -%}
{%- set inner_type = render_typescript_type(param_spec['items'], required_params) -%}
{%- if inner_type == "object | object" or inner_type|length > 50 -%}
{{- "any[]" }}
{%- else -%}
{{- inner_type + "[]" }}
{%- endif -%}
{%- endif -%}
{%- if param_spec.nullable -%}
{{- " | null" }}
{%- endif -%}
{%- else -%}
{{- "any[]" }}
{%- if param_spec.nullable -%}
{{- " | null" }}
{%- endif -%}
{%- endif -%}
{%- elif param_spec.type is defined and param_spec.type is iterable and param_spec.type is not string and param_spec.type is not mapping and param_spec.type[0] is defined -%}
{#- Handle array of types like ["object", "object"] from Union[dict, list] #}
{%- if param_spec.type | length > 1 -%}
{{- param_spec.type | join(" | ") }}
{%- else -%}
{{- param_spec.type[0] }}
{%- endif -%}
{%- elif param_spec.oneOf -%}
{#- Handle oneOf schemas - check for complex unions and fallback to any #}
{%- set has_object_variants = false -%}
{%- for variant in param_spec.oneOf -%}
{%- if variant.type == "object" -%}
{%- set has_object_variants = true -%}
{%- endif -%}
{%- endfor -%}
{%- if has_object_variants and param_spec.oneOf|length > 1 -%}
{{- "any" }}
{%- else -%}
{%- for variant in param_spec.oneOf -%}
{{- render_typescript_type(variant, required_params) -}}
{%- if variant.description %}
{{- "// " + variant.description }}
{%- endif -%}
{%- if variant.default is defined %}
{{ "// default: " + variant.default|tojson }}
{%- endif -%}
{%- if not loop.last %}
{{- " | " }}
{% endif -%}
{%- endfor -%}
{%- endif -%}
{%- elif param_spec.type == "string" -%}
{%- if param_spec.enum -%}
{{- '"' + param_spec.enum|join('" | "') + '"' -}}
{%- else -%}
{{- "string" }}
{%- if param_spec.nullable %}
{{- " | null" }}
{%- endif -%}
{%- endif -%}
{%- elif param_spec.type == "number" -%}
{{- "number" }}
{%- elif param_spec.type == "integer" -%}
{{- "number" }}
{%- elif param_spec.type == "boolean" -%}
{{- "boolean" }}
{%- elif param_spec.type == "object" -%}
{%- if param_spec.properties -%}
{{- "{\n" }}
{%- for prop_name, prop_spec in param_spec.properties.items() -%}
{{- prop_name -}}
{%- if prop_name not in (param_spec.required or []) -%}
{{- "?" }}
{%- endif -%}
{{- ": " }}
{{ render_typescript_type(prop_spec, param_spec.required or []) }}
{%- if not loop.last -%}
{{-", " }}
{%- endif -%}
{%- endfor -%}
{{- "}" }}
{%- else -%}
{{- "object" }}
{%- endif -%}
{%- else -%}
{{- "any" }}
{%- endif -%}
{%- endmacro -%}
{%- macro render_tools(tools) -%}
{%- for tool in tools %}
{%- if tool.function is defined -%}
{#- Chat Completions format: {"type": "function", "function": {...}} #}
{%- set func = tool.function -%}
{%- if func.description is defined -%}
{{- "// " + func.description + "\n" }}
{%- endif -%}
{{- "type "+ func.name + " = " }}
{%- if func.parameters and func.parameters.properties %}
{{- "(_: {\n" }}
{%- for param_name, param_spec in func.parameters.properties.items() %}
{%- if param_spec.description is defined %}
{{- "// " + param_spec.description + "\n" }}
{%- endif %}
{{- param_name }}
{%- if param_name not in (func.parameters.required or []) -%}
{{- "?" }}
{%- endif -%}
{{- ": " }}
{{- render_typescript_type(param_spec, func.parameters.required or []) }}
{%- if param_spec.default is defined -%}
{%- if param_spec.enum %}
{{- ", // default: " + param_spec.default }}
{%- elif param_spec.oneOf %}
{{- "// default: " + param_spec.default }}
{%- else %}
{{- ", // default: " + param_spec.default|tojson }}
{%- endif -%}
{%- endif -%}
{%- if not loop.last %}
{{- ",\n" }}
{%- else %}
{{- "\n" }}
{%- endif -%}
{%- endfor %}
{{- "}) => any;" }}
{%- else -%}
{{- "() => any;" }}
{%- endif -%}
{%- else -%}
{#- Responses format: {"type": "function", "name": "...", ...} #}
{%- if tool.description is defined -%}
{{- "// " + tool.description + "\n" }}
{%- endif -%}
{{- "type "+ tool.name + " = " }}
{%- if tool.parameters and tool.parameters.properties %}
{{- "(_: {\n" }}
{%- for param_name, param_spec in tool.parameters.properties.items() %}
{%- if param_spec.description is defined %}
{{- "// " + param_spec.description + "\n" }}
{%- endif %}
{{- param_name }}
{%- if param_name not in (tool.parameters.required or []) -%}
{{- "?" }}
{%- endif -%}
{{- ": " }}
{{- render_typescript_type(param_spec, tool.parameters.required or []) }}
{%- if param_spec.default is defined -%}
{%- if param_spec.enum %}
{{- ", // default: " + param_spec.default }}
{%- elif param_spec.oneOf %}
{{- "// default: " + param_spec.default }}
{%- else %}
{{- ", // default: " + param_spec.default|tojson }}
{%- endif -%}
{%- endif -%}
{%- if not loop.last %}
{{- ",\n" }}
{%- else %}
{{- "\n" }}
{%- endif -%}
{%- endfor %}
{{- "}) => any;" }}
{%- else -%}
{{- "() => any;" }}
{%- endif -%}
{%- endif -%}
{%- if not loop.last -%}
{{- "\n" }}
{%- endif -%}
{%- endfor %}
{%- endmacro -%}
{{ bos_token }}
{%- set system_token = '<|system_start|>' -%}
{%- set end_system_token = '<|system_end|>' -%}
{%- set developer_token = '<|developer_start|>' -%}
{%- set end_developer_token = '<|developer_end|>' -%}
{%- set user_token = '<|user_start|>' -%}
{%- set end_user_token = '<|user_end|>' -%}
{%- set assistant_token = '<|assistant_start|>' -%}
{%- set end_assistant_token = '<|assistant_end|>' -%}
{%- set inner_token = '<|inner_prefix|>' -%}
{%- set outer_token = '<|inner_suffix|>' -%}
{%- set tool_calls_token = '<|tools_prefix|>' -%}
{%- set end_tool_calls_token = '<|tools_suffix|>' -%}
{%- set ns = namespace(in_assistant=false, in_tool=false, in_inner=false, assistant_format=none) -%}
{%- if messages and messages[0].role == 'system' -%}
{%- if "content" in messages[0] -%}
{%- if messages[0].content is string -%}
{{ system_token + messages[0].content + end_system_token }}
{%- elif messages[0].content is mapping and "text" in messages[0].content -%}
{{ system_token + messages[0].content.text + end_system_token }}
{%- else -%}
{{- raise_exception("Invalid system message") -}}
{%- endif -%}
{%- else -%}
{{- raise_exception("Invalid system message") -}}
{%- endif -%}
{%- set loop_messages = messages[1:] -%}
{%- else -%}
{{ system_token + 'You are Apertus, a helpful assistant created by the SwissAI initiative.\nKnowledge cutoff: 2024-04\nCurrent date: ' + strftime_now('%Y-%m-%d') + end_system_token }}
{%- set loop_messages = messages -%}
{%- endif -%}
{{ developer_token + 'Deliberation: ' }}
{%- if enable_thinking is defined and enable_thinking -%}
{{ 'enabled\n' }}
{%- else -%}
{{ 'disabled\n' }}
{%- endif -%}
{%- if tools is defined and tools -%}
{{ 'Tool Capabilities:\n' + render_tools(tools) }}
{%- else -%}
{{ 'Tool Capabilities: disabled' }}
{%- endif -%}
{{ end_developer_token }}
{%- for message in loop_messages -%}
{%- if message.role == 'user' -%}
{%- set ns.in_inner = false -%}
{%- if ns.in_tool -%}
{{ ']' }}
{%- set ns.in_tool = false -%}
{%- endif -%}
{%- if ns.in_assistant -%}
{{ end_assistant_token }}
{%- set ns.in_assistant = false -%}
{%- endif -%}
{%- if "content" in message -%}
{{ user_token }}
{%- if message.content is string -%}
{{ message.content }}
{%- elif message.content is mapping and "parts" in message.content -%}
{%- set parts = message.content.parts -%}
{%- for part in parts -%}
{%- if part.type == "text" -%}
{{ part.text }}
{%- else -%}
{{- raise_exception("Invalid user part: " + part.type) -}}
{%- endif -%}
{%- endfor -%}
{%- else -%}
{{- raise_exception("Invalid user message: " + message.role) -}}
{%- endif -%}
{{ end_user_token }}
{%- endif -%}
{%- elif message.role == 'assistant' -%}
{%- if not ns.in_assistant -%}
{{ assistant_token }}
{%- set ns.in_assistant = true -%}
{%- endif -%}
{%- if "content" in message and message.content is not none -%}
{%- if message.content is string and (ns.assistant_format is none or ns.assistant_format == "string") -%}
{%- if ns.in_tool -%}
{{ ']' }}
{%- set ns.in_tool = false -%}
{%- endif -%}
{%- set ns.assistant_format = "string" -%}
{{ message.content }}
{%- elif message.content is mapping and "blocks" in message.content and (ns.assistant_format is none or ns.assistant_format == "mapping") -%}
{%- set ns.assistant_format = "mapping" -%}
{%- set blocks = message.content.blocks -%}
{%- for block in blocks -%}
{%- if block.type == 'thoughts' -%}
{%- if ns.in_tool -%}
{{ ']' }}
{%- set ns.in_tool = false -%}
{%- endif -%}
{%- if not ns.in_inner -%}
{%- set ns.in_inner = true -%}
{{ inner_token }}
{%- endif -%}
{{ block.text }}
{%- elif block.type == 'tool_calls' -%}
{%- if ns.in_tool -%}
{{ ']' }}
{%- set ns.in_tool = false -%}
{%- endif -%}
{%- if ns.in_inner and not loop.first and block.calls|length == 1 and block.calls[0].name == 'display_answers' -%}
{%- set ns.in_inner = false -%}
{{ outer_token }}
{%- endif -%}
{{ tool_calls_token + '[' }}
{%- for tool_call in block.calls -%}
{%- set args = tool_call.arguments -%}
{%- if args is string -%}
{{- '{"' + tool_call.name + '": ' + args + '}' }}
{%- else -%}
{{- '{"' + tool_call.name + '": ' + args|tojson + '}' }}
{%- endif -%}
{%- if not loop.last -%}
{{- ", " }}
{%- endif -%}
{%- endfor -%}
{{ ']' + end_tool_calls_token }}
{%- elif block.type == 'tool_outputs' -%}
{%- if ns.in_tool -%}
{{- raise_exception("Cannot have both tool outputs as separate messages and tool outputs as blocks") -}}
{%- endif -%}
{{ '[' }}
{%- for tool_output in block.outputs -%}
{{- tool_output.output }}
{%- if not loop.last -%}
{{- ", " }}
{%- endif -%}
{%- endfor -%}
{{- ']' }}
{%- elif block.type == 'response' -%}
{%- if ns.in_tool -%}
{{ ']' }}
{%- set ns.in_tool = false -%}
{%- endif -%}
{%- if (not loop.first and ns.in_inner) or (ns.in_assistant and ns.in_inner) -%}
{%- set ns.in_inner = false -%}
{{ outer_token }}
{%- endif -%}
{{ block.text }}
{%- else -%}
{{- raise_exception("Invalid assistant block type: " + block.type) -}}
{%- endif -%}
{%- endfor -%}
{%- else -%}
{{- raise_exception("Invalid assistant content") -}}
{%- endif -%}
{%- elif not ("tool_calls" in message and message.tool_calls) -%}
{{- raise_exception("Invalid assistant message") -}}
{%- endif -%}
{%- if "tool_calls" in message and message.tool_calls -%}
{{ tool_calls_token + '[' }}
{%- for tool_call in message.tool_calls -%}
{%- if tool_call.type == 'function' -%}
{%- set function = tool_call.function -%}
{%- set args = function.arguments -%}
{%- if args is string -%}
{{- '{"' + function.name + '": ' + args + '}' }}
{%- else -%}
{{- '{"' + function.name + '": ' + args|tojson + '}' }}
{%- endif -%}
{%- if not loop.last -%}
{{- ", " }}
{%- endif -%}
{%- else -%}
{{- raise_exception("Invalid tool call type: " + tool_call.type) -}}
{%- endif -%}
{%- endfor -%}
{{ ']' + end_tool_calls_token }}
{%- endif -%}
{%- elif message.role == 'tool' -%}
{%- if not ns.in_assistant -%}
{{- raise_exception("Tool message outside of assistant") -}}
{%- endif -%}
{%- if not ns.in_tool -%}
{{ '[' }}
{%- set ns.in_tool = true -%}
{%- else -%}
{{ ", "}}
{%- endif -%}
{%- if message.content is string -%}
{{ message.content }}
{%- else -%}
{{ message.content|tojson }}
{%- endif -%}
{%- else -%}
{{- raise_exception("Invalid message role") -}}
{%- endif -%}
{%- endfor -%}
{%- if ns.in_tool -%}
{{ ']' }}
{%- endif -%}
{%- if add_generation_prompt -%}
{{ assistant_token }}
{%- endif -%}
tests/tool_parsers/test_apertus_tool_parser.py
+433
View File
@@ -0,0 +1,433 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from typing import Any
from unittest.mock import MagicMock
import pytest
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.tool_parsers.apertus_tool_parser import (
TOOL_CALLS_PREFIX,
TOOL_CALLS_SUFFIX,
ApertusToolParser,
)
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture
def mock_tokenizer():
tokenizer = MagicMock()
tokenizer.encode.return_value = [1, 2, 3]
# Include the tool call tokens in the vocab for the parser
tokenizer.get_vocab.return_value = {TOOL_CALLS_PREFIX: 100, TOOL_CALLS_SUFFIX: 101}
return tokenizer
@pytest.fixture
def parser(mock_tokenizer):
return ApertusToolParser(mock_tokenizer)
@pytest.fixture
def mock_request():
request = MagicMock(spec=ChatCompletionRequest)
request.tools = []
request.tool_choice = "auto"
return request
# ---------------------------------------------------------------------------
# Non-streaming extraction tests
# ---------------------------------------------------------------------------
class TestExtractToolCalls:
def test_no_tool_calls(self, parser, mock_request):
model_output = "Hello, how can I help you today?"
result = parser.extract_tool_calls(model_output, mock_request)
assert result.tools_called is False
assert result.tool_calls == []
assert result.content == model_output
def test_single_tool_call(self, parser, mock_request):
model_output = (
'<|tools_prefix|>[{"get_weather": {"location": "London"}}]<|tools_suffix|>'
)
result = parser.extract_tool_calls(model_output, mock_request)
assert result.tools_called is True
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "get_weather"
args = json.loads(result.tool_calls[0].function.arguments)
assert args == {"location": "London"}
def test_multiple_arguments(self, parser, mock_request):
model_output = (
'<|tools_prefix|>[{"get_weather": '
'{"location": "San Francisco", '
'"unit": "celsius"}}]<|tools_suffix|>'
)
result = parser.extract_tool_calls(model_output, mock_request)
assert result.tools_called is True
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "get_weather"
args = json.loads(result.tool_calls[0].function.arguments)
assert args == {"location": "San Francisco", "unit": "celsius"}
def test_text_before_tool_call(self, parser, mock_request):
model_output = (
"Let me check the weather for you. "
'<|tools_prefix|>[{"get_weather": {"location": "Paris"}}]<|tools_suffix|>'
)
result = parser.extract_tool_calls(model_output, mock_request)
assert result.tools_called is True
assert result.content == "Let me check the weather for you."
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "get_weather"
def test_multiple_tool_calls(self, parser, mock_request):
model_output = (
'<|tools_prefix|>[{"get_weather": '
'{"location": "London"}}, '
'{"get_time": {"location": "London"}}]<|tools_suffix|>'
)
result = parser.extract_tool_calls(model_output, mock_request)
assert result.tools_called is True
assert len(result.tool_calls) == 2
assert result.tool_calls[0].function.name == "get_weather"
assert result.tool_calls[1].function.name == "get_time"
def test_nested_arguments(self, parser, mock_request):
model_output = (
'<|tools_prefix|>[{"complex_function": '
'{"nested": {"inner": "value"}, '
'"list": ["a", "b"]}}]<|tools_suffix|>'
)
result = parser.extract_tool_calls(model_output, mock_request)
assert result.tools_called is True
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "complex_function"
args = json.loads(result.tool_calls[0].function.arguments)
assert args == {"nested": {"inner": "value"}, "list": ["a", "b"]}
def test_incomplete_tool_call(self, parser, mock_request):
model_output = '<|tools_prefix|>[{"get_weather": {"location": "London"}'
result = parser.extract_tool_calls(model_output, mock_request)
assert result.tools_called is True
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "get_weather"
args = json.loads(result.tool_calls[0].function.arguments)
assert args == {"location": "London"}
def test_missing_tool_suffix(self, parser, mock_request):
model_output = (
'<|tools_prefix|>[{"get_weather": '
'{"location": "San Francisco", "unit": "celsius"}}]'
)
result = parser.extract_tool_calls(model_output, mock_request)
assert result.tools_called is True
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "get_weather"
args = json.loads(result.tool_calls[0].function.arguments)
assert args == {"location": "San Francisco", "unit": "celsius"}
# ---------------------------------------------------------------------------
# Streaming extraction tests
# ---------------------------------------------------------------------------
class TestStreamingExtraction:
def _simulate_streaming(
self, parser: ApertusToolParser, mock_request: Any, chunks: list[str]
) -> list[tuple[Any, str]]:
results: list[tuple[Any, str]] = []
previous_text: str = ""
previous_token_ids: list[int] = []
for chunk in chunks:
current_text = previous_text + chunk
# Simulate a token ID sequence matching the chunk progression
delta_token_ids: list[int] = [0] * max(1, len(chunk) // 4)
current_token_ids = previous_token_ids + delta_token_ids
delta = parser.extract_tool_calls_streaming(
previous_text=previous_text,
current_text=current_text,
delta_text=chunk,
previous_token_ids=tuple(previous_token_ids),
current_token_ids=tuple(current_token_ids),
delta_token_ids=tuple(delta_token_ids),
request=mock_request,
)
results.append((delta, current_text))
previous_text = current_text
previous_token_ids = list(current_token_ids)
return results
def _collect_tool_calls(self, results) -> dict[int, dict[str, Any]]:
"""Properly tracks and concatenates streamed tool arguments by their Index."""
tool_calls = {}
for delta, _ in results:
if not delta or not getattr(delta, "tool_calls", None):
continue
for tc in delta.tool_calls:
idx = (
tc.get("index", 0)
if isinstance(tc, dict)
else getattr(tc, "index", 0)
)
func = (
tc.get("function", {})
if isinstance(tc, dict)
else getattr(tc, "function", None)
)
if not func:
continue
name = (
func.get("name")
if isinstance(func, dict)
else getattr(func, "name", None)
)
args = (
func.get("arguments")
if isinstance(func, dict)
else getattr(func, "arguments", None)
)
if idx not in tool_calls:
tool_calls[idx] = {"name": "", "arguments": ""}
if name:
tool_calls[idx]["name"] += name
if args:
tool_calls[idx]["arguments"] += args
return tool_calls
def _collect_content(self, results) -> str:
"""Collects generated normal text outside of the tool calls."""
return "".join(
delta.content
for delta, _ in results
if delta and getattr(delta, "content", None)
)
def test_basic_streaming_single_tool(self, parser, mock_request):
chunks = [
"<|tools_prefix|>",
'[{"get_weather": ',
'{"location": "Paris, ',
'France"}}]',
"<|tools_suffix|>",
]
results = self._simulate_streaming(parser, mock_request, chunks)
tcs = self._collect_tool_calls(results)
assert len(tcs) == 1
assert tcs[0]["name"] == "get_weather"
assert json.loads(tcs[0]["arguments"]) == {"location": "Paris, France"}
def test_streaming_missing_tool_suffix(self, parser, mock_request):
chunks = [
"<|tools_prefix|>",
'[{"get_weather": ',
'{"location": "Paris, ',
'France"}}]',
]
results = self._simulate_streaming(parser, mock_request, chunks)
tcs = self._collect_tool_calls(results)
assert len(tcs) == 1
assert tcs[0]["name"] == "get_weather"
assert json.loads(tcs[0]["arguments"]) == {"location": "Paris, France"}
def test_streaming_partial_tag_buffering_missing_tool_suffix(
self, parser, mock_request
):
chunks = ["Content", "<|tools_", "prefix|>", '[{"f": ', '{"a": 1}}]']
results = self._simulate_streaming(parser, mock_request, chunks)
content = self._collect_content(results)
assert "Content" in content
assert "<|tools_prefix|>" not in content
assert "<|tools_suffix|>" not in content
tcs = self._collect_tool_calls(results)
assert len(tcs) == 1
assert tcs[0]["name"] == "f"
assert json.loads(tcs[0]["arguments"]) == {"a": 1}
def test_streaming_multi_tool(self, parser, mock_request):
chunks = [
"<|tools_prefix|>",
'[{"get_weather": {"location": "Tokyo"}}',
', {"get_time": {"location": "Tokyo"}}]',
"<|tools_suffix|>",
]
results = self._simulate_streaming(parser, mock_request, chunks)
tcs = self._collect_tool_calls(results)
assert len(tcs) == 2
assert tcs[0]["name"] == "get_weather"
assert json.loads(tcs[0]["arguments"]) == {"location": "Tokyo"}
assert tcs[1]["name"] == "get_time"
assert json.loads(tcs[1]["arguments"]) == {"location": "Tokyo"}
def test_streaming_text_before_tool_call(self, parser, mock_request):
chunks = [
"Let me check ",
"the weather. ",
"<|tools_prefix|>",
'[{"get_weather": {"location": "London"}}]',
"<|tools_suffix|>",
]
results = self._simulate_streaming(parser, mock_request, chunks)
content = self._collect_content(results)
assert content.strip() == "Let me check the weather."
tcs = self._collect_tool_calls(results)
assert len(tcs) == 1
assert tcs[0]["name"] == "get_weather"
assert json.loads(tcs[0]["arguments"]) == {"location": "London"}
def test_streaming_partial_tag_buffering(self, parser, mock_request):
chunks = [
"Content",
"<|tools_",
"prefix|>",
'[{"f": {"a": 1}}]',
"<|tools_suf",
"fix|>",
]
results = self._simulate_streaming(parser, mock_request, chunks)
content = self._collect_content(results)
assert "Content" in content
assert "<|tools_prefix|>" not in content
assert "<|tools_suffix|>" not in content
tc = self._collect_tool_calls(results)
assert len(tc) == 1
assert tc[0]["name"] == "f"
assert json.loads(tc[0]["arguments"]) == {"a": 1}
# ---------------------------------------------------------------------------
# Edge Cases: Multi-Token Prediction (MTP) & vLLM Chunking Anomalies
# ---------------------------------------------------------------------------
def test_mtp_streaming_massive_chunk(self, parser, mock_request):
"""Simulates MTP predicting text, tool calls,
and trailing text all in a single chunk."""
chunks = [
"Sure! "
'<|tools_prefix|>[{"get_weather": {"location": "London"}}]<|tools_suffix|>'
]
results = self._simulate_streaming(parser, mock_request, chunks)
content = self._collect_content(results)
assert "Sure! " in content
tc = self._collect_tool_calls(results)
assert len(tc) == 1
assert tc[0]["name"] == "get_weather"
assert json.loads(tc[0]["arguments"]) == {"location": "London"}
def test_mtp_streaming_multiple_tools_burst(self, parser, mock_request):
"""Simulates MTP predicting an array of multiple tools in one single chunk."""
chunks = [
'<|tools_prefix|>[{"get_weather": '
'{"location": "London"}}, '
'{"get_time": {"location": "Paris"}}]<|tools_suffix|>'
]
results = self._simulate_streaming(parser, mock_request, chunks)
tc = self._collect_tool_calls(results)
assert len(tc) == 2
assert tc[0]["name"] == "get_weather"
assert json.loads(tc[0]["arguments"]) == {"location": "London"}
assert tc[1]["name"] == "get_time"
assert json.loads(tc[1]["arguments"]) == {"location": "Paris"}
def test_mtp_streaming_skip_and_catch_up(self, parser, mock_request):
"""Simulates MTP chunks that jump over entire tools
(e.g., from middle of tool 1 to middle of tool 3)."""
chunks = [
'<|tools_prefix|>[{"t1": {"a": 1}',
'}, {"t2": {"b": 2}}, {"t3": {"c": 3',
"}}]<|tools_suffix|>",
]
results = self._simulate_streaming(parser, mock_request, chunks)
tc = self._collect_tool_calls(results)
assert len(tc) == 3
assert tc[0]["name"] == "t1"
assert json.loads(tc[0]["arguments"]) == {"a": 1}
assert tc[1]["name"] == "t2"
assert json.loads(tc[1]["arguments"]) == {"b": 2}
assert tc[2]["name"] == "t3"
assert json.loads(tc[2]["arguments"]) == {"c": 3}
def test_vllm_streaming_character_by_character(self, parser, mock_request):
"""Simulates worst-case vLLM fragmentation where
chunks arrive character-by-character."""
text = (
'Hi <|tools_prefix|>[{"get_weather": '
'{"location": "London"}}]<|tools_suffix|> '
)
chunks = list(text)
results = self._simulate_streaming(parser, mock_request, chunks)
content = self._collect_content(results)
assert "Hi" in content
tc = self._collect_tool_calls(results)
assert len(tc) == 1
assert tc[0]["name"] == "get_weather"
assert json.loads(tc[0]["arguments"]) == {"location": "London"}
def test_vllm_streaming_empty_deltas(self, parser, mock_request):
"""Simulates vLLM stream producing empty string chunks
(e.g., hidden tokens or artifacts)."""
chunks = [
"Wait",
"",
"<|tools_prefix|>",
"",
'[{"get_weather": ',
"",
'{"location": "London"}}]',
"<|tools_suffix|>",
]
results = self._simulate_streaming(parser, mock_request, chunks)
content = self._collect_content(results)
assert content == "Wait"
tc = self._collect_tool_calls(results)
assert len(tc) == 1
assert tc[0]["name"] == "get_weather"
assert json.loads(tc[0]["arguments"]) == {"location": "London"}
vllm/tool_parsers/__init__.py
+4
View File
@@ -186,6 +186,10 @@ _TOOL_PARSERS_TO_REGISTER = {
"gemma4_tool_parser",
"Gemma4ToolParser",
),
"apertus": (
"apertus_tool_parser",
"ApertusToolParser",
),
}
vllm/tool_parsers/apertus_tool_parser.py
+553
View File
@@ -0,0 +1,553 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tool call parser for Apertus models.
Extracts tool calls from the format:
<|tools_prefix|>[{"function_name": {"arg1": "value1", ...}}, ...]<|tools_suffix|>
Used when --enable-auto-tool-choice --tool-call-parser apertus are set.
"""
import json
from collections.abc import Sequence
import regex as re
from partial_json_parser.core.options import Allow
from vllm.entrypoints.chat_utils import make_tool_call_id
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.engine.protocol import (
DeltaFunctionCall,
DeltaMessage,
DeltaToolCall,
ExtractedToolCallInformation,
FunctionCall,
ToolCall,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.logger import init_logger
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers.abstract_tool_parser import (
Tool,
ToolParser,
)
from vllm.tool_parsers.utils import (
find_common_prefix,
partial_json_loads,
)
logger = init_logger(__name__)
# Apertus special tokens for tool calls
TOOL_CALLS_PREFIX = "<|tools_prefix|>"
TOOL_CALLS_SUFFIX = "<|tools_suffix|>"
class ApertusToolParser(ToolParser):
"""
Tool call parser for Apertus models.
Handles the extraction of tool calls from text in both non-streaming
(complete string) and streaming (chunked token) environments.
The expected Apertus function call format is a JSON array of single-key dictionaries
sandwiched between special tokens:
`<|tools_prefix|>[{"function_name": {"arg1": "value1"}}, ...]<|tools_suffix|>`
Examples:
>>> tokenizer = ... # Mock tokenizer
>>> parser = ApertusToolParser(tokenizer)
>>> output = 'I will check. <|tools_prefix|>[{"get_weather": '\
'{"city": "Paris"}}]<|tools_suffix|>'
>>> request = ChatCompletionRequest(...)
>>> info = parser.extract_tool_calls(output, request)
>>> info.content
"I will check."
>>> info.tool_calls[0].function.name
"get_weather"
>>> info.tool_calls[0].function.arguments
'{"city": "Paris"}'
"""
def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
"""
Initializes the ApertusToolParser.
Args:
tokenizer: The model's tokenizer.
Must be provided to interact with special tokens.
tools: Optional list of tools available for the current request.
Raises:
ValueError: If the `model_tokenizer`
is not successfully passed to the base class.
"""
super().__init__(tokenizer, tools)
if not self.model_tokenizer:
raise ValueError(
"The model tokenizer must be passed to the ToolParser "
"constructor during construction."
)
# Regex to extract tool calls block (suffix is optional for incomplete outputs)
self.tool_call_regex = re.compile(
rf"{re.escape(TOOL_CALLS_PREFIX)}"
rf"(.*?)"
rf"(?:{re.escape(TOOL_CALLS_SUFFIX)}|$)",
re.DOTALL,
)
self._reset_streaming_state()
def _reset_streaming_state(self) -> None:
"""
Resets all streaming state variables for a new completion request.
This clears the delta text buffer and resets the pointers used to
track the currently streaming tool index and arguments. Called implicitly
during initialization and should be called between separate streams.
"""
self.buffered_delta_text = ""
self.current_tool_id = -1
self.current_tool_name_sent = False
self.streamed_args_for_tool: list[str] = []
def adjust_request(
self, request: ChatCompletionRequest | ResponsesRequest
) -> ChatCompletionRequest | ResponsesRequest:
"""
Adjusts the generation request to ensure special tool tokens are not skipped.
Forces `skip_special_tokens=False` if tools are actively being evaluated,
ensuring the tools special tokens are surfaced to the engine for parsing.
Args:
request: The incoming OpenAI-compatible chat completion request.
Returns:
The potentially modified chat completion request.
"""
request = super().adjust_request(request)
if request.tools and request.tool_choice != "none":
request.skip_special_tokens = False
return request
def _buffer_delta_text(self, delta_text: str) -> str:
"""
Buffers incoming delta chunks to prevent
fragmentation of multi-token special tags.
If a chunk ends with a partial match of
`<|tools_prefix|>` or `<|tools_suffix|>`,
it holds that part back until the next chunk clarifies if it's the actual tag
or just normal text.
Args:
delta_text: The newly generated text chunk
Returns:
The safe, verified text chunk free of partial tag collisions.
Examples:
>>> parser = ApertusToolParser(...)
>>> parser._buffer_delta_text("Let me check <|tool" \
"Let me check " # "<|tool" is buffered internally
>>> parser._buffer_delta_text("s_prefix|>" \
"<|tools_prefix|>" # Buffer released on completion
"""
self.buffered_delta_text += delta_text
text = self.buffered_delta_text
for tag in (TOOL_CALLS_PREFIX, TOOL_CALLS_SUFFIX):
if text.endswith(tag):
self.buffered_delta_text = ""
return text
# Evaluate longest possible partial match first
for i in range(len(tag) - 1, 0, -1):
if text.endswith(tag[:i]):
self.buffered_delta_text = text[-i:]
return text[:-i]
self.buffered_delta_text = ""
return text
def extract_tool_calls(
self,
model_output: str,
request: ChatCompletionRequest,
) -> ExtractedToolCallInformation:
"""
Extracts tool calls from a completely generated model response (Non-Streaming).
Args:
model_output: The full completion string generated by the model.
request: The current chat completion
request context containing tool schemas.
Returns:
An `ExtractedToolCallInformation` object containing normal text content
and a list of fully formatted `ToolCall` objects.
Examples:
>>> output = 'Let me see. <|tools_prefix|>[{"get_weather":' \
'{"loc": "Paris"}}]<|tools_suffix|>'
>>> info = parser.extract_tool_calls(output, request)
>>> info.tools_called
True
>>> info.content
'Let me see.'
>>> info.tool_calls[0].function.name
'get_weather'
"""
match = self.tool_call_regex.search(model_output)
if not match:
return ExtractedToolCallInformation(
tools_called=False, tool_calls=[], content=model_output
)
try:
# group(1) might contain trailing text if the suffix is missing
matched_text = match.group(1)
stripped_text = matched_text.lstrip()
try:
# Use raw_decode to robustly isolate
# the valid JSON array from any trailing garbage
parsed_json, idx = json.JSONDecoder().raw_decode(stripped_text)
trailing_in_group = stripped_text[idx:]
except json.JSONDecodeError:
# Fallback sequentially to partial parser for token-truncated requests
parsed_json, _ = partial_json_loads(matched_text, Allow.ALL)
trailing_in_group = ""
if not isinstance(parsed_json, list):
parsed_json = [parsed_json] if parsed_json else []
tool_calls: list[ToolCall] = []
for obj in parsed_json:
if isinstance(obj, dict) and obj:
name, args = next(iter(obj.items()))
tool_calls.append(
ToolCall(
type="function",
id=make_tool_call_id(),
function=FunctionCall(
name=name,
arguments=json.dumps(args, ensure_ascii=False),
),
)
)
# Content combines any generated text
# prior to and safely after the tool block
content_str = model_output[: match.start()].strip()
# Surface any hallucinated text inside
# the regex group (due to missing suffix)
if trailing_in_group.strip():
trailing = trailing_in_group.replace(TOOL_CALLS_SUFFIX, "").strip()
if trailing:
content_str = (content_str + "\n" + trailing).strip()
# Surface text natively generated after the explicit suffix
after_suffix = (
model_output[match.end() :].replace(TOOL_CALLS_SUFFIX, "").strip()
)
if after_suffix:
content_str = (content_str + "\n" + after_suffix).strip()
return ExtractedToolCallInformation(
tools_called=True,
tool_calls=tool_calls,
content=content_str if content_str else None,
)
except Exception:
logger.exception("Error extracting tool calls from Apertus response")
return ExtractedToolCallInformation(
tools_called=False, tool_calls=[], content=model_output
)
def extract_tool_calls_streaming(
self,
previous_text: str,
current_text: str,
delta_text: str,
previous_token_ids: Sequence[int],
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> DeltaMessage | None:
"""
Handles streaming chunks
Args:
previous_text: The complete model text generated prior to this chunk.
current_text: The complete model text including this chunk.
delta_text: The incremental text addition.
previous_token_ids: Tokens generated prior to this chunk.
current_token_ids: Total tokens generated.
delta_token_ids: Incremental token additions.
request: The chat completion request.
Returns:
A `DeltaMessage` with updated content or tool argument diffs, or `None` if
the chunk shouldn't emit visible changes yet (e.g. it was purely buffered).
Examples:
>>> prev = '<|tools_prefix|>[{"get_weather": {"loc'
>>> cur = '<|tools_prefix|>[{"get_weather": {"location": "Paris"}}'
>>> delta = 'ation": "Paris"}}'
>>> msg = parser.extract_tool_calls_streaming(
... prev, cur, delta, ..., request
... )
>>> msg.tool_calls[0].function.arguments
'ation": "Paris"}'
"""
delta_text = self._buffer_delta_text(delta_text)
if not delta_text:
return None
# Fast path: normal text generation before any tools are invoked
if TOOL_CALLS_PREFIX not in current_text:
return DeltaMessage(content=delta_text)
try:
return self._extract_streaming(current_text, delta_text)
except Exception:
logger.exception("Error in Apertus streaming tool call extraction")
return None
def _extract_streaming(
self, current_text: str, delta_text: str
) -> DeltaMessage | None:
"""
Core streaming logic.
Separates visible chat text from JSON blocks and computes diffs.
Args:
current_text: The full generated output string so far.
delta_text: The latest chunk of text added.
Returns:
A `DeltaMessage` containing the `content` delta and/or `tool_calls` delta.
"""
prefix_idx = current_text.rfind(TOOL_CALLS_PREFIX)
suffix_idx = current_text.rfind(TOOL_CALLS_SUFFIX)
is_inside_tools = prefix_idx > suffix_idx
json_completed = False
json_end_idx: int | None = None
# Check if the JSON array successfully closed implicitly
if is_inside_tools:
json_start = prefix_idx + len(TOOL_CALLS_PREFIX)
s = current_text[json_start:].lstrip()
try:
# If raw_decode succeeds,
# the JSON array is fully formed and implicitly closed
_, idx = json.JSONDecoder().raw_decode(s)
json_end_idx = len(current_text) - len(s) + idx
json_completed, is_inside_tools = True, False
except Exception:
pass
just_finished = (TOOL_CALLS_SUFFIX in delta_text) or json_completed
# 1. Fast path: Output normal text immediately
# if we are completely outside tool block constraints
if not is_inside_tools and not just_finished:
text = delta_text.replace(TOOL_CALLS_PREFIX, "").replace(
TOOL_CALLS_SUFFIX, ""
)
return DeltaMessage(content=text) if text else None
# 2. Extract leading and trailing normal text directly adjacent to tool blocks
content_str = ""
if TOOL_CALLS_PREFIX in delta_text:
content_str += delta_text.split(TOOL_CALLS_PREFIX)[0].replace(
TOOL_CALLS_SUFFIX, ""
)
if just_finished:
if json_completed and json_end_idx is not None:
# The tool block finished in this chunk via implicit JSON completion
# Ensure we strictly isolate
# and extract only trailing text that is part of `delta_text`
delta_start_idx = len(current_text) - len(delta_text)
content_start = max(json_end_idx, delta_start_idx)
if content_start < len(current_text):
content_str += current_text[content_start:].replace(
TOOL_CALLS_SUFFIX, ""
)
else:
content_str += delta_text.split(TOOL_CALLS_SUFFIX)[-1]
# 3. Extract the isolated JSON array string for the active block
json_start = prefix_idx + len(TOOL_CALLS_PREFIX)
json_end = suffix_idx if suffix_idx > prefix_idx else json_end_idx
json_str = current_text[json_start:json_end]
tool_calls = self._parse_and_diff_json(json_str, is_final=not is_inside_tools)
if tool_calls or content_str:
return DeltaMessage(
content=content_str if content_str else None,
tool_calls=tool_calls if tool_calls else None,
)
return None
def _parse_and_diff_json(
self, json_str: str, is_final: bool
) -> list[DeltaToolCall]:
"""
Parses an isolated, potentially incomplete streaming JSON array and returns
newly accumulated tool call diffs.
Args:
json_str: The extracted JSON array string so far
(e.g. `[{"weather": {"city": "Par"}]`).
is_final: True if the tool block has received its closing`<|tools_suffix|>`
Returns:
A list of `DeltaToolCall`
items representing string diffs in function arguments
to stream back to the client.
"""
try:
parsed, _ = partial_json_loads(json_str, Allow.ALL)
if not isinstance(parsed, list):
parsed = [parsed] if parsed else []
except Exception:
return []
if not parsed:
return []
tool_calls: list[DeltaToolCall] = []
latest_index = len(parsed) - 1
# Catch up and finalize any tools we fully skipped over in one large text delta
while self.current_tool_id < latest_index:
if self.current_tool_id >= 0:
if not self.current_tool_name_sent:
self._emit_tool_name(parsed, self.current_tool_id, tool_calls)
delta = self._get_tool_diff(parsed, self.current_tool_id, is_final=True)
if delta:
tool_calls.append(delta)
self.current_tool_id += 1
self.current_tool_name_sent = False
while len(self.streamed_args_for_tool) <= self.current_tool_id:
self.streamed_args_for_tool.append("")
# Stream the currently active tool
if self.current_tool_id >= 0:
if not self.current_tool_name_sent:
self._emit_tool_name(parsed, self.current_tool_id, tool_calls)
delta = self._get_tool_diff(parsed, self.current_tool_id, is_final)
if delta:
tool_calls.append(delta)
return tool_calls
def _emit_tool_name(
self, parsed: list, index: int, tool_calls: list[DeltaToolCall]
) -> None:
"""
Extracts and emits the function name mapped to a new tool call ID.
Args:
parsed: The partially parsed JSON list containing tool dictionaries.
index: The active index within the JSON list.
tool_calls: The running list of delta chunks to mutate.
Examples:
Appends `DeltaToolCall(index=0,
function=DeltaFunctionCall(name="get_weather", ...))`
to the `tool_calls` list and marks the name as sent.
"""
obj = parsed[index]
if isinstance(obj, dict) and obj:
name = next(iter(obj))
self.current_tool_name_sent = True
tool_calls.append(
DeltaToolCall(
index=index,
type="function",
id=make_tool_call_id(),
function=DeltaFunctionCall(name=name, arguments="").model_dump(
exclude_none=True
),
)
)
def _get_tool_diff(
self, parsed: list, index: int, is_final: bool
) -> DeltaToolCall | None:
"""
Calculates the exact string difference to safely append new tool parameters.
This ensures characters like `{`, `}`, and `"` don't jump around unevenly
in the UI frontend while streaming incomplete JSON arguments.
Args:
parsed: The latest list of parsed JSON objects.
index: The active tool's array index.
is_final: Whether to emit
trailing structural brackets (True if block is done).
Returns:
A `DeltaToolCall` mapping to the arguments diff,
or None if no text was appended.
Examples:
>>> # Previous streamed state: '{"city": "Pari'
>>> # Current full parse state: '{"city": "Paris"}'
>>> # Returns diff (closing bracket suppressed until final):
>>> parser._get_tool_diff(parsed, index=0, is_final=False)
DeltaToolCall(index=0, function=DeltaFunctionCall(arguments='s'))
"""
obj = parsed[index]
if not isinstance(obj, dict) or not obj:
return None
name, args = next(iter(obj.items()))
if args is None:
return None
args_json = json.dumps(args, ensure_ascii=False)
# Suppress trailing structural characters
# during stream (looks cleaner in frontends)
if not is_final:
while args_json and args_json[-1] in ("}", '"', "]", " ", ","):
args_json = args_json[:-1]
prev_sent = self.streamed_args_for_tool[index]
if args_json == prev_sent:
return None
prefix = find_common_prefix(prev_sent, args_json)
if len(prefix) < len(prev_sent):
# Backtrack state if partial parser structurally updates a past assumption
self.streamed_args_for_tool[index] = prefix
return None
diff = args_json[len(prev_sent) :]
if diff:
self.streamed_args_for_tool[index] = args_json
return DeltaToolCall(
index=index,
function=DeltaFunctionCall(arguments=diff).model_dump(
exclude_none=True
),
)
return None