obscura/vllm
2
0
Fork 0
mirror of https://github.com/vllm-project/vllm.git synced 2026-08-18 11:40:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

[Frontend] Fix Kimi K2 tool call IDs for required tool choice (#46344)

Browse Source 
Signed-off-by: chaunceyjiang <[email protected]>
This commit is contained in:
Chauncey
2026-06-24 19:59:40 +00:00
committed by GitHub
parent 49f2104c53
commit 84c2f9f0fb
11 changed files with 406 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/entrypoints/openai/chat_completion/test_completion_with_function_calling.py
+1 -74
View File
@@ -24,6 +24,7 @@ tools = [
"description": "Get the current weather in a given location",
"parameters": {
"type": "object",
"strict": True,
"properties": {
"city": {
"type": "string",
@@ -215,80 +216,6 @@ async def test_function_tool_use(
assert len(reasoning) > 0
@pytest.fixture(scope="module")
def k2_server():
args = [
# use half precision for speed and memory savings in CI environment
"--dtype",
"half",
"--enable-auto-tool-choice",
"--structured-outputs-config.backend",
"xgrammar",
"--tool-call-parser",
"hermes",
"--reasoning-parser",
"qwen3",
"--gpu-memory-utilization",
"0.4",
] + ROCM_EXTRA_ARGS
# Test kimi_k2 tool use tool_id format by overriding model_type.
# is_deepseek_mla safely returns False via getattr when kv_lora_rank
# is absent from the underlying config.
with RemoteOpenAIServer(
MODEL_NAME,
args,
env_dict=ROCM_ENV_OVERRIDES,
override_hf_configs={"model_type": "kimi_k2"},
) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def k2_client(k2_server):
async with k2_server.get_async_client() as async_client:
yield async_client
@pytest.mark.asyncio
@pytest.mark.skip(reason="Skipping Kimi K2 tool ID test")
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("stream", [True, False])
@pytest.mark.parametrize("tool_choice", ["required"])
async def test_tool_id_kimi_k2(
k2_client: openai.AsyncOpenAI, model_name: str, stream: bool, tool_choice: str
):
if not stream:
# Non-streaming test
chat_completion = await k2_client.chat.completions.create(
messages=messages, model=model_name, tools=tools, tool_choice=tool_choice
)
assert chat_completion.choices[0].message.tool_calls is not None
assert len(chat_completion.choices[0].message.tool_calls) > 0
assert chat_completion.choices[0].message.tool_calls[0].id in [
"functions.get_current_weather:0",
"functions.get_forecast:1",
]
else:
# Streaming test
output_stream = await k2_client.chat.completions.create(
messages=messages,
model=model_name,
tools=tools,
tool_choice=tool_choice,
stream=True,
)
output = []
async for chunk in output_stream:
if chunk.choices and chunk.choices[0].delta.tool_calls:
output.extend(chunk.choices[0].delta.tool_calls)
for o in output:
assert o.id is None or o.id in [
"functions.get_current_weather:0",
"functions.get_forecast:1",
]
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("arguments", ["{}", ""])
tests/entrypoints/openai/responses/test_parsable_context_unit.py
+9 -1
View File
@@ -183,11 +183,19 @@ def _make_request_output(
def _make_context(parser_cls, **overrides):
# ParsableContext no longer lazily builds a parser from ``parser_cls``;
# the caller (here, the serving layer in production) must supply one.
request = overrides.get("request", _make_request())
response_parser = overrides.pop("response_parser", None)
if response_parser is None and parser_cls is not None:
response_parser = parser_cls(MagicMock(), request.tools)
defaults = dict(
tokenizer=MagicMock(),
parser_cls=parser_cls,
response_parser=response_parser,
response_messages=[],
request=_make_request(),
request=request,
available_tools=None,
chat_template=None,
chat_template_content_format="auto",
tests/parser/test_parse.py
+149 -2
View File
@@ -3,6 +3,7 @@
import json
import os
from types import SimpleNamespace
import pytest
@@ -13,7 +14,9 @@ os.environ[_STRICT_TOOL_CALLING_ENV] = "0"
from vllm.entrypoints.openai.chat_completion.protocol import ( # noqa: E402
ChatCompletionRequest,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest # noqa: E402
from vllm.parser.abstract_parser import DelegatingParser # noqa: E402
from vllm.parser.utils import count_history_tool_calls # noqa: E402
from vllm.reasoning.basic_parsers import ( # noqa: E402
BaseThinkingReasoningParser,
)
@@ -82,12 +85,55 @@ TOOLS = [
]
def make_parser(tokenizer, reasoning=False, tool=False):
KIMI_K2_MODEL_CONFIG = SimpleNamespace(
hf_text_config=SimpleNamespace(model_type="kimi_k2"),
hf_overrides=None,
)
HISTORY_MESSAGES = [
{"role": "user", "content": "first"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "functions.get_current_weather:0",
"type": "function",
"function": {
"name": "get_current_weather",
"arguments": "{}",
},
},
{
"id": "functions.get_forecast:1",
"type": "function",
"function": {
"name": "get_forecast",
"arguments": "{}",
},
},
],
},
{
"role": "tool",
"tool_call_id": "functions.get_current_weather:0",
"content": "{}",
},
{
"role": "tool",
"tool_call_id": "functions.get_forecast:1",
"content": "{}",
},
{"role": "user", "content": "again"},
]
def make_parser(tokenizer, reasoning=False, tool=False, **kwargs):
class TestParser(DelegatingParser):
reasoning_parser_cls = ThinkReasoningParser if reasoning else None
tool_parser_cls = Hermes2ProToolParser if tool else None
return TestParser(tokenizer)
return TestParser(tokenizer, **kwargs)
@pytest.mark.parametrize(
@@ -232,6 +278,107 @@ def test_parse_required_tool_choice(tokenizer):
assert json.loads(tool_calls[1].arguments) == {"timezone": "UTC"}
def test_parse_required_tool_choice_kimi_k2_ids(tokenizer):
parser = make_parser(
tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG
)
functions_json = json.dumps(
[
{"name": "get_current_weather", "parameters": {"city": "Dallas"}},
{"name": "get_forecast", "parameters": {"city": "Dallas", "days": 2}},
]
)
request = make_request(tools=TOOLS, tool_choice="required")
_, content, tool_calls = parser.parse(
functions_json, request, enable_auto_tools=True
)
assert content is None
assert tool_calls is not None
assert [tc.id for tc in tool_calls] == [
"functions.get_current_weather:0",
"functions.get_forecast:1",
]
def test_parse_required_tool_choice_kimi_k2_ids_after_history(tokenizer):
parser = make_parser(
tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG
)
functions_json = json.dumps(
[{"name": "get_current_weather", "parameters": {"city": "Dallas"}}]
)
request = make_request(
messages=HISTORY_MESSAGES,
tools=TOOLS,
tool_choice="required",
)
_, _, tool_calls = parser.parse(functions_json, request, enable_auto_tools=True)
assert tool_calls is not None
assert tool_calls[0].id == "functions.get_current_weather:2"
def test_count_history_tool_calls_responses_request():
request = ResponsesRequest.model_validate(
{
"model": "test-model",
"input": [
{
"type": "function_call",
"call_id": "call_0",
"name": "get_current_weather",
"arguments": "{}",
},
{
"type": "function_call",
"call_id": "call_1",
"name": "get_forecast",
"arguments": "{}",
},
],
}
)
assert count_history_tool_calls(request) == 2
def test_parse_required_tool_choice_random_ids_deferred(tokenizer):
parser = make_parser(tokenizer, reasoning=False, tool=True)
functions_json = json.dumps(
[{"name": "get_current_weather", "parameters": {"city": "Dallas"}}]
)
request = make_request(
messages=HISTORY_MESSAGES,
tools=TOOLS,
tool_choice="required",
)
_, _, tool_calls = parser.parse(functions_json, request, enable_auto_tools=True)
assert tool_calls is not None
assert tool_calls[0].id is None
def test_parse_named_tool_choice_kimi_k2_id(tokenizer):
parser = make_parser(
tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG
)
request = make_request(
tools=TOOLS,
tool_choice={
"type": "function",
"function": {"name": "get_weather"},
},
)
_, content, tool_calls = parser.parse(
TOOL_ARGUMENTS, request, enable_auto_tools=True
)
assert content is None
assert tool_calls is not None
assert tool_calls[0].id == "functions.get_weather:0"
def test_parse_named_tool_choice_content_none(tokenizer):
parser = make_parser(tokenizer, reasoning=False, tool=True)
request = make_request(
tests/parser/test_streaming.py
+106 -2
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from types import SimpleNamespace
import pytest
@@ -47,6 +48,36 @@ TOOLS = [
]
KIMI_K2_MODEL_CONFIG = SimpleNamespace(
hf_text_config=SimpleNamespace(model_type="kimi_k2"),
hf_overrides=None,
)
HISTORY_MESSAGES = [
{"role": "user", "content": "first"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "functions.get_current_weather:0",
"type": "function",
"function": {
"name": "get_current_weather",
"arguments": "{}",
},
}
],
},
{
"role": "tool",
"tool_call_id": "functions.get_current_weather:0",
"content": "{}",
},
{"role": "user", "content": "again"},
]
@pytest.fixture
def request_obj():
return ChatCompletionRequest(
@@ -57,12 +88,12 @@ def request_obj():
)
def make_parser(tokenizer, reasoning=False, tool=False):
def make_parser(tokenizer, reasoning=False, tool=False, **kwargs):
class TestParser(DelegatingParser):
reasoning_parser_cls = ThinkReasoningParser if reasoning else None
tool_parser_cls = Hermes2ProToolParser if tool else None
return TestParser(tokenizer)
return TestParser(tokenizer, **kwargs)
def stream_text(parser, tokenizer, text, request, prompt_token_ids=None):
@@ -365,3 +396,76 @@ def test_parse_delta_tool_choice_none_with_reasoning(tokenizer, request_obj):
assert len(tool_calls) == 0
assert "<tool_call>" in content
assert "get_weather" in content
def test_parse_delta_required_tool_choice_kimi_k2_ids(tokenizer, request_obj):
parser = make_parser(
tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG
)
request = request_obj.model_copy(update={"tool_choice": "required"})
output = json.dumps(
[
{
"name": "get_current_weather",
"parameters": {"city": "Dallas"},
}
]
)
results: list[DeltaMessage | None] = []
prompt_token_ids: list[int] | None = []
for i in range(0, len(output), 3):
chunk = output[i : i + 3]
results.append(
parser.parse_delta(
chunk,
[],
request,
prompt_token_ids=prompt_token_ids,
finished=False,
)
)
prompt_token_ids = None
_, content, tool_calls = collect_fields(results)
assert content == ""
assert any(tc.id == "functions.get_current_weather:0" for tc in tool_calls)
assert all(tc.id in (None, "functions.get_current_weather:0") for tc in tool_calls)
def test_parse_delta_required_tool_choice_kimi_k2_ids_after_history(
tokenizer, request_obj
):
parser = make_parser(
tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG
)
request = request_obj.model_copy(
update={"messages": HISTORY_MESSAGES, "tool_choice": "required"}
)
output = json.dumps(
[
{
"name": "get_current_weather",
"parameters": {"city": "Dallas"},
}
]
)
results: list[DeltaMessage | None] = []
prompt_token_ids: list[int] | None = []
for i in range(0, len(output), 3):
chunk = output[i : i + 3]
results.append(
parser.parse_delta(
chunk,
[],
request,
prompt_token_ids=prompt_token_ids,
finished=False,
)
)
prompt_token_ids = None
_, _, tool_calls = collect_fields(results)
assert any(tc.id == "functions.get_current_weather:1" for tc in tool_calls)
assert all(tc.id in (None, "functions.get_current_weather:1") for tc in tool_calls)
vllm/entrypoints/openai/chat_completion/serving.py
+17 -68
View File
@@ -17,8 +17,6 @@ from vllm.engine.protocol import EngineClient
from vllm.entrypoints.chat_utils import (
ChatTemplateContentFormatOption,
ConversationMessage,
get_history_tool_calls_cnt,
get_tool_call_id_type,
make_tool_call_id,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
@@ -170,8 +168,6 @@ class OpenAIServingChat(OpenAIServing):
if mc.generation_config not in ("auto", "vllm")
else getattr(mc, "override_generation_config", {}).get("max_new_tokens")
)
self.tool_call_id_type = get_tool_call_id_type(self.model_config)
# NOTE(woosuk): While OpenAI's chat completion API supports browsing
# for some models, currently vLLM doesn't support it. Please use the
# Responses API instead.
@@ -261,6 +257,7 @@ class OpenAIServingChat(OpenAIServing):
tokenizer,
request.tools,
chat_template_kwargs=chat_template_kwargs,
model_config=self.model_config,
)
result = await self.render_chat_request(request)
if isinstance(result, ErrorResponse):
@@ -433,11 +430,6 @@ class OpenAIServingChat(OpenAIServing):
else:
tool_choice_function_name = None
if self.tool_call_id_type == "kimi_k2":
history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
else:
history_tool_call_cnt = 0
previous_texts = [""] * num_choices
try:
@@ -451,14 +443,10 @@ class OpenAIServingChat(OpenAIServing):
tokenizer,
request.tools,
chat_template_kwargs=chat_template_kwargs,
model_config=self.model_config,
)
for _ in range(num_choices)
]
for p in parsers:
if p is not None:
# NOTE: HarmonyParser ignores _stream_state (uses its own FSM).
p._stream_state.tool_call_id_type = self.tool_call_id_type
p._stream_state.history_tool_call_cnt = history_tool_call_cnt
else:
parsers = [None] * num_choices
except Exception as e:
@@ -842,10 +830,6 @@ class OpenAIServingChat(OpenAIServing):
)
choices: list[ChatCompletionResponseChoice] = []
if self.tool_call_id_type == "kimi_k2":
history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
else:
history_tool_call_cnt = 0
role = self.get_chat_request_role(request)
tool_parser_cls = (
@@ -885,54 +869,26 @@ class OpenAIServingChat(OpenAIServing):
tool_calls = []
auto_tools_called = False
is_named_tool_choice = (
request.tool_choice is not None
and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
)
is_required_tool_choice = request.tool_choice == "required"
if (not self.enable_auto_tools or not tool_parser_cls) and (
not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
and request.tool_choice != "required"
not is_named_tool_choice and not is_required_tool_choice
):
message = ChatMessage(role=role, reasoning=reasoning, content=content)
elif (
request.tool_choice
and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
):
tool_call_items = []
tool_calls = tool_calls or []
for tc in tool_calls:
if not tc.id:
tc.id = make_tool_call_id(
id_type=self.tool_call_id_type,
func_name=tc.name,
idx=history_tool_call_cnt,
)
tool_call_items.append(ToolCall(id=tc.id, function=tc))
history_tool_call_cnt += 1
elif is_named_tool_choice or is_required_tool_choice:
message = ChatMessage(
role=role,
reasoning=reasoning,
content=content or "",
tool_calls=tool_call_items,
)
elif request.tool_choice and request.tool_choice == "required":
tool_call_items = []
tool_calls = tool_calls or []
for tool_call in tool_calls:
if not tool_call.id:
tool_call.id = make_tool_call_id(
id_type=self.tool_call_id_type,
func_name=tool_call.name,
idx=history_tool_call_cnt,
)
tool_call_items.append(
ToolCall(id=tool_call.id, function=tool_call)
)
history_tool_call_cnt += 1
message = ChatMessage(
role=role,
content=content or "",
tool_calls=tool_call_items,
reasoning=reasoning,
tool_calls=[
ToolCall(id=tc.id or make_tool_call_id(), function=tc)
for tc in (tool_calls or [])
],
)
# if the request doesn't use tool choice
@@ -949,21 +905,14 @@ class OpenAIServingChat(OpenAIServing):
):
auto_tools_called = tool_calls is not None and len(tool_calls) > 0
if tool_calls:
tool_call_items = []
for tc in tool_calls:
if not tc.id:
tc.id = make_tool_call_id(
id_type=self.tool_call_id_type,
func_name=tc.name,
idx=history_tool_call_cnt,
)
tool_call_items.append(ToolCall(id=tc.id, function=tc))
history_tool_call_cnt += 1
message = ChatMessage(
role=role,
reasoning=reasoning,
content=content,
tool_calls=tool_call_items,
tool_calls=[
ToolCall(id=tc.id or make_tool_call_id(), function=tc)
for tc in tool_calls
],
)
else:
vllm/entrypoints/openai/responses/context.py
-14
View File
@@ -301,7 +301,6 @@ class ParsableContext(ConversationContext):
chat_template_content_format: ChatTemplateContentFormatOption,
response_parser: Parser | None = None,
enable_auto_tools: bool = False,
tool_call_id_type: str = "random",
):
self.num_prompt_tokens = 0
self.num_output_tokens = 0
@@ -314,20 +313,8 @@ class ParsableContext(ConversationContext):
self.num_init_messages = len(response_messages)
self.finish_reason: str | None = None
self.enable_auto_tools = enable_auto_tools
self.tool_call_id_type = tool_call_id_type
self.response_parser = response_parser
if self.response_parser is None and parser_cls is not None:
chat_template_kwargs = request.build_chat_params(
default_template=chat_template,
default_template_content_format=chat_template_content_format,
).chat_template_kwargs
self.response_parser = parser_cls(
tokenizer,
tools=request.tools,
chat_template_kwargs=chat_template_kwargs,
)
self.parser_cls = parser_cls
self.request = request
@@ -365,7 +352,6 @@ class ParsableContext(ConversationContext):
reasoning=reasoning,
content=content,
tool_calls=tool_calls,
tool_call_id_type=self.tool_call_id_type,
)
)
elif completion.text:
vllm/entrypoints/openai/responses/serving.py
+1 -6
View File
@@ -30,7 +30,6 @@ from vllm.engine.protocol import EngineClient
from vllm.entrypoints.chat_utils import (
ChatCompletionMessageParam,
ChatTemplateContentFormatOption,
get_tool_call_id_type,
)
from vllm.entrypoints.mcp.tool_server import ToolServer
from vllm.entrypoints.openai.engine.protocol import (
@@ -222,9 +221,6 @@ class OpenAIServingResponses(OpenAIServing):
"For gpt-oss, we ignore --enable-auto-tool-choice "
"and always enable tool use."
)
self.tool_call_id_type = get_tool_call_id_type(self.model_config)
self.enable_auto_tools = enable_auto_tools
# HACK(woosuk): This is a hack. We should use a better store.
# FIXME: If enable_store=True, this may cause a memory leak since we
@@ -272,6 +268,7 @@ class OpenAIServingResponses(OpenAIServing):
tokenizer,
request.tools,
chat_template_kwargs=chat_template_kwargs,
model_config=self.model_config,
)
def _validate_generator_input(
@@ -493,7 +490,6 @@ class OpenAIServingResponses(OpenAIServing):
chat_template=self.chat_template,
chat_template_content_format=self.chat_template_content_format,
enable_auto_tools=self.enable_auto_tools,
tool_call_id_type=self.tool_call_id_type,
)
else:
context = SimpleContext(
@@ -1073,7 +1069,6 @@ class OpenAIServingResponses(OpenAIServing):
content=content,
tool_calls=tool_calls,
logprobs=logprobs,
tool_call_id_type=self.tool_call_id_type,
)
# Fallback when no parser is configured
vllm/entrypoints/openai/responses/utils.py
+1 -7
View File
@@ -45,7 +45,6 @@ def build_response_output_items(
content: str | None,
tool_calls: list[FunctionCall] | None,
logprobs: list[Logprob] | None = None,
tool_call_id_type: str = "random",
) -> list[ResponseOutputItem]:
outputs: list[ResponseOutputItem] = []
@@ -86,12 +85,7 @@ def build_response_output_items(
ResponseFunctionToolCall(
id=f"fc_{random_uuid()}",
call_id=tool_call.id
if tool_call.id
else make_tool_call_id(
id_type=tool_call_id_type,
func_name=tool_call.name,
idx=idx,
),
or make_tool_call_id(func_name=tool_call.name, idx=idx),
type="function_call",
status="completed",
name=tool_call.name,
vllm/parser/abstract_parser.py
+46 -2
View File
@@ -11,6 +11,10 @@ from functools import cached_property
from openai.types.responses import ToolChoiceFunction
from pydantic import TypeAdapter, ValidationError
from vllm.entrypoints.chat_utils import (
get_tool_call_id_type,
make_tool_call_id,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionNamedToolChoiceParam,
ChatCompletionRequest,
@@ -24,6 +28,7 @@ from vllm.entrypoints.openai.engine.protocol import (
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.logger import init_logger
from vllm.parser.metrics import record_tool_parser_invocation
from vllm.parser.utils import count_history_tool_calls
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
from vllm.sampling_params import StructuredOutputsParams
from vllm.tokenizers import TokenizerLike
@@ -46,6 +51,7 @@ class StreamState:
previous_text: str = ""
previous_token_ids: list[int] = field(default_factory=list)
history_tool_call_cnt: int = 0
history_tool_call_cnt_initialized: bool = False
tool_call_id_type: str = "random"
# only used for "required" and "named tool" choices,
# tracks whether function name has been fully returned in the stream yet
@@ -108,6 +114,7 @@ class Parser:
tokenizer: TokenizerLike,
tools: list[Tool] | None = None,
*args,
model_config=None,
**kwargs,
):
self.model_tokenizer = tokenizer
@@ -124,7 +131,14 @@ class Parser:
self._reasoning_parser is None
or self._reasoning_parser.engine_based_streaming
) and (self._tool_parser is None or self._tool_parser.engine_based_streaming)
self._stream_state = StreamState(engine_based=self._engine_based)
self._stream_state = StreamState(
tool_call_id_type=(
get_tool_call_id_type(model_config)
if model_config is not None
else "random"
),
engine_based=self._engine_based,
)
@cached_property
def vocab(self) -> dict[str, int]:
@@ -149,6 +163,19 @@ class Parser:
def tool_parser(self, parser: ToolParser | None) -> None:
self._tool_parser = parser
def _initialize_history_tool_call_cnt(
self,
request: ChatCompletionRequest | ResponsesRequest,
) -> None:
state = self._stream_state
if state.history_tool_call_cnt_initialized:
return
if state.tool_call_id_type != "kimi_k2":
state.history_tool_call_cnt_initialized = True
return
state.history_tool_call_cnt = count_history_tool_calls(request)
state.history_tool_call_cnt_initialized = True
# ========== Reasoning Parser Methods ==========
@abstractmethod
@@ -375,6 +402,18 @@ class DelegatingParser(Parser):
return request.tool_choice.function.name
raise ValueError("Invalid tool_choice for function name extraction.")
def _make_tool_call_id(self, function_name: str) -> str | None:
state = self._stream_state
if state.tool_call_id_type != "kimi_k2":
return None
tool_call_id = make_tool_call_id(
id_type=state.tool_call_id_type,
func_name=function_name,
idx=state.history_tool_call_cnt,
)
state.history_tool_call_cnt += 1
return tool_call_id
def _extract_tool_calls(
self,
content: str | None,
@@ -404,9 +443,11 @@ class DelegatingParser(Parser):
if is_named_tool_choice and supports_required_and_named:
if content is None:
return [], None
function_name = self._get_function_name(request)
tool_calls.append(
FunctionCall(
name=self._get_function_name(request),
id=self._make_tool_call_id(function_name),
name=function_name,
arguments=content,
)
)
@@ -422,6 +463,7 @@ class DelegatingParser(Parser):
for tc in parsed_calls:
tool_calls.append(
FunctionCall(
id=self._make_tool_call_id(tc.name),
name=tc.name,
arguments=json.dumps(tc.parameters, ensure_ascii=False),
)
@@ -733,6 +775,7 @@ class DelegatingParser(Parser):
enable_auto_tools: bool = False,
model_output_token_ids: Sequence[int] = (),
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
self._initialize_history_tool_call_cnt(request)
reasoning, content = self.extract_reasoning(model_output, request)
tool_calls, content = self._extract_tool_calls(
content=content,
@@ -750,6 +793,7 @@ class DelegatingParser(Parser):
*,
finished: bool,
) -> DeltaMessage | None:
self._initialize_history_tool_call_cnt(request)
state = self._stream_state
if not state.prompt_reasoning_checked and prompt_token_ids is not None:
vllm/parser/engine/parser_engine.py
+11 -2
View File
@@ -13,7 +13,7 @@ from typing import TYPE_CHECKING
import regex as re
from vllm.entrypoints.chat_utils import make_tool_call_id
from vllm.entrypoints.chat_utils import get_tool_call_id_type, make_tool_call_id
from vllm.entrypoints.openai.engine.protocol import (
DeltaFunctionCall,
DeltaMessage,
@@ -89,11 +89,18 @@ class ParserEngine(Parser):
tools: list[Tool] | None = None,
*,
parser_engine_config: ParserEngineConfig,
model_config=None,
**kwargs,
) -> None:
self.model_tokenizer = tokenizer
self._tools = tools
self._stream_state = StreamState()
self._stream_state = StreamState(
tool_call_id_type=(
get_tool_call_id_type(model_config)
if model_config is not None
else "random"
),
)
self._reasoning_parser = None
self._tool_parser = None
self.parser_engine_config = parser_engine_config
@@ -419,6 +426,7 @@ class ParserEngine(Parser):
*,
finished: bool,
) -> DeltaMessage | None:
self._initialize_history_tool_call_cnt(request)
if not self._prompt_streaming_prepared and prompt_token_ids is not None:
# NOTE: call the hook BEFORE setting the flag, because the hook
# may invoke ``_reset`` (e.g. via ``initialize_streaming``) which
@@ -658,6 +666,7 @@ class ParserEngine(Parser):
enable_auto_tools: bool = False,
model_output_token_ids: Sequence[int] = (),
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
self._initialize_history_tool_call_cnt(request)
self._check_skip_tool_parsing(request)
reasoning, content, tool_call_info = self._single_pass_parse(
model_output,
vllm/parser/utils.py
+65
View File
@@ -0,0 +1,65 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable, Sequence
from openai.types.responses import ResponseFunctionToolCall
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.responses.protocol import (
ResponseInputOutputItem,
ResponsesRequest,
)
def count_tool_calls(tool_calls: object) -> int:
if tool_calls is None:
return 0
if isinstance(tool_calls, (str, bytes, dict)):
return 1
if isinstance(tool_calls, Iterable):
return sum(1 for _ in tool_calls)
return 1
def count_chat_history_tool_calls(
messages: Sequence[ChatCompletionMessageParam],
) -> int:
return sum(
count_tool_calls(msg.get("tool_calls"))
for msg in messages
if isinstance(msg, dict) and msg.get("role") == "assistant"
)
def count_response_history_tool_calls(
response_items: Sequence[ResponseInputOutputItem],
) -> int:
count = 0
for item in response_items:
if isinstance(item, ResponseFunctionToolCall):
count += 1
continue
if isinstance(item, dict):
item_type = item.get("type")
if item_type == "function_call":
count += 1
elif item.get("role") == "assistant":
count += count_tool_calls(item.get("tool_calls"))
return count
def count_history_tool_calls(
request: ChatCompletionRequest | ResponsesRequest,
) -> int:
if isinstance(request, ChatCompletionRequest):
return count_chat_history_tool_calls(request.messages)
request_input = request.input
if isinstance(request_input, str):
return 0
return count_response_history_tool_calls(request_input)