From 54d0c36fff6a33469e10e8276ff9ea05c156331b Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Mon, 1 Jun 2026 21:56:15 -0500 Subject: [PATCH] [CI] Stabilize OpenAI schema fuzzing for malformed structural tags (#44131) Signed-off-by: Andreas Karatzas --- .../openai/chat_completion/test_chat_error.py | 48 +++++++++++- .../completion/test_completion_error.py | 31 ++++++++ .../tool_parsers/test_mistral_tool_parser.py | 28 ++++++- .../openai/chat_completion/protocol.py | 16 ++++ .../entrypoints/openai/completion/protocol.py | 6 ++ vllm/entrypoints/openai/engine/protocol.py | 75 +++++++++++++++++++ 6 files changed, 201 insertions(+), 3 deletions(-) diff --git a/tests/entrypoints/openai/chat_completion/test_chat_error.py b/tests/entrypoints/openai/chat_completion/test_chat_error.py index 582e0792156..e099c282f42 100644 --- a/tests/entrypoints/openai/chat_completion/test_chat_error.py +++ b/tests/entrypoints/openai/chat_completion/test_chat_error.py @@ -6,9 +6,13 @@ from typing import Any from unittest.mock import AsyncMock, MagicMock, patch import pytest +from pydantic import ValidationError from vllm.config.multimodal import MultiModalConfig -from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest +from vllm.entrypoints.openai.chat_completion.protocol import ( + BatchChatCompletionRequest, + ChatCompletionRequest, +) from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat from vllm.entrypoints.openai.engine.protocol import GenerationError from vllm.entrypoints.openai.models.protocol import BaseModelPath @@ -444,3 +448,45 @@ def test_json_schema_response_format_missing_schema(): messages=[{"role": "user", "content": "hello"}], response_format={"type": "json_schema"}, ) + + +@pytest.mark.parametrize("format_value", [None, {}]) +def test_structural_tag_response_format_invalid(format_value): + """Malformed structural tags should be rejected during request validation.""" + with pytest.raises( + ValidationError, + match="Invalid response_format structural_tag", + ): + ChatCompletionRequest( + model=MODEL_NAME, + messages=[{"role": "user", "content": "hello"}], + response_format={"type": "structural_tag", "format": format_value}, + ) + + +@pytest.mark.parametrize("format_value", [None, {}]) +def test_batch_structural_tag_response_format_invalid(format_value): + """Batch chat should reject malformed structural tags at request parsing.""" + with pytest.raises( + ValidationError, + match="Invalid response_format structural_tag", + ): + BatchChatCompletionRequest( + model=MODEL_NAME, + messages=[[{"role": "user", "content": "hello"}]], + response_format={"type": "structural_tag", "format": format_value}, + ) + + +@pytest.mark.parametrize("structural_tag", ["not json", ""]) +def test_structured_outputs_structural_tag_invalid(structural_tag): + """Malformed direct structured_outputs structural tags should be rejected.""" + with pytest.raises( + ValidationError, + match="Invalid structured_outputs structural_tag", + ): + ChatCompletionRequest( + model=MODEL_NAME, + messages=[{"role": "user", "content": "hello"}], + structured_outputs={"structural_tag": structural_tag}, + ) diff --git a/tests/entrypoints/openai/completion/test_completion_error.py b/tests/entrypoints/openai/completion/test_completion_error.py index c95e47fa1b1..71a70a4d0eb 100644 --- a/tests/entrypoints/openai/completion/test_completion_error.py +++ b/tests/entrypoints/openai/completion/test_completion_error.py @@ -6,6 +6,7 @@ from typing import Any from unittest.mock import AsyncMock, MagicMock import pytest +from pydantic import ValidationError from vllm.config.multimodal import MultiModalConfig from vllm.entrypoints.openai.completion.protocol import CompletionRequest @@ -302,6 +303,36 @@ def test_json_schema_response_format_missing_schema(): ) +@pytest.mark.parametrize("format_value", [None, {}]) +def test_structural_tag_response_format_invalid(format_value): + """Malformed structural tags should be rejected during request validation.""" + with pytest.raises( + ValidationError, + match="Invalid response_format structural_tag", + ): + CompletionRequest( + model=MODEL_NAME, + prompt="Test prompt", + max_tokens=10, + response_format={"type": "structural_tag", "format": format_value}, + ) + + +@pytest.mark.parametrize("structural_tag", ["not json", ""]) +def test_structured_outputs_structural_tag_invalid(structural_tag): + """Malformed direct structured_outputs structural tags should be rejected.""" + with pytest.raises( + ValidationError, + match="Invalid structured_outputs structural_tag", + ): + CompletionRequest( + model=MODEL_NAME, + prompt="Test prompt", + max_tokens=10, + structured_outputs={"structural_tag": structural_tag}, + ) + + def test_negative_prompt_token_ids_nested(): """Negative token IDs in prompt (nested list) should raise validation error.""" with pytest.raises(Exception, match="greater than or equal to 0"): diff --git a/tests/tool_parsers/test_mistral_tool_parser.py b/tests/tool_parsers/test_mistral_tool_parser.py index f6a5c6bfb26..c9582159abb 100644 --- a/tests/tool_parsers/test_mistral_tool_parser.py +++ b/tests/tool_parsers/test_mistral_tool_parser.py @@ -1382,7 +1382,20 @@ def test_adjust_request_non_mistral_tokenizer( [ {"regex": r"\d+"}, {"choice": ["a", "b"]}, - {"structural_tag": '{"key": "value"}'}, + { + "structural_tag": json.dumps( + { + "structures": [ + { + "begin": "", + "schema": {"type": "object"}, + "end": "", + } + ], + "triggers": [""], + } + ) + }, {"grammar": "start: 'hello'"}, ], ids=["regex", "choice", "structural_tag", "grammar"], @@ -1404,7 +1417,18 @@ def test_adjust_request_unsupported_response_format( ) -> None: request = _make_request( response_format=StructuralTagResponseFormat( - type="structural_tag", format={"some": "config"} + type="structural_tag", + format={ + "type": "triggered_tags", + "tags": [ + { + "begin": "", + "content": {"type": "any_text"}, + "end": "", + } + ], + "triggers": [""], + }, ), ) result = mistral_tool_parser.adjust_request(request) diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index 0be220fff77..184ace56805 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -30,6 +30,8 @@ from vllm.entrypoints.openai.engine.protocol import ( StructuralTagResponseFormat, ToolCall, UsageInfo, + validate_structural_tag_response_format, + validate_structured_outputs_structural_tag, ) from vllm.exceptions import VLLMValidationError from vllm.logger import init_logger @@ -671,6 +673,9 @@ class ChatCompletionRequest(OpenAIBaseModel): parameter="response_format", ) + if rf_type == "structural_tag": + validate_structural_tag_response_format(response_format) + return data @model_validator(mode="before") @@ -754,6 +759,7 @@ class ChatCompletionRequest(OpenAIBaseModel): "You can only either use constraints for structured outputs " "or tools, not both.", ) + validate_structured_outputs_structural_tag(structured_outputs_kwargs) return data @model_validator(mode="before") @@ -979,6 +985,16 @@ class BatchChatCompletionRequest(OpenAIBaseModel): "Batch chat completions do not support beam search. " "Please set `use_beam_search` to False." ) + response_format = data.get("response_format") + rf_type = ( + response_format.get("type") + if isinstance(response_format, dict) + else getattr(response_format, "type", None) + ) + if rf_type == "structural_tag": + validate_structural_tag_response_format(response_format) + if (structured_outputs := data.get("structured_outputs")) is not None: + validate_structured_outputs_structural_tag(structured_outputs) n = data.get("n", 1) if n is not None and n != 1: raise ValueError( diff --git a/vllm/entrypoints/openai/completion/protocol.py b/vllm/entrypoints/openai/completion/protocol.py index a6c3f9c93dc..30a4f20084e 100644 --- a/vllm/entrypoints/openai/completion/protocol.py +++ b/vllm/entrypoints/openai/completion/protocol.py @@ -18,6 +18,8 @@ from vllm.entrypoints.openai.engine.protocol import ( StreamOptions, StructuralTagResponseFormat, UsageInfo, + validate_structural_tag_response_format, + validate_structured_outputs_structural_tag, ) from vllm.exceptions import VLLMValidationError from vllm.logger import init_logger @@ -370,6 +372,9 @@ class CompletionRequest(OpenAIBaseModel): parameter="response_format", ) + if rf_type == "structural_tag": + validate_structural_tag_response_format(response_format) + return data @model_validator(mode="before") @@ -397,6 +402,7 @@ class CompletionRequest(OpenAIBaseModel): "outputs ('json', 'regex' or 'choice').", parameter="structured_outputs", ) + validate_structured_outputs_structural_tag(structured_outputs_kwargs) return data @model_validator(mode="before") diff --git a/vllm/entrypoints/openai/engine/protocol.py b/vllm/entrypoints/openai/engine/protocol.py index 890af0300ef..434888df9ef 100644 --- a/vllm/entrypoints/openai/engine/protocol.py +++ b/vllm/entrypoints/openai/engine/protocol.py @@ -17,6 +17,7 @@ from pydantic import ( ) from vllm.entrypoints.chat_utils import make_tool_call_id +from vllm.exceptions import VLLMValidationError from vllm.logger import init_logger from vllm.utils import random_uuid from vllm.utils.import_utils import resolve_obj_by_qualname @@ -158,6 +159,80 @@ AnyResponseFormat: TypeAlias = ( ) +def validate_structural_tag_response_format( + response_format: AnyStructuralTagResponseFormat | dict[str, Any], +) -> None: + """Validate structural tags before they are sent to the engine. + + Engine-side validation reports malformed structural tags as generation + failures. OpenAI request parsing should classify them as bad requests. + """ + import json + + from pydantic import TypeAdapter, ValidationError + + if isinstance(response_format, dict): + try: + response_format = TypeAdapter( + AnyStructuralTagResponseFormat + ).validate_python(response_format) + except ValidationError as exc: + raise VLLMValidationError( + "Invalid response_format structural_tag specification.", + parameter="response_format", + ) from exc + + try: + payload = json.dumps(response_format.model_dump(by_alias=True)) + validate_structural_tag_payload(payload, parameter="response_format") + except (TypeError, ValueError) as exc: + raise VLLMValidationError( + "Invalid response_format structural_tag specification.", + parameter="response_format", + ) from exc + + +def validate_structural_tag_payload(payload: Any, *, parameter: str) -> None: + from vllm.sampling_params import SamplingParams, StructuredOutputsParams + from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar + + if isinstance(payload, str) and not payload: + raise VLLMValidationError( + f"Invalid {parameter} structural_tag specification.", + parameter=parameter, + ) + + try: + validate_xgrammar_grammar( + SamplingParams( + structured_outputs=StructuredOutputsParams(structural_tag=payload) + ) + ) + except (TypeError, ValueError) as exc: + raise VLLMValidationError( + f"Invalid {parameter} structural_tag specification.", + parameter=parameter, + ) from exc + + +def validate_structured_outputs_structural_tag( + structured_outputs: Any, +) -> None: + from vllm.sampling_params import StructuredOutputsParams + + if isinstance(structured_outputs, StructuredOutputsParams): + structural_tag = structured_outputs.structural_tag + elif isinstance(structured_outputs, dict): + structural_tag = structured_outputs.get("structural_tag") + else: + return + if structural_tag is not None: + validate_structural_tag_payload( + structural_tag, + parameter="structured_outputs", + ) + + class StreamOptions(OpenAIBaseModel): include_usage: bool | None = False continuous_usage_stats: bool | None = False