[Bugfix][Parser] Fix special tokens (EOS/BOS) leaking into reasoning content (#48748)

Signed-off-by: Ben Browning <[email protected]>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

tests/parser/engine/test_deepseek_v4.py

@@ -920,3 +920,45 @@ class TestDelegatingParserLargeDelta:
         assert output.tool_calls[0]["name"] == "get_weather"
         args = json.loads(output.tool_calls[0]["arguments"])
         assert args == {"location": "Berlin"}
+
+    @pytest.mark.parametrize(
+        "chunk_size",
+        [1, 2, 3, 5, None],
+        ids=lambda c: f"chunk={c}",
+    )
+    def test_eos_not_leaked_when_reasoning_never_ends(self, chunk_size):
+        """EOS must not leak into reasoning_content when the model never
+        emits </think> (generation ends while still in REASONING state)."""
+        eos_text = "<｜end▁of▁sentence｜>"
+        eos_id = 128801
+        vocab = {
+            **_DSV4_FULL_VOCAB,
+            eos_text: eos_id,
+        }
+
+        reasoning_text = "Good morning! How can I help you today?"
+        tokens: list[tuple[int, str]] = []
+        tid = 100
+        for word in reasoning_text.split(" "):
+            prefix = " " if tokens else ""
+            tokens.append((tid, prefix + word))
+            tid += 1
+        tokens.append((eos_id, eos_text))
+
+        tokenizer = MockTokenizer(vocab=vocab, tokens=tokens)
+        parser = _DeepSeekV4Delegating(
+            tokenizer,
+            chat_template_kwargs={"thinking": True},
+        )
+        deltas = replay_streaming(
+            parser,
+            tokens,
+            chunk_size=chunk_size,
+            finished_on_last=True,
+        )
+        output = collect_output(deltas)
+
+        assert reasoning_text in output.reasoning
+        assert eos_text not in output.reasoning
+        assert output.content == ""
+        assert output.tool_calls == []

tests/parser/engine/test_parser_engine.py

@@ -1624,19 +1624,41 @@ class TestDropSpecialTokens:
         assert delta is not None
         assert "<bos>" in delta.reasoning
 
-    def test_drops_suppressed_with_skip_tool_parsing(self):
-        """When skip_tool_parsing is active, drop tokens are preserved
-        as content so a later tool-call pass can see them."""
+    def test_drops_applied_with_skip_tool_parsing(self):
+        """Drop tokens are always dropped, even with skip_tool_parsing.
+        DROP_TERMINALs have no transitions by construction, so no parser
+        pass can use them."""
+        for initial_state in (ParserState.REASONING, ParserState.CONTENT):
+            engine = _make_engine(
+                vocab=_DROP_VOCAB,
+                special_tokens=list(_DROP_VOCAB.keys()),
+            )
+            engine._engine.skip_tool_parsing = True
+            engine._engine.reset(initial_state=initial_state)
+            events = engine._engine.feed("hello<bos>world", [72, 204, 73])
+            delta = engine._events_to_delta(events)
+            assert delta is not None
+            output = (delta.reasoning or "") + (delta.content or "")
+            assert "<bos>" not in output, f"<bos> leaked in state {initial_state}"
+
+    def test_transitions_unaffected_by_drop_in_reasoning_with_skip_tool_parsing(self):
+        """With skip_tool_parsing in REASONING state, drop tokens are
+        removed but configured terminals still fire their transitions."""
         engine = _make_engine(
             vocab=_DROP_VOCAB,
             special_tokens=list(_DROP_VOCAB.keys()),
         )
         engine._engine.skip_tool_parsing = True
         engine._engine.reset()
-        events = engine._engine.feed("hello<bos>world", [72, 204, 73])
-        delta = engine._events_to_delta(events)
-        assert delta is not None
-        assert "<bos>" in delta.reasoning
+        events = engine._engine.feed("thought<bos></think>answer", [72, 204, 201, 73])
+        types = [e.type for e in events]
+        assert EventType.REASONING_CHUNK in types
+        assert EventType.REASONING_END in types
+        assert EventType.TEXT_CHUNK in types
+        reasoning_text = "".join(
+            e.value for e in events if e.type == EventType.REASONING_CHUNK
+        )
+        assert "<bos>" not in reasoning_text
 
     def test_drops_in_tool_args_state(self):
         """Drop tokens in TOOL_ARGS state are silently discarded."""

vllm/parser/engine/streaming_parser_engine.py

@@ -304,14 +304,7 @@ class StreamingParserEngine:
         transition = self.config.transitions.get(key)
 
         if transition is None:
-            if (
-                self._has_drops
-                and terminal == DROP_TERMINAL
-                # Preserve drop tokens when skip_tool_parsing is active so
-                # the reasoning pass doesn't silently remove tokens that a
-                # later tool-call pass might need to see.
-                and not self.skip_tool_parsing
-            ):
+            if self._has_drops and terminal == DROP_TERMINAL:
                 return []
             return self._emit_for_state(value)