A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 05:00 UTC
Fix Qwen3 streaming content routing (#40820)
Signed-off-by: xy3 <120182408@qq.com> Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: sfeng33 <4florafeng@gmail.com>
xy3 committed
May 7, 2026 at 05:22 UTC
50acdc5b5cc00f10408d8f98b21fc97efc615173
4 files changed
+220
-7
tests/entrypoints/openai/chat_completion/test_thinking_token_budget.py
+44
@@ -287,3 +287,47 @@ async def test_thinking_token_budget_qwen35_fp8_mtp_concurrent_mixed_budget_and_
287
f"index {i}: reasoning decode token ids ({n_reason}) != "
288
f"thinking_token_budget ({expected_budget})"
289
)
290
+
291
+
292
+@pytest.mark.asyncio
293
+@pytest.mark.parametrize("client", ["default", "auto_config"], indirect=True)
294
+async def test_streaming_with_thinking_disabled_stays_in_content(
295
+ client: openai.AsyncOpenAI,
296
+):
297
+ request_kwargs = {
298
+ "model": MODEL_NAME,
299
+ "messages": [
300
+ {
301
+ "role": "user",
302
+ "content": "Which is larger, 4 or 12?"
303
+ " Output exactly one token: 4 or 12.",
304
+ }
305
+ ],
306
+ "max_tokens": 16,
307
+ "temperature": 0.0,
308
+ "extra_body": {"chat_template_kwargs": {"enable_thinking": False}},
309
+ }
310
+
311
+ response = await client.chat.completions.create(**request_kwargs)
312
+ message = response.choices[0].message
313
+ assert message.content is not None and message.content.strip() != ""
314
+ assert getattr(message, "reasoning", None) in (None, "")
315
+
316
+ stream = await client.chat.completions.create(
317
+ **request_kwargs,
318
+ stream=True,
319
+ )
320
+
321
+ content_chunks = []
322
+ reasoning_chunks = []
323
+ async for chunk in stream:
324
+ if not chunk.choices:
325
+ continue
326
+ delta = chunk.choices[0].delta
327
+ if getattr(delta, "content", None):
328
+ content_chunks.append(delta.content)
329
+ if getattr(delta, "reasoning", None):
330
+ reasoning_chunks.append(delta.reasoning)
331
+
332
+ assert "".join(content_chunks).strip() != ""
333
+ assert reasoning_chunks == []
tests/parser/__init__.py
new
+2
@@ -0,0 +1,2 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
tests/parser/test_streaming.py
new
+165
@@ -0,0 +1,165 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import json
5
+
6
+import pytest
7
+
8
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
9
+from vllm.entrypoints.openai.engine.protocol import DeltaMessage
10
+from vllm.parser.abstract_parser import _WrappedParser
11
+from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
12
+from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
13
+
14
+
15
+class ThinkReasoningParser(BaseThinkingReasoningParser):
16
+ @property
17
+ def start_token(self) -> str:
18
+ return "<think>"
19
+
20
+ @property
21
+ def end_token(self) -> str:
22
+ return "</think>"
23
+
24
+
25
+MODEL_OUTPUT = (
26
+ "<think>let me think about this</think>"
27
+ '<tool_call>\n{"name": "get_weather", '
28
+ '"arguments": {"city": "Dallas"}}\n</tool_call>'
29
+)
30
+
31
+
32
+@pytest.fixture(scope="module")
33
+def tokenizer():
34
+ from vllm.tokenizers import get_tokenizer
35
+
36
+ return get_tokenizer("Qwen/Qwen3-32B")
37
+
38
+
39
+@pytest.fixture
40
+def request_obj():
41
+ return ChatCompletionRequest(
42
+ model="test-model",
43
+ messages=[{"role": "user", "content": "hi"}],
44
+ )
45
+
46
+
47
+def make_parser(tokenizer, reasoning=False, tool=False):
48
+ _WrappedParser.reasoning_parser_cls = ThinkReasoningParser if reasoning else None
49
+ _WrappedParser.tool_parser_cls = Hermes2ProToolParser if tool else None
50
+ return _WrappedParser(tokenizer)
51
+
52
+
53
+def stream_text(parser, tokenizer, text, request, prompt_token_ids=None):
54
+ token_ids = tokenizer.encode(text, add_special_tokens=False)
55
+ results: list[DeltaMessage | None] = []
56
+ for tid in token_ids:
57
+ delta_text = tokenizer.decode([tid])
58
+ result = parser.parse_delta(
59
+ delta_text, [tid], request, prompt_token_ids=prompt_token_ids
60
+ )
61
+ prompt_token_ids = None
62
+ results.append(result)
63
+ return results
64
+
65
+
66
+def collect_fields(results):
67
+ all_reasoning = "".join(r.reasoning for r in results if r and r.reasoning)
68
+ all_content = "".join(r.content for r in results if r and r.content)
69
+ all_tool_calls = [tc for r in results if r and r.tool_calls for tc in r.tool_calls]
70
+ return all_reasoning, all_content, all_tool_calls
71
+
72
+
73
+def test_parse_delta_neither_parser(tokenizer, request_obj):
74
+ parser = make_parser(tokenizer, reasoning=False, tool=False)
75
+ results = stream_text(
76
+ parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
77
+ )
78
+ reasoning, content, tool_calls = collect_fields(results)
79
+
80
+ assert reasoning == ""
81
+ assert len(tool_calls) == 0
82
+ assert "<think>" in content
83
+ assert "let me think about this" in content
84
+ assert "<tool_call>" in content
85
+ assert "get_weather" in content
86
+
87
+
88
+def test_parse_delta_tool_parser_only(tokenizer, request_obj):
89
+ parser = make_parser(tokenizer, reasoning=False, tool=True)
90
+ results = stream_text(
91
+ parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
92
+ )
93
+ reasoning, content, tool_calls = collect_fields(results)
94
+
95
+ assert reasoning == ""
96
+ assert "<think>" in content
97
+ assert "let me think about this" in content
98
+ assert "</think>" in content
99
+
100
+ assert len(tool_calls) > 0
101
+ assert tool_calls[0].function.name == "get_weather"
102
+ tool_args = "".join(
103
+ tc.function.arguments for tc in tool_calls if tc.function.arguments
104
+ )
105
+ assert json.loads(tool_args) == {"city": "Dallas"}
106
+
107
+
108
+def test_parse_delta_reasoning_parser_only(tokenizer, request_obj):
109
+ parser = make_parser(tokenizer, reasoning=True, tool=False)
110
+ results = stream_text(
111
+ parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
112
+ )
113
+ reasoning, content, tool_calls = collect_fields(results)
114
+
115
+ assert "let me think about this" in reasoning
116
+ assert len(tool_calls) == 0
117
+ assert "<tool_call>" in content
118
+ assert "get_weather" in content
119
+ assert "</tool_call>" in content
120
+
121
+
122
+def test_parse_delta_both_parsers(tokenizer, request_obj):
123
+ parser = make_parser(tokenizer, reasoning=True, tool=True)
124
+ results = stream_text(
125
+ parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
126
+ )
127
+ reasoning, content, tool_calls = collect_fields(results)
128
+
129
+ assert "let me think about this" in reasoning
130
+ assert content == ""
131
+
132
+ assert len(tool_calls) > 0
133
+ assert tool_calls[0].function.name == "get_weather"
134
+ tool_args = "".join(
135
+ tc.function.arguments for tc in tool_calls if tc.function.arguments
136
+ )
137
+ assert json.loads(tool_args) == {"city": "Dallas"}
138
+
139
+
140
+def test_parse_delta_reasoning_only_thinking_disabled(tokenizer, request_obj):
141
+ """Regression test for vllm-project/vllm#40466.
142
+
143
+ When enable_thinking=False, the chat template places <think>\\n\\n</think>
144
+ in the prompt. The model then generates pure content (no think tokens).
145
+ All streaming output must go to delta.content, not delta.reasoning.
146
+ """
147
+ parser = make_parser(tokenizer, reasoning=True, tool=False)
148
+
149
+ end_token_id = parser._reasoning_parser.end_token_id
150
+ prompt_token_ids = [1, 2, end_token_id, 3]
151
+
152
+ content_text = "Hello! How can I assist you today?"
153
+ results = stream_text(
154
+ parser,
155
+ tokenizer,
156
+ content_text,
157
+ request_obj,
158
+ prompt_token_ids=prompt_token_ids,
159
+ )
160
+ reasoning, content, tool_calls = collect_fields(results)
161
+
162
+ assert reasoning == "", f"Expected no reasoning, got: {reasoning!r}"
163
+ assert "Hello" in content
164
+ assert "assist" in content
165
+ assert len(tool_calls) == 0
vllm/parser/abstract_parser.py
+9
-7
@@ -636,15 +636,11 @@ class DelegatingParser(Parser):
636
def _in_reasoning_phase(self, state: StreamState) -> bool:
637
if self._reasoning_parser is None:
638
return False
639
- if self._tool_parser is None:
640
- return True
639
return not state.reasoning_ended
640
641
def _in_tool_call_phase(self, state: StreamState) -> bool:
642
if self._tool_parser is None:
643
return False
646
- if self._reasoning_parser is None:
647
- return True
644
return state.reasoning_ended
645
646
def parse_delta(
@@ -658,7 +654,9 @@ class DelegatingParser(Parser):
654
655
if not state.prompt_reasoning_checked and prompt_token_ids is not None:
656
state.prompt_reasoning_checked = True
661
- if self.is_reasoning_end(prompt_token_ids):
657
+ if self._reasoning_parser is None or self.is_reasoning_end(
658
+ prompt_token_ids
659
+ ):
660
state.reasoning_ended = True
661
662
current_text = state.previous_text + delta_text
@@ -709,8 +707,12 @@ class DelegatingParser(Parser):
707
)
708
)
709
712
- # No parsers: pass through as content
713
- if self._reasoning_parser is None and self._tool_parser is None:
710
+ # No phase active: pass through as content
711
+ if (
712
+ delta_message is None
713
+ and not self._in_reasoning_phase(state)
714
+ and not self._in_tool_call_phase(state)
715
+ ):
716
delta_message = DeltaMessage(content=delta_text)
717
718
state.previous_text = current_text