@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 40f88d8318
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 7, 22:45 UTC

[Bugfix] Fix Qwen3/Qwen3.5 Reasoning Parser (#34779)

Signed-off-by: Roger Wang <hey@rogerw.io>

Roger Wang committed Feb 21, 2026 at 23:15 UTC 40f88d8318aea1792ac7eabfe33241fd26660be7
3 files changed +233 -47
tests/reasoning/test_qwen3_reasoning_parser.py
+121 -12
@@ -4,46 +4,79 @@
4 import pytest
5 from transformers import AutoTokenizer
6
7 -from tests.reasoning.utils import run_reasoning_extraction
7 +from tests.reasoning.utils import (
8 + StreamingReasoningReconstructor,
9 + run_reasoning_extraction,
10 + run_reasoning_extraction_streaming,
11 +)
12 from vllm.reasoning import ReasoningParser, ReasoningParserManager
13
14 parser_name = "qwen3"
15 start_token = "<think>"
16 end_token = "</think>"
17
14 -REASONING_MODEL_NAME = "Qwen/Qwen3-0.6B"
18 +REASONING_MODEL_NAMES = [
19 + "Qwen/Qwen3-0.6B",
20 + "Qwen/Qwen3.5-397B-A17B",
21 + "Qwen/Qwen3-4B-Thinking-2507",
22 +]
23 +
24 +
25 +@pytest.fixture(scope="module", params=REASONING_MODEL_NAMES)
26 +def qwen3_tokenizer(request):
27 + return AutoTokenizer.from_pretrained(request.param)
28 +
29
30 +# --- <think> in prompt, only </think> in output (typical) ---
31
17 -@pytest.fixture(scope="module")
18 -def qwen3_tokenizer():
19 - return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
32 +WITHOUT_START_TOKEN = {
33 + "output": "This is a reasoning section</think>This is the rest",
34 + "reasoning": "This is a reasoning section",
35 + "content": "This is the rest",
36 +}
37 +WITHOUT_START_TOKEN_STREAM = {
38 + "output": "This is a reasoning section</think>This is the rest",
39 + "reasoning": "This is a reasoning section",
40 + "content": "This is the rest",
41 +}
42 +WITHOUT_START_TOKEN_COMPLETE_REASONING = {
43 + "output": "This is a reasoning section</think>",
44 + "reasoning": "This is a reasoning section",
45 + "content": None,
46 +}
47
48 +# --- <think> present in output (old template / edge case) ---
49
22 -# 带 <think></think>，非stream
50 WITH_THINK = {
51 "output": "<think>This is a reasoning section</think>This is the rest",
52 "reasoning": "This is a reasoning section",
53 "content": "This is the rest",
54 }
28 -# 带 <think></think>，stream
55 WITH_THINK_STREAM = {
56 "output": "<think>This is a reasoning section</think>This is the rest",
57 "reasoning": "This is a reasoning section",
58 "content": "This is the rest",
59 }
34 -# 不带 <think></think>，非stream
60 +
61 +# --- No think tokens at all (thinking disabled) ---
62 +
63 WITHOUT_THINK = {
64 "output": "This is the rest",
65 "reasoning": None,
66 "content": "This is the rest",
67 }
40 -# 不带 <think></think>，stream
68 +# In streaming, the parser cannot distinguish "thinking disabled" from
69 +# "reasoning in progress" when no think tokens have appeared yet.
70 +# It assumes reasoning. The serving layer handles the "thinking disabled"
71 +# case by checking prompt_is_reasoning_end_arr before calling the parser.
72 WITHOUT_THINK_STREAM = {
73 "output": "This is the rest",
43 - "reasoning": None,
44 - "content": "This is the rest",
74 + "reasoning": "This is the rest",
75 + "content": None,
76 }
77
78 +# --- Edge cases ---
79 +
80 COMPLETE_REASONING = {
81 "output": "<think>This is a reasoning section</think>",
82 "reasoning": "This is a reasoning section",
@@ -57,7 +90,7 @@ MULTILINE_REASONING = {
90 ONLY_OPEN_TAG = {
91 "output": "<think>This is a reasoning section",
92 "reasoning": None,
60 - "content": "<think>This is a reasoning section",
93 + "content": "This is a reasoning section",
94 }
95
96 ONLY_OPEN_TAG_STREAM = {
@@ -67,6 +100,26 @@ ONLY_OPEN_TAG_STREAM = {
100 }
101
102 TEST_CASES = [
103 + pytest.param(
104 + False,
105 + WITHOUT_START_TOKEN,
106 + id="without_start_token",
107 + ),
108 + pytest.param(
109 + True,
110 + WITHOUT_START_TOKEN_STREAM,
111 + id="without_start_token_stream",
112 + ),
113 + pytest.param(
114 + False,
115 + WITHOUT_START_TOKEN_COMPLETE_REASONING,
116 + id="without_start_token_complete_reasoning",
117 + ),
118 + pytest.param(
119 + True,
120 + WITHOUT_START_TOKEN_COMPLETE_REASONING,
121 + id="without_start_token_complete_reasoning_stream",
122 + ),
123 pytest.param(
124 False,
125 WITH_THINK,
@@ -140,3 +193,59 @@ def test_reasoning(
193
194 assert reasoning == param_dict["reasoning"]
195 assert content == param_dict["content"]
196 +
197 +
198 +# Multi-token delta tests: simulate real-world streaming where a single
199 +# delta can contain multiple tokens (e.g., speculative decoding).
200 +MULTI_TOKEN_DELTA_CASES = [
201 + pytest.param(
202 + # <think> grouped with following text in one delta
203 + ["<think>This is a reasoning section", "</think>", "This is the rest"],
204 + "This is a reasoning section",
205 + "This is the rest",
206 + id="start_token_grouped_with_text",
207 + ),
208 + pytest.param(
209 + # </think> grouped with following content in one delta
210 + ["reasoning section", "</think>This is the rest"],
211 + "reasoning section",
212 + "This is the rest",
213 + id="end_token_grouped_with_content",
214 + ),
215 + pytest.param(
216 + # <think> and </think> in the same delta, no content after
217 + ["<think>reasoning</think>"],
218 + "reasoning",
219 + None,
220 + id="start_and_end_in_one_delta_no_content",
221 + ),
222 + pytest.param(
223 + # No start token, end grouped with content (Qwen3.5 style)
224 + ["reasoning section", "</think>content"],
225 + "reasoning section",
226 + "content",
227 + id="no_start_end_grouped_with_content",
228 + ),
229 +]
230 +
231 +
232 +@pytest.mark.parametrize(
233 + "deltas, expected_reasoning, expected_content", MULTI_TOKEN_DELTA_CASES
234 +)
235 +def test_reasoning_streaming_multi_token_deltas(
236 + deltas: list[str],
237 + expected_reasoning: str | None,
238 + expected_content: str | None,
239 + qwen3_tokenizer,
240 +):
241 + """Test that multi-token deltas don't leak <think> into reasoning."""
242 + parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(parser_name)(
243 + qwen3_tokenizer
244 + )
245 +
246 + reconstructor: StreamingReasoningReconstructor = run_reasoning_extraction_streaming(
247 + parser, deltas
248 + )
249 +
250 + assert reconstructor.reasoning == expected_reasoning
251 + assert (reconstructor.other_content or None) == expected_content
vllm/entrypoints/openai/chat_completion/serving.py
+31 -16
@@ -900,6 +900,17 @@ class OpenAIServingChat(OpenAIServing):
900 harmony_tools_streamed[i] |= tools_streamed_flag
901 # handle streaming deltas for tools with named tool_choice
902 elif tool_choice_function_name:
903 + # When encountering think end id in prompt_token_ids
904 + # i.e {"enable_thinking": False},
905 + # check BEFORE calling the parser to avoid a spurious
906 + # reasoning delta on the first chunk.
907 + if (
908 + reasoning_parser
909 + and not reasoning_end_arr[i]
910 + and prompt_is_reasoning_end_arr[i]
911 + ):
912 + reasoning_end_arr[i] = True
913 +
914 if (
915 reasoning_parser
916 and not reasoning_end_arr[i]
@@ -918,16 +929,11 @@ class OpenAIServingChat(OpenAIServing):
929 output.token_ids,
930 )
931 )
921 - # When encountering think end id in delta_token_ids
922 - # or think end id in prompt_token_ids
923 - # i.e {"enable_thinking": False},
932 + # When encountering think end id in delta_token_ids,
933 # set reasoning status to end.
934 # Only keep 'content', remove 'reasoning'.
926 - if (
927 - reasoning_parser.is_reasoning_end(
928 - as_list(output.token_ids)
929 - )
930 - or prompt_is_reasoning_end_arr[i]
935 + if reasoning_parser.is_reasoning_end(
936 + as_list(output.token_ids)
937 ):
938 reasoning_end_arr[i] = True
939 if delta_message and delta_message.content:
@@ -1116,14 +1122,23 @@ class OpenAIServingChat(OpenAIServing):
1122
1123 # when only reasoning
1124 elif reasoning_parser:
1119 - delta_message = reasoning_parser.extract_reasoning_streaming(
1120 - previous_text,
1121 - current_text,
1122 - delta_text,
1123 - previous_token_ids,
1124 - current_token_ids,
1125 - output.token_ids,
1126 - )
1125 + # When encountering think end id in prompt_token_ids
1126 + # i.e {"enable_thinking": False},
1127 + # set reasoning status to end.
1128 + # Route all generated tokens as content directly.
1129 + if prompt_is_reasoning_end_arr[i]:
1130 + delta_message = DeltaMessage(content=delta_text)
1131 + else:
1132 + delta_message = (
1133 + reasoning_parser.extract_reasoning_streaming(
1134 + previous_text,
1135 + current_text,
1136 + delta_text,
1137 + previous_token_ids,
1138 + current_token_ids,
1139 + output.token_ids,
1140 + )
1141 + )
1142 # handle streaming just a content delta
1143 else:
1144 delta_message = DeltaMessage(content=delta_text)
vllm/reasoning/qwen3_reasoning_parser.py
+81 -19
@@ -1,9 +1,12 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 +from collections.abc import Sequence
5 +
6 from vllm.entrypoints.openai.chat_completion.protocol import (
7 ChatCompletionRequest,
8 )
9 +from vllm.entrypoints.openai.engine.protocol import DeltaMessage
10 from vllm.entrypoints.openai.responses.protocol import (
11 ResponsesRequest,
12 )
@@ -12,13 +15,22 @@ from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
15
16 class Qwen3ReasoningParser(BaseThinkingReasoningParser):
17 """
15 - Reasoning parser for the Qwen3 model.
18 + Reasoning parser for the Qwen3/Qwen3.5 model family.
19 +
20 + The Qwen3 model family uses <think>...</think> tokens to denote reasoning
21 + text. Starting with Qwen3.5, the chat template places <think> in the
22 + prompt so only </think> appears in the generated output. The model
23 + provides a strict switch to disable reasoning output via the
24 + 'enable_thinking=False' parameter.
25 +
26 + When thinking is disabled, the template places <think>\\n\\n</think>\\n\\n
27 + in the prompt. The serving layer detects this via prompt_is_reasoning_end
28 + and routes deltas as content without calling the streaming parser.
29
17 - The Qwen3 model uses <think>...</think> tokens to denote reasoning text
18 - within its output. The model provides a strict switch to disable reasoning
19 - output via the 'enable_thinking=False' parameter. This parser extracts the
20 - reasoning content enclosed by <think> and </think> tokens from the model's
21 - output.
30 + NOTE: Models up to the 2507 release (e.g., Qwen/Qwen3-235B-A22B-Instruct-2507)
31 + use an older chat template where the model generates <think> itself.
32 + This parser handles both styles: if <think> appears in the generated output
33 + it is stripped before extraction (non-streaming) or skipped (streaming).
34 """
35
36 @property
@@ -37,31 +49,27 @@ class Qwen3ReasoningParser(BaseThinkingReasoningParser):
49 """
50 Extract reasoning content from the model output.
51
40 - Qwen3 has stricter requirements - it needs both start and end tokens
41 - to be present, unlike other models that work with just the end token.
52 + The <think> token is placed in the prompt by the chat template,
53 + so typically only </think> appears in the generated output.
54 + If <think> is present (e.g. from a different template), it is
55 + stripped before extraction.
56
43 - For text <think>abc</think>xyz:
44 - - 'abc' goes to reasoning
45 - - 'xyz' goes to content
57 + When thinking is disabled (no </think> in output), returns
58 + (None, model_output) to indicate all output is content.
59
60 Returns:
61 tuple[Optional[str], Optional[str]]: reasoning content and content
62 """
63
51 - # Check if the model output contains both <think> and </think> tokens.
52 - if self.start_token not in model_output or self.end_token not in model_output:
53 - return None, model_output
54 -
55 - # Check if the <think> is present in the model output, remove it
56 - # if it is present.
64 + # Strip <think> if present in the generated output.
65 model_output_parts = model_output.partition(self.start_token)
66 model_output = (
67 model_output_parts[2] if model_output_parts[1] else model_output_parts[0]
68 )
69
62 - # Check if the model output contains the </think> tokens.
63 - # If the end token is not found, return the model output as is.
70 if self.end_token not in model_output:
71 + # No end token means thinking is disabled or the model
72 + # did not produce reasoning. Treat everything as content.
73 return None, model_output
74
75 # Extract reasoning content from the model output.
@@ -69,3 +77,57 @@ class Qwen3ReasoningParser(BaseThinkingReasoningParser):
77
78 final_content = content or None
79 return reasoning, final_content
80 +
81 + def extract_reasoning_streaming(
82 + self,
83 + previous_text: str,
84 + current_text: str,
85 + delta_text: str,
86 + previous_token_ids: Sequence[int],
87 + current_token_ids: Sequence[int],
88 + delta_token_ids: Sequence[int],
89 + ) -> DeltaMessage | None:
90 + """
91 + Extract reasoning content from a streaming delta.
92 +
93 + Since <think> is placed in the prompt by the chat template, all
94 + generated tokens before </think> are reasoning and tokens after
95 + are content.
96 +
97 + NOTE: When thinking is disabled, no think tokens appear in the
98 + generated output. The serving layer detects this via
99 + prompt_is_reasoning_end and routes deltas as content without
100 + calling this method.
101 + """
102 + # Strip <think> from delta if present (old template / edge case
103 + # where the model generates <think> itself).
104 + if self.start_token_id in delta_token_ids:
105 + start_idx = delta_text.find(self.start_token)
106 + if start_idx >= 0:
107 + delta_text = delta_text[start_idx + len(self.start_token) :]
108 +
109 + if self.end_token_id in delta_token_ids:
110 + # End token in this delta: split reasoning from content.
111 + end_index = delta_text.find(self.end_token)
112 + if end_index >= 0:
113 + reasoning = delta_text[:end_index]
114 + content = delta_text[end_index + len(self.end_token) :]
115 + if not reasoning and not content:
116 + return None
117 + return DeltaMessage(
118 + reasoning=reasoning if reasoning else None,
119 + content=content if content else None,
120 + )
121 + # end_token_id in IDs but not in text (already stripped)
122 + return None
123 +
124 + # No end token in this delta.
125 + if not delta_text:
126 + # Nothing left after stripping start token.
127 + return None
128 + elif self.end_token_id in previous_token_ids:
129 + # End token already passed: everything is content now.
130 + return DeltaMessage(content=delta_text)
131 + else:
132 + # No end token yet: still in reasoning phase.
133 + return DeltaMessage(reasoning=delta_text)