A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 17:30 UTC
[Bugfix] Treat <tool_call> as implicit reasoning end in Qwen3 parser (#35687)
Signed-off-by: Doug Campos <qmx@qmx.me>
Doug Campos committed
Apr 23, 2026 at 21:10 UTC
92762edc535c696c3b8a5f3ffee9bc1c0fac10e6
2 files changed
+148
-12
tests/reasoning/test_qwen3_reasoning_parser.py
+52
@@ -78,6 +78,25 @@ WITHOUT_THINK_STREAM = {
78
"content": None,
79
}
80
81
+# --- <tool_call> without </think> (implicit reasoning end) ---
82
+
83
+TOOL_CALL_BODY = (
84
+ "<tool_call>\n<function=bash>\n<parameter=command>"
85
+ "\ncat /etc/hosts\n</parameter>\n</function>\n</tool_call>"
86
+)
87
+
88
+TOOL_CALL_NO_THINK_END = {
89
+ "output": "I need to read the file.\n\n" + TOOL_CALL_BODY,
90
+ "reasoning": "I need to read the file.\n\n",
91
+ "content": TOOL_CALL_BODY,
92
+}
93
+
94
+TOOL_CALL_WITH_THINK_NO_END = {
95
+ "output": "<think>I need to read the file.\n\n" + TOOL_CALL_BODY,
96
+ "reasoning": "I need to read the file.\n\n",
97
+ "content": TOOL_CALL_BODY,
98
+}
99
+
100
# --- Edge cases ---
101
102
COMPLETE_REASONING = {
@@ -199,6 +218,26 @@ TEST_CASES = [
218
TRUNCATED_NO_START_TOKEN_STREAM,
219
id="truncated_no_start_token_stream",
220
),
221
+ pytest.param(
222
+ False,
223
+ TOOL_CALL_NO_THINK_END,
224
+ id="tool_call_no_think_end",
225
+ ),
226
+ pytest.param(
227
+ True,
228
+ TOOL_CALL_NO_THINK_END,
229
+ id="tool_call_no_think_end_stream",
230
+ ),
231
+ pytest.param(
232
+ False,
233
+ TOOL_CALL_WITH_THINK_NO_END,
234
+ id="tool_call_with_think_no_end",
235
+ ),
236
+ pytest.param(
237
+ True,
238
+ TOOL_CALL_WITH_THINK_NO_END,
239
+ id="tool_call_with_think_no_end_stream",
240
+ ),
241
]
242
243
@@ -255,6 +294,13 @@ MULTI_TOKEN_DELTA_CASES = [
294
"content",
295
id="no_start_end_grouped_with_content",
296
),
297
+ pytest.param(
298
+ # <tool_call> arrives in a separate delta after reasoning text
299
+ ["I need to read the file.\n\n", "<tool_call>\n<function=bash>"],
300
+ "I need to read the file.\n\n",
301
+ "<tool_call>\n<function=bash>",
302
+ id="tool_call_implicit_reasoning_end",
303
+ ),
304
]
305
306
@@ -296,6 +342,12 @@ THINKING_DISABLED_CASES = [
342
"Some output without think tokens",
343
id="thinking_disabled_no_think_tokens",
344
),
345
+ pytest.param(
346
+ "I need to read the file.\n\n" + TOOL_CALL_BODY,
347
+ None,
348
+ "I need to read the file.\n\n" + TOOL_CALL_BODY,
349
+ id="thinking_disabled_with_tool_call",
350
+ ),
351
]
352
353
vllm/reasoning/qwen3_reasoning_parser.py
+96
-12
@@ -1,7 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-from collections.abc import Sequence
4
+from collections.abc import Iterable, Sequence
5
from typing import TYPE_CHECKING
6
7
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
@@ -31,6 +31,10 @@ class Qwen3ReasoningParser(BaseThinkingReasoningParser):
31
use an older chat template where the model generates <think> itself.
32
This parser handles both styles: if <think> appears in the generated output
33
it is stripped before extraction (non-streaming) or skipped (streaming).
34
+
35
+ NOTE: Qwen3.5 models may emit <tool_call> inside the thinking block
36
+ without closing </think> first. <tool_call> is treated as an implicit
37
+ end of reasoning, matching the approach in KimiK2ReasoningParser.
38
"""
39
40
def __init__(self, tokenizer: "TokenizerLike", *args, **kwargs):
@@ -41,6 +45,11 @@ class Qwen3ReasoningParser(BaseThinkingReasoningParser):
45
# pure content when the user explicitly disables it.
46
self.thinking_enabled = chat_kwargs.get("enable_thinking", True)
47
48
+ self._tool_call_tag = "<tool_call>"
49
+ self._tool_call_token_id = self.vocab.get(self._tool_call_tag)
50
+ self._tool_call_end_tag = "</tool_call>"
51
+ self._tool_call_end_token_id = self.vocab.get(self._tool_call_end_tag)
52
+
53
@property
54
def start_token(self) -> str:
55
"""The token that starts reasoning content."""
@@ -51,6 +60,58 @@ class Qwen3ReasoningParser(BaseThinkingReasoningParser):
60
"""The token that ends reasoning content."""
61
return "</think>"
62
63
+ def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
64
+ start_token_id = self.start_token_id
65
+ end_token_id = self.end_token_id
66
+ tool_call_token_id = self._tool_call_token_id
67
+ tool_call_end_token_id = self._tool_call_end_token_id
68
+
69
+ for i in range(len(input_ids) - 1, -1, -1):
70
+ token_id = input_ids[i]
71
+ if token_id == start_token_id:
72
+ # Found <think> before </think> or <tool_call>
73
+ return False
74
+ if token_id == end_token_id:
75
+ return True
76
+ if tool_call_token_id is not None and token_id == tool_call_token_id:
77
+ # Only treat as implicit reasoning end if this <tool_call>
78
+ # is NOT followed by </tool_call>. Paired occurrences are
79
+ # template examples in the prompt, not model output.
80
+ if tool_call_end_token_id is not None and any(
81
+ input_ids[j] == tool_call_end_token_id
82
+ for j in range(i + 1, len(input_ids))
83
+ ):
84
+ continue
85
+ return True
86
+ return False
87
+
88
+ def is_reasoning_end_streaming(
89
+ self, input_ids: Sequence[int], delta_ids: Iterable[int]
90
+ ) -> bool:
91
+ if super().is_reasoning_end_streaming(input_ids, delta_ids):
92
+ return True
93
+ if self._tool_call_token_id is not None:
94
+ return self._tool_call_token_id in delta_ids
95
+ return False
96
+
97
+ def extract_content_ids(self, input_ids: list[int]) -> list[int]:
98
+ """
99
+ Extract content token ids from the input_ids.
100
+ """
101
+ result = super().extract_content_ids(input_ids)
102
+ if result:
103
+ return result
104
+ # Fall back: content starts at <tool_call> (implicit reasoning end).
105
+ if (
106
+ self._tool_call_token_id is not None
107
+ and self._tool_call_token_id in input_ids
108
+ ):
109
+ tool_call_index = (
110
+ len(input_ids) - 1 - input_ids[::-1].index(self._tool_call_token_id)
111
+ )
112
+ return input_ids[tool_call_index:]
113
+ return []
114
+
115
def extract_reasoning(
116
self, model_output: str, request: "ChatCompletionRequest | ResponsesRequest"
117
) -> tuple[str | None, str | None]:
@@ -78,19 +139,23 @@ class Qwen3ReasoningParser(BaseThinkingReasoningParser):
139
model_output_parts[2] if model_output_parts[1] else model_output_parts[0]
140
)
141
81
- if self.end_token not in model_output:
82
- if not self.thinking_enabled:
83
- # Thinking explicitly disabled — treat everything as content.
84
- return None, model_output
85
- # Thinking enabled but no </think>: output was truncated.
86
- # Everything generated so far is reasoning.
87
- return model_output, None
142
+ if self.end_token in model_output:
143
+ reasoning, _, content = model_output.partition(self.end_token)
144
+ return reasoning, content or None
145
89
- # Extract reasoning content from the model output.
90
- reasoning, _, content = model_output.partition(self.end_token)
146
+ if not self.thinking_enabled:
147
+ # Thinking explicitly disabled — treat everything as content.
148
+ return None, model_output
149
92
- final_content = content or None
93
- return reasoning, final_content
150
+ # No </think> — check for implicit reasoning end via <tool_call>.
151
+ tool_call_index = model_output.find(self._tool_call_tag)
152
+ if tool_call_index != -1:
153
+ reasoning = model_output[:tool_call_index]
154
+ content = model_output[tool_call_index:]
155
+ return reasoning or None, content or None
156
+ # Thinking enabled but no </think>: output was truncated.
157
+ # Everything generated so far is reasoning.
158
+ return model_output, None
159
160
def extract_reasoning_streaming(
161
self,
@@ -135,6 +200,20 @@ class Qwen3ReasoningParser(BaseThinkingReasoningParser):
200
# end_token_id in IDs but not in text (already stripped)
201
return None
202
203
+ # Implicit reasoning end via <tool_call>.
204
+ if (
205
+ self._tool_call_token_id is not None
206
+ and self._tool_call_token_id in delta_token_ids
207
+ ):
208
+ tool_index = delta_text.find(self._tool_call_tag)
209
+ if tool_index >= 0:
210
+ reasoning = delta_text[:tool_index]
211
+ content = delta_text[tool_index:]
212
+ return DeltaMessage(
213
+ reasoning=reasoning if reasoning else None,
214
+ content=content if content else None,
215
+ )
216
+
217
# No end token in this delta.
218
if not delta_text:
219
# Nothing left after stripping start token.
@@ -142,6 +221,11 @@ class Qwen3ReasoningParser(BaseThinkingReasoningParser):
221
elif self.end_token_id in previous_token_ids:
222
# End token already passed: everything is content now.
223
return DeltaMessage(content=delta_text)
224
+ elif (
225
+ self._tool_call_token_id is not None
226
+ and self._tool_call_token_id in previous_token_ids
227
+ ):
228
+ return DeltaMessage(content=delta_text)
229
else:
230
# No end token yet: still in reasoning phase.
231
return DeltaMessage(reasoning=delta_text)