[Bugfix][Parser] Scope reasoning-end detection to the current turn via turn-boundary tokens (#54089)
Signed-off-by: Mikhail Podvitskii <podvitskiymichael@gmail.com>
Mikhail Podvitskii committed
Aug 27, 2026 at 22:57 UTC
3a9bfc209f9443203d795149da40a7e5b94f27dc
5 files changed
+147
-4
tests/parser/engine/test_qwen3_reasoning.py
+120
@@ -27,6 +27,8 @@ _THINK_START_ID = 50
27
_THINK_END_ID = 51
28
_TOOL_CALL_ID = 60
29
_TOOL_CALL_END_ID = 61
30
+_IM_START_ID = 70
31
+_IM_END_ID = 71
32
_TEXT_ID = 100
33
34
_QWEN3_VOCAB = {
@@ -34,6 +36,8 @@ _QWEN3_VOCAB = {
36
"</think>": _THINK_END_ID,
37
"<tool_call>": _TOOL_CALL_ID,
38
"</tool_call>": _TOOL_CALL_END_ID,
39
+ "<|im_start|>": _IM_START_ID,
40
+ "<|im_end|>": _IM_END_ID,
41
}
42
43
@@ -192,6 +196,72 @@ class TestIsReasoningEnd:
196
assert not parser.is_reasoning_end([])
197
198
199
+class TestIsReasoningEndTurnBoundaries:
200
+ """Reasoning markers from earlier turns must not mark the new turn's
201
+ reasoning as finished (think-marker injection / interleaved-thinking
202
+ replay).
203
+ """
204
+
205
+ def test_injected_think_end_in_history_not_end(self, parser):
206
+ assert not parser.is_reasoning_end(
207
+ [_IM_START_ID, _TEXT_ID, _THINK_END_ID, _IM_END_ID, _IM_START_ID]
208
+ )
209
+
210
+ def test_replayed_reasoning_block_in_history_not_end(self, parser):
211
+ assert not parser.is_reasoning_end(
212
+ [
213
+ _IM_START_ID,
214
+ _THINK_START_ID,
215
+ _TEXT_ID,
216
+ _THINK_END_ID,
217
+ _TEXT_ID,
218
+ _IM_END_ID,
219
+ _IM_START_ID,
220
+ ]
221
+ )
222
+
223
+ def test_think_end_in_current_turn_is_end(self, parser):
224
+ assert parser.is_reasoning_end(
225
+ [_IM_START_ID, _TEXT_ID, _IM_END_ID, _IM_START_ID, _THINK_END_ID]
226
+ )
227
+
228
+ def test_unpaired_tool_call_in_history_not_end(self, parser):
229
+ assert not parser.is_reasoning_end(
230
+ [_TOOL_CALL_ID, _TEXT_ID, _IM_END_ID, _IM_START_ID]
231
+ )
232
+
233
+ def test_continued_turn_with_closed_thinking_is_end(self, parser):
234
+ assert parser.is_reasoning_end(
235
+ [_IM_START_ID, _THINK_START_ID, _TEXT_ID, _THINK_END_ID, _TEXT_ID]
236
+ )
237
+
238
+ def test_continued_turn_with_open_thinking_not_end(self, parser):
239
+ assert not parser.is_reasoning_end([_IM_START_ID, _THINK_START_ID, _TEXT_ID])
240
+
241
+ def test_empty_boundary_config_keeps_global_walk(self):
242
+ """Configs without turn_boundary_tokens keep the pre-existing
243
+ behavior: any </think> in the sequence ends reasoning."""
244
+ cfg = qwen3_config(turn_boundary_tokens=frozenset())
245
+ parser = Qwen3Parser(
246
+ make_mock_tokenizer(_QWEN3_VOCAB), parser_engine_config=cfg
247
+ )
248
+ assert parser.is_reasoning_end(
249
+ [_IM_START_ID, _TEXT_ID, _THINK_END_ID, _IM_END_ID, _IM_START_ID]
250
+ )
251
+
252
+ def test_boundary_tokens_missing_from_vocab_keep_global_walk(self):
253
+ """Boundary tokens absent from the vocabulary resolve to nothing,
254
+ so subclasses reusing the default on a non-ChatML vocab keep the
255
+ pre-existing behavior."""
256
+ vocab = {
257
+ token: token_id
258
+ for token, token_id in _QWEN3_VOCAB.items()
259
+ if not token.startswith("<|im_")
260
+ }
261
+ parser = Qwen3Parser(make_mock_tokenizer(vocab))
262
+ assert parser.is_reasoning_end([_TEXT_ID, _THINK_END_ID, _TEXT_ID])
263
+
264
+
265
class TestDelegatingPromptDetection:
266
def test_prompt_tool_example_does_not_skip_streaming_reasoning(
267
self, mock_tokenizer, mock_request
@@ -211,6 +281,56 @@ class TestDelegatingPromptDetection:
281
assert delta.reasoning == "thinking"
282
assert delta.content is None
283
284
+ def test_injected_think_end_in_history_keeps_streaming_reasoning(
285
+ self, mock_tokenizer, mock_request
286
+ ):
287
+ parser = _Qwen3DelegatingParser(mock_tokenizer)
288
+ prompt_ids = [
289
+ _IM_START_ID,
290
+ _TEXT_ID,
291
+ _THINK_END_ID,
292
+ _TEXT_ID,
293
+ _IM_END_ID,
294
+ _IM_START_ID,
295
+ ]
296
+
297
+ delta = parser.parse_delta(
298
+ "thinking",
299
+ [_TEXT_ID],
300
+ mock_request,
301
+ prompt_token_ids=prompt_ids,
302
+ finished=False,
303
+ )
304
+
305
+ assert delta is not None
306
+ assert delta.reasoning == "thinking"
307
+ assert delta.content is None
308
+
309
+ def test_think_end_in_generation_prompt_still_seeds_reasoning_ended(
310
+ self, mock_tokenizer, mock_request
311
+ ):
312
+ parser = _Qwen3DelegatingParser(mock_tokenizer)
313
+ prompt_ids = [
314
+ _IM_START_ID,
315
+ _TEXT_ID,
316
+ _IM_END_ID,
317
+ _IM_START_ID,
318
+ _THINK_START_ID,
319
+ _THINK_END_ID,
320
+ ]
321
+
322
+ delta = parser.parse_delta(
323
+ "answer",
324
+ [_TEXT_ID],
325
+ mock_request,
326
+ prompt_token_ids=prompt_ids,
327
+ finished=False,
328
+ )
329
+
330
+ assert delta is not None
331
+ assert delta.reasoning is None
332
+ assert delta.content == "answer"
333
+
334
335
class TestStreaming:
336
def test_basic_streaming(self, parser):
vllm/parser/engine/parser_engine.py
+13
-2
@@ -147,6 +147,11 @@ class ParserEngine(Parser):
147
self._reasoning_start_token_id = vocab.get(start_text)
148
if end_text:
149
self._reasoning_end_token_id = vocab.get(end_text)
150
+ self._turn_boundary_token_ids: frozenset[int] = frozenset(
151
+ token_id
152
+ for token in parser_engine_config.turn_boundary_tokens
153
+ if (token_id := vocab.get(token)) is not None
154
+ )
155
156
@property
157
def reasoning_start_str(self) -> str | None:
@@ -601,11 +606,17 @@ class ParserEngine(Parser):
606
if end_id is not None:
607
if not input_ids:
608
return self.parser_engine_config.initial_state != ParserState.REASONING
609
+ boundary_ids = self._turn_boundary_token_ids
610
for i in range(len(input_ids) - 1, -1, -1):
605
- if input_ids[i] == end_id:
611
+ token_id = input_ids[i]
612
+ if token_id == end_id:
613
return True
607
- if start_id is not None and input_ids[i] == start_id:
614
+ if start_id is not None and token_id == start_id:
615
return False
616
+ if token_id in boundary_ids:
617
+ return (
618
+ self.parser_engine_config.initial_state != ParserState.REASONING
619
+ )
620
return False
621
return self._reasoning_ended
622
vllm/parser/engine/parser_engine_config.py
+4
@@ -84,6 +84,10 @@ class ParserEngineConfig:
84
# Special tokens exempt from auto-drop but not state-machine terminals.
85
preserve_tokens: frozenset[str] = field(default_factory=frozenset)
86
87
+ # Special tokens delimiting conversation turns in the prompt only
88
+ # considers reasoning markers after the last boundary token.
89
+ turn_boundary_tokens: frozenset[str] = field(default_factory=frozenset)
90
+
91
# Prevents trailing-whitespace accumulation across multi-turn conversations.
92
strip_trailing_reasoning_whitespace: bool = True
93
vllm/parser/nemotron_v3.py
+2
-2
@@ -16,7 +16,7 @@ import dataclasses
16
import functools
17
from typing import TYPE_CHECKING
18
19
-from vllm.parser.qwen3 import Qwen3Parser, qwen3_config
19
+from vllm.parser.qwen3 import CHATML_TURN_BOUNDARIES, Qwen3Parser, qwen3_config
20
21
if TYPE_CHECKING:
22
from vllm.entrypoints.openai.chat_completion.protocol import (
@@ -33,7 +33,7 @@ if TYPE_CHECKING:
33
@functools.cache
34
def nemotron_v3_config(thinking: bool = True) -> ParserEngineConfig:
35
return dataclasses.replace(
36
- qwen3_config(thinking=thinking),
36
+ qwen3_config(thinking=thinking, turn_boundary_tokens=CHATML_TURN_BOUNDARIES),
37
name="nemotron_v3",
38
strip_trailing_reasoning_whitespace=True,
39
)
vllm/parser/qwen3.py
+8
@@ -42,6 +42,7 @@ THINK_START = "<think>"
42
THINK_END = "</think>"
43
TOOL_CALL_START = "<tool_call>"
44
TOOL_CALL_END = "</tool_call>"
45
+CHATML_TURN_BOUNDARIES = frozenset(("<|im_start|>", "<|im_end|>"))
46
FUNC_PREFIX = "<function="
47
FUNC_END = "</function>"
48
PARAM_START = "<parameter="
@@ -94,10 +95,12 @@ def qwen3_config(
95
think_end: str = THINK_END,
96
tool_start: str = TOOL_CALL_START,
97
tool_end: str = TOOL_CALL_END,
98
+ turn_boundary_tokens: frozenset[str] = frozenset(),
99
) -> ParserEngineConfig:
100
return ParserEngineConfig(
101
name=name,
102
initial_state=ParserState.REASONING if thinking else ParserState.CONTENT,
103
+ turn_boundary_tokens=turn_boundary_tokens,
104
terminals={
105
# Reasoning terminals
106
"THINK_START": think_start,
@@ -215,6 +218,7 @@ class Qwen3Parser(ParserEngine):
218
THINK_END = THINK_END
219
TOOL_START = TOOL_CALL_START
220
TOOL_END = TOOL_CALL_END
221
+ TURN_BOUNDARIES: frozenset[str] = CHATML_TURN_BOUNDARIES
222
223
def __init__(
224
self,
@@ -233,6 +237,7 @@ class Qwen3Parser(ParserEngine):
237
think_end=self.THINK_END,
238
tool_start=self.TOOL_START,
239
tool_end=self.TOOL_END,
240
+ turn_boundary_tokens=self.TURN_BOUNDARIES,
241
),
242
)
243
super().__init__(
@@ -259,6 +264,7 @@ class Qwen3Parser(ParserEngine):
264
tool_call_id = self._tool_call_token_id
265
tool_call_end_id = self._tool_call_end_token_id
266
reasoning_start_id = self._reasoning_start_token_id
267
+ boundary_ids = self._turn_boundary_token_ids
268
if tool_call_id is not None:
269
for i in range(len(input_ids) - 1, -1, -1):
270
if (
@@ -266,6 +272,8 @@ class Qwen3Parser(ParserEngine):
272
and input_ids[i] == reasoning_start_id
273
):
274
return False
275
+ if input_ids[i] in boundary_ids:
276
+ return False
277
if input_ids[i] == tool_call_id:
278
if tool_call_end_id is not None and any(
279
input_ids[j] == tool_call_end_id