A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 22:00 UTC
[Bugfix] Fix reasoning dropped on streaming boundary deltas (#42691)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Flora Feng committed
May 23, 2026 at 19:18 UTC
b32fe416ea1a036cda7373fc701b12b75bf86ac9
2 files changed
+95
-7
tests/parser/test_streaming.py
+72
@@ -137,6 +137,78 @@ def test_parse_delta_both_parsers(tokenizer, request_obj):
137
assert json.loads(tool_args) == {"city": "Dallas"}
138
139
140
+def stream_chunks(parser, tokenizer, chunks, request_obj):
141
+ """Stream pre-split token-ID chunks through the parser."""
142
+ results: list[DeltaMessage | None] = []
143
+ prompt_token_ids: list[int] | None = []
144
+ for chunk in chunks:
145
+ delta_text = tokenizer.decode(chunk)
146
+ result = parser.parse_delta(
147
+ delta_text, chunk, request_obj, prompt_token_ids=prompt_token_ids
148
+ )
149
+ prompt_token_ids = None
150
+ results.append(result)
151
+ return results
152
+
153
+
154
+def _boundary_chunks(tokenizer, parser):
155
+ """Split MODEL_OUTPUT into 3 chunks that straddle the </think> boundary."""
156
+ token_ids = tokenizer.encode(MODEL_OUTPUT, add_special_tokens=False)
157
+ end_token_id = parser._reasoning_parser.end_token_id
158
+ end_idx = token_ids.index(end_token_id)
159
+ return [
160
+ token_ids[: end_idx - 1],
161
+ token_ids[end_idx - 1 : end_idx + 2],
162
+ token_ids[end_idx + 2 :],
163
+ ]
164
+
165
+
166
+def test_parse_delta_reasoning_not_dropped_on_boundary(tokenizer, request_obj):
167
+ """Regression: reasoning must not be lost when a multi-token delta
168
+ spans the reasoning/tool-call boundary."""
169
+ parser = make_parser(tokenizer, reasoning=True, tool=True)
170
+ chunks = _boundary_chunks(tokenizer, parser)
171
+ results = stream_chunks(parser, tokenizer, chunks, request_obj)
172
+ reasoning, content, tool_calls = collect_fields(results)
173
+
174
+ assert "think about this" in reasoning
175
+ assert content == ""
176
+ assert len(tool_calls) > 0
177
+ assert tool_calls[0].function.name == "get_weather"
178
+ tool_args = "".join(
179
+ tc.function.arguments for tc in tool_calls if tc.function.arguments
180
+ )
181
+ assert json.loads(tool_args) == {"city": "Dallas"}
182
+
183
+
184
+def test_parse_delta_reasoning_boundary_no_tool_parser(tokenizer, request_obj):
185
+ """When no tool parser is active, boundary-spanning chunks must still
186
+ preserve reasoning and pass post-</think> text as content."""
187
+ parser = make_parser(tokenizer, reasoning=True, tool=False)
188
+ chunks = _boundary_chunks(tokenizer, parser)
189
+ results = stream_chunks(parser, tokenizer, chunks, request_obj)
190
+ reasoning, content, tool_calls = collect_fields(results)
191
+
192
+ assert "think about this" in reasoning
193
+ assert len(tool_calls) == 0
194
+ assert "<tool_call>" in content
195
+ assert "get_weather" in content
196
+
197
+
198
+def test_parse_delta_reasoning_only_no_think_leak(tokenizer, request_obj):
199
+ """Regression: </think> must not leak into content when streaming
200
+ token-by-token with reasoning=True, tool=False."""
201
+ parser = make_parser(tokenizer, reasoning=True, tool=False)
202
+ results = stream_text(
203
+ parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
204
+ )
205
+ reasoning, content, tool_calls = collect_fields(results)
206
+
207
+ assert "let me think about this" in reasoning
208
+ assert "</think>" not in content
209
+ assert "<think>" not in content
210
+
211
+
212
def test_parse_delta_reasoning_only_thinking_disabled(tokenizer, request_obj):
213
"""Regression test for vllm-project/vllm#40466.
214
vllm/parser/abstract_parser.py
+23
-7
@@ -634,6 +634,13 @@ class DelegatingParser(Parser):
634
return False
635
return self._reasoning_parser.is_reasoning_end(input_ids)
636
637
+ def is_reasoning_end_streaming(
638
+ self, input_ids: list[int], delta_ids: list[int]
639
+ ) -> bool:
640
+ if self._reasoning_parser is None:
641
+ return False
642
+ return self._reasoning_parser.is_reasoning_end_streaming(input_ids, delta_ids)
643
+
644
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
645
if self._reasoning_parser is None:
646
return input_ids
@@ -679,15 +686,16 @@ class DelegatingParser(Parser):
686
current_token_ids=current_token_ids,
687
delta_token_ids=delta_token_ids,
688
)
682
- # Hand off remaining content to tool parser
683
- if self._tool_parser and self.is_reasoning_end(delta_token_ids):
689
+ if self.is_reasoning_end_streaming(current_token_ids, delta_token_ids):
690
state.reasoning_ended = True
691
current_token_ids = self.extract_content_ids(delta_token_ids)
686
- if delta_message and delta_message.content:
687
- current_text = delta_message.content
688
- delta_message.content = None
689
- else:
690
- current_text = ""
692
+ current_text = (
693
+ delta_message.content
694
+ if delta_message and delta_message.content
695
+ else ""
696
+ )
697
+ delta_text = current_text
698
+ delta_token_ids = current_token_ids
699
700
# Tool call extraction
701
if self._in_tool_call_phase(state):
@@ -698,6 +706,9 @@ class DelegatingParser(Parser):
706
delta_text = current_text
707
delta_token_ids = current_token_ids
708
709
+ # A boundary delta may carry both reasoning and tool call,
710
+ # save it before the tool parser overwrites delta_message.
711
+ reasoning = delta_message.reasoning if delta_message else None
712
delta_message, state.function_name_returned = (
713
self._extract_tool_calls_streaming(
714
previous_text=state.previous_text,
@@ -712,6 +723,11 @@ class DelegatingParser(Parser):
723
function_name_returned=state.function_name_returned,
724
)
725
)
726
+ if reasoning:
727
+ if not delta_message:
728
+ delta_message = DeltaMessage()
729
+ delta_message.reasoning = reasoning
730
+
731
if (
732
delta_message
733
and delta_message.tool_calls