A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 02:15 UTC
[Bugfix] Fix unstreamed tool call args dropped in Responses API streaming (#44348)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Flora Feng committed
Jun 3, 2026 at 06:19 UTC
209709a8c131f15ed5c1ea231f4365f61a05d009
3 files changed
+15
-4
tests/parser/test_streaming.py
+10
-2
@@ -58,7 +58,11 @@ def stream_text(parser, tokenizer, text, request, prompt_token_ids=None):
58
for tid in token_ids:
59
delta_text = tokenizer.decode([tid])
60
result = parser.parse_delta(
61
- delta_text, [tid], request, prompt_token_ids=prompt_token_ids
61
+ delta_text,
62
+ [tid],
63
+ request,
64
+ prompt_token_ids=prompt_token_ids,
65
+ finished=False,
66
)
67
prompt_token_ids = None
68
results.append(result)
@@ -146,7 +150,11 @@ def stream_chunks(parser, tokenizer, chunks, request_obj):
150
for chunk in chunks:
151
delta_text = tokenizer.decode(chunk)
152
result = parser.parse_delta(
149
- delta_text, chunk, request_obj, prompt_token_ids=prompt_token_ids
153
+ delta_text,
154
+ chunk,
155
+ request_obj,
156
+ prompt_token_ids=prompt_token_ids,
157
+ finished=False,
158
)
159
prompt_token_ids = None
160
results.append(result)
vllm/entrypoints/openai/responses/serving.py
+1
@@ -1408,6 +1408,7 @@ class OpenAIServingResponses(OpenAIServing):
1408
delta_token_ids=delta_token_ids,
1409
request=request,
1410
prompt_token_ids=ctx.last_output.prompt_token_ids,
1411
+ finished=output.finish_reason is not None,
1412
)
1413
else:
1414
delta_message = DeltaMessage(content=output.text)
vllm/parser/abstract_parser.py
+4
-2
@@ -344,7 +344,8 @@ class Parser:
344
delta_token_ids: list[int],
345
request: ChatCompletionRequest | ResponsesRequest,
346
prompt_token_ids: list[int] | None = None,
347
- finished: bool = False,
347
+ *,
348
+ finished: bool,
349
) -> DeltaMessage | None:
350
"""Parse a single streaming delta, orchestrating reasoning then
351
tool call extraction via internal stream state.
@@ -809,7 +810,8 @@ class DelegatingParser(Parser):
810
delta_token_ids: list[int],
811
request: ChatCompletionRequest | ResponsesRequest,
812
prompt_token_ids: list[int] | None = None,
812
- finished: bool = False,
813
+ *,
814
+ finished: bool,
815
) -> DeltaMessage | None:
816
state = self._stream_state
817