A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 10:00 UTC
[Bugfix] fix streaming final output for non harmony (#30237)
Signed-off-by: penfree <qiupengfei@baidu.com> Co-authored-by: penfree <qiupengfei@baidu.com>
penfree committed
Dec 16, 2025 at 09:03 UTC
bbd850e597b92ed92ccc4d6698a0563a2e1fb74a
3 files changed
+79
-3
tests/entrypoints/openai/test_response_api_simple.py
+45
@@ -87,3 +87,48 @@ async def test_reasoning_item(client: OpenAI, model_name: str):
87
assert response.output[0].type == "reasoning"
88
assert response.output[1].type == "message"
89
assert type(response.output[1].content[0].text) is str
90
+
91
+
92
+@pytest.mark.asyncio
93
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
94
+async def test_streaming_output_consistency(client: OpenAI, model_name: str):
95
+ """Test that streaming delta text matches the final response output_text.
96
+
97
+ This test verifies that when using streaming mode:
98
+ 1. The concatenated text from all 'response.output_text.delta' events
99
+ 2. Matches the 'output_text' in the final 'response.completed' event
100
+ """
101
+ response = await client.responses.create(
102
+ model=model_name,
103
+ input="Say hello in one sentence.",
104
+ stream=True,
105
+ )
106
+
107
+ events = []
108
+ async for event in response:
109
+ events.append(event)
110
+
111
+ assert len(events) > 0
112
+
113
+ # Concatenate all delta text from streaming events
114
+ streaming_text = "".join(
115
+ event.delta for event in events if event.type == "response.output_text.delta"
116
+ )
117
+
118
+ # Get the final response from the last event
119
+ response_completed_event = events[-1]
120
+ assert response_completed_event.type == "response.completed"
121
+ assert response_completed_event.response.status == "completed"
122
+
123
+ # Get output_text from the final response
124
+ final_output_text = response_completed_event.response.output_text
125
+
126
+ # Verify final response has output
127
+ assert len(response_completed_event.response.output) > 0
128
+
129
+ # Verify streaming text matches final output_text
130
+ assert streaming_text == final_output_text, (
131
+ f"Streaming text does not match final output_text.\n"
132
+ f"Streaming: {streaming_text!r}\n"
133
+ f"Final: {final_output_text!r}"
134
+ )
vllm/entrypoints/context.py
+32
-2
@@ -2,11 +2,13 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import asyncio
4
import contextlib
5
+import copy
6
import json
7
import logging
8
from abc import ABC, abstractmethod
9
from collections.abc import Callable
10
from contextlib import AsyncExitStack
11
+from dataclasses import replace
12
from typing import TYPE_CHECKING, Union
13
14
from openai.types.responses.response_function_tool_call_output_item import (
@@ -164,6 +166,12 @@ class SimpleContext(ConversationContext):
166
167
def __init__(self):
168
self.last_output = None
169
+
170
+ # Accumulated final output for streaming mode
171
+ self._accumulated_text: str = ""
172
+ self._accumulated_token_ids: list[int] = []
173
+ self._accumulated_logprobs: list = []
174
+
175
self.num_prompt_tokens = 0
176
self.num_output_tokens = 0
177
self.num_cached_tokens = 0
@@ -183,6 +191,13 @@ class SimpleContext(ConversationContext):
191
self.num_cached_tokens = output.num_cached_tokens or 0
192
self.num_output_tokens += len(output.outputs[0].token_ids or [])
193
194
+ # Accumulate text, token_ids, and logprobs for streaming mode
195
+ delta_output = output.outputs[0]
196
+ self._accumulated_text += delta_output.text
197
+ self._accumulated_token_ids.extend(delta_output.token_ids)
198
+ if delta_output.logprobs is not None:
199
+ self._accumulated_logprobs.extend(delta_output.logprobs)
200
+
201
if len(self.input_messages) == 0:
202
output_prompt = output.prompt or ""
203
output_prompt_token_ids = output.prompt_token_ids or []
@@ -194,11 +209,26 @@ class SimpleContext(ConversationContext):
209
)
210
self.output_messages.append(
211
ResponseRawMessageAndToken(
197
- message=output.outputs[0].text,
198
- tokens=output.outputs[0].token_ids,
212
+ message=delta_output.text,
213
+ tokens=delta_output.token_ids,
214
)
215
)
216
217
+ @property
218
+ def final_output(self) -> RequestOutput | None:
219
+ """Return the final output, with complete text/token_ids/logprobs."""
220
+ if self.last_output is not None and self.last_output.outputs:
221
+ assert isinstance(self.last_output, RequestOutput)
222
+ final_output = copy.copy(self.last_output)
223
+ # copy inner item to avoid modify last_output
224
+ final_output.outputs = [replace(item) for item in self.last_output.outputs]
225
+ final_output.outputs[0].text = self._accumulated_text
226
+ final_output.outputs[0].token_ids = tuple(self._accumulated_token_ids)
227
+ if self._accumulated_logprobs:
228
+ final_output.outputs[0].logprobs = self._accumulated_logprobs
229
+ return final_output
230
+ return self.last_output
231
+
232
def append_tool_output(self, output) -> None:
233
raise NotImplementedError("Should not be called.")
234
vllm/entrypoints/openai/serving_responses.py
+2
-1
@@ -675,7 +675,8 @@ class OpenAIServingResponses(OpenAIServing):
675
num_tool_output_tokens = 0
676
else:
677
assert isinstance(context, SimpleContext)
678
- final_res = context.last_output
678
+ # Use final_output which has accumulated text/token_ids/logprobs
679
+ final_res = context.final_output
680
assert final_res is not None
681
assert len(final_res.outputs) == 1
682
final_output = final_res.outputs[0]