A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[gpt-oss] Add IncompleteDetails to ResponsesRepsonse (#24561)
Signed-off-by: Andrew Xia <axia@meta.com>
Andrew Xia committed
Sep 15, 2025 at 13:07 UTC
25aba2b6a323e083004705c05077ef5327beaf39
7 files changed
+67
-25
tests/entrypoints/openai/test_response_api_with_harmony.py
+14
@@ -74,6 +74,20 @@ async def test_basic_with_reasoning_effort(client: OpenAI, model_name: str):
74
assert response.status == "completed"
75
76
77
+@pytest.mark.asyncio
78
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
79
+async def test_max_tokens(client: OpenAI, model_name: str):
80
+ response = await client.responses.create(
81
+ model=model_name,
82
+ input="What is the first paragraph of Moby Dick?",
83
+ reasoning={"effort": "low"},
84
+ max_output_tokens=30,
85
+ )
86
+ assert response is not None
87
+ assert response.status == "incomplete"
88
+ assert response.incomplete_details.reason == "max_output_tokens"
89
+
90
+
91
@pytest.mark.asyncio
92
@pytest.mark.parametrize("model_name", [MODEL_NAME])
93
async def test_chat(client: OpenAI, model_name: str):
vllm/entrypoints/context.py
+15
-10
@@ -112,6 +112,7 @@ class HarmonyContext(ConversationContext):
112
available_tools: list[str],
113
):
114
self._messages = messages
115
+ self.finish_reason: Optional[str] = None
116
self.available_tools = available_tools
117
self._tool_sessions: dict[str, Union[ClientSession, Tool]] = {}
118
self.called_tools: set[str] = set()
@@ -135,7 +136,8 @@ class HarmonyContext(ConversationContext):
136
if self.parser.current_channel in {"analysis", "commentary"}:
137
self.num_reasoning_tokens += 1
138
138
- def append_output(self, output) -> None:
139
+ def append_output(self, output: Union[RequestOutput,
140
+ list[Message]]) -> None:
141
if isinstance(output, RequestOutput):
142
output_token_ids = output.outputs[0].token_ids
143
self.parser = get_streamable_parser_for_assistant()
@@ -150,6 +152,8 @@ class HarmonyContext(ConversationContext):
152
# Move current turn to previous turn for next turn's calculations
153
self.previous_turn = self.current_turn.copy()
154
output_msgs = self.parser.messages
155
+ # The responses finish reason is set in the last message
156
+ self.finish_reason = output.outputs[0].finish_reason
157
else:
158
# Tool output.
159
output_msgs = output
@@ -157,18 +161,18 @@ class HarmonyContext(ConversationContext):
161
162
def _update_prefill_token_usage(self, output: RequestOutput) -> None:
163
"""Update token usage statistics for the prefill phase of generation.
160
-
164
+
165
The prefill phase processes the input prompt tokens. This method:
166
1. Counts the prompt tokens for this turn
167
2. Calculates tool output tokens for multi-turn conversations
168
3. Updates cached token counts
169
4. Tracks state for next turn calculations
166
-
170
+
171
Tool output tokens are calculated as:
168
- current_prompt_tokens - last_turn_prompt_tokens -
172
+ current_prompt_tokens - last_turn_prompt_tokens -
173
last_turn_output_tokens
174
This represents tokens added between turns (typically tool responses).
171
-
175
+
176
Args:
177
output: The RequestOutput containing prompt token information
178
"""
@@ -214,18 +218,18 @@ class HarmonyContext(ConversationContext):
218
219
def _update_decode_token_usage(self, output: RequestOutput) -> int:
220
"""Update token usage statistics for the decode phase of generation.
217
-
221
+
222
The decode phase processes the generated output tokens. This method:
223
1. Counts output tokens from all completion outputs
224
2. Updates the total output token count
225
3. Tracks tokens generated in the current turn
222
-
226
+
227
In streaming mode, this is called for each token generated.
228
In non-streaming mode, this is called once with all output tokens.
225
-
229
+
230
Args:
231
output: The RequestOutput containing generated token information
228
-
232
+
233
Returns:
234
int: Number of output tokens processed in this call
235
"""
@@ -385,7 +389,8 @@ class StreamingHarmonyContext(HarmonyContext):
389
def messages(self) -> list:
390
return self.parser.messages
391
388
- def append_output(self, output) -> None:
392
+ def append_output(self, output: Union[RequestOutput,
393
+ list[Message]]) -> None:
394
if isinstance(output, RequestOutput):
395
# append_output is called for each output token in streaming case,
396
# so we only want to add the prompt tokens once for each message.
vllm/entrypoints/harmony_utils.py
+3
-1
@@ -387,7 +387,9 @@ def parse_remaining_state(
387
id=f"msg_{random_uuid()}",
388
content=[output_text],
389
role="assistant",
390
- status="completed",
390
+ # if the parser still has messages (ie if the generator got cut
391
+ # abruptly), this should be incomplete
392
+ status="incomplete",
393
type="message",
394
)
395
return [text_item]
vllm/entrypoints/openai/protocol.py
+13
-4
@@ -30,7 +30,7 @@ except ImportError: # For newer openai versions (>= 1.100.0)
30
from openai.types.responses import (ResponseFormatTextConfig as
31
ResponseTextConfig)
32
33
-from openai.types.responses.response import ToolChoice
33
+from openai.types.responses.response import IncompleteDetails, ToolChoice
34
from openai.types.responses.tool import Tool
35
from openai.types.shared import Metadata, Reasoning
36
from pydantic import (BaseModel, ConfigDict, Field, TypeAdapter,
@@ -1868,7 +1868,7 @@ class ResponsesResponse(OpenAIBaseModel):
1868
id: str = Field(default_factory=lambda: f"resp_{random_uuid()}")
1869
created_at: int = Field(default_factory=lambda: int(time.time()))
1870
# error: Optional[ResponseError] = None
1871
- # incomplete_details: Optional[IncompleteDetails] = None
1871
+ incomplete_details: Optional[IncompleteDetails] = None
1872
instructions: Optional[str] = None
1873
metadata: Optional[Metadata] = None
1874
model: str
@@ -1904,9 +1904,18 @@ class ResponsesResponse(OpenAIBaseModel):
1904
status: ResponseStatus,
1905
usage: Optional[ResponseUsage] = None,
1906
) -> "ResponsesResponse":
1907
+
1908
+ incomplete_details: Optional[IncompleteDetails] = None
1909
+ if status == 'incomplete':
1910
+ incomplete_details = IncompleteDetails(reason='max_output_tokens')
1911
+ # TODO: implement the other reason for incomplete_details,
1912
+ # which is content_filter
1913
+ # incomplete_details = IncompleteDetails(reason='content_filter')
1914
+
1915
return cls(
1916
id=request.request_id,
1917
created_at=created_time,
1918
+ incomplete_details=incomplete_details,
1919
instructions=request.instructions,
1920
metadata=request.metadata,
1921
model=model_name,
@@ -2109,7 +2118,7 @@ class DetokenizeResponse(OpenAIBaseModel):
2118
2119
class TokenizerInfoResponse(OpenAIBaseModel):
2120
"""
2112
- Response containing tokenizer configuration
2121
+ Response containing tokenizer configuration
2122
equivalent to tokenizer_config.json
2123
"""
2124
@@ -2199,7 +2208,7 @@ class TranscriptionRequest(OpenAIBaseModel):
2208
to_language: Optional[str] = None
2209
"""The language of the output audio we transcribe to.
2210
2202
- Please note that this is not currently used by supported models at this
2211
+ Please note that this is not currently used by supported models at this
2212
time, but it is a placeholder for future use, matching translation api.
2213
"""
2214
vllm/entrypoints/openai/serving_responses.py
+15
-3
@@ -27,7 +27,7 @@ from openai.types.responses import (ResponseCreatedEvent,
27
ResponseReasoningItem,
28
ResponseReasoningTextDeltaEvent,
29
ResponseReasoningTextDoneEvent,
30
- response_text_delta_event)
30
+ ResponseStatus, response_text_delta_event)
31
from openai.types.responses.response_output_text import (Logprob,
32
LogprobTopLogprob)
33
# yapf: enable
@@ -461,10 +461,22 @@ class OpenAIServingResponses(OpenAIServing):
461
# TODO: Use a vllm-specific Validation Error
462
return self.create_error_response(str(e))
463
464
+ # NOTE: Implementation of stauts is still WIP, but for now
465
+ # we guarantee that if the status is not "completed", it is accurate.
466
+ # "completed" is implemented as the "catch-all" for now.
467
+ status: ResponseStatus = "completed"
468
+
469
if self.use_harmony:
470
assert isinstance(context, HarmonyContext)
471
output = self._make_response_output_items_with_harmony(context)
472
num_tool_output_tokens = context.num_tool_output_tokens
473
+ if len(output) > 0:
474
+ if context.finish_reason == "length":
475
+ status = "incomplete"
476
+ elif context.finish_reason == "abort":
477
+ status = "cancelled"
478
+ else:
479
+ status = "incomplete"
480
else:
481
assert isinstance(context, SimpleContext)
482
final_res = context.last_output
@@ -501,7 +513,7 @@ class OpenAIServingResponses(OpenAIServing):
513
model_name=model_name,
514
created_time=created_time,
515
output=output,
504
- status="completed",
516
+ status=status,
517
usage=usage,
518
)
519
@@ -658,7 +670,7 @@ class OpenAIServingResponses(OpenAIServing):
670
self,
671
context: HarmonyContext,
672
) -> list[ResponseOutputItem]:
661
- output_items = []
673
+ output_items: list[ResponseOutputItem] = []
674
num_init_messages = context.num_init_messages
675
for msg in context.messages[num_init_messages:]:
676
output_items.extend(parse_output_message(msg))
vllm/v1/core/sched/utils.py
+4
-4
@@ -10,19 +10,19 @@ from vllm.v1.request import Request, RequestStatus
10
11
def remove_all(lst: list, items_to_remove: set) -> list:
12
"""Remove all items from a list that are in the items_to_remove set.
13
-
13
+
14
This method optimizes for the common case of removing a single item,
15
falling back to list comprehension for multiple items.
16
-
16
+
17
Args:
18
lst: The list to remove items from
19
items_to_remove: Set of items to remove
20
-
20
+
21
Returns:
22
Either the modified original list (for single item removal) or
23
a new list (for multiple item removal). Callers should use the
24
returned value.
25
-
25
+
26
Note:
27
For single item removal, this modifies the original list in-place
28
and returns it. For multiple items, it creates and returns a new list.
vllm/v1/engine/output_processor.py
+3
-3
@@ -373,17 +373,17 @@ class OutputProcessor:
373
1) Compute stats for logging
374
2) Detokenize
375
3) Create and handle RequestOutput objects:
376
- * If there is a queue (for usage with AsyncLLM),
376
+ * If there is a queue (for usage with AsyncLLM),
377
put the RequestOutput objects into the queue for
378
handling by the per-request generate() tasks.
379
380
- * If there is no queue (for usage with LLMEngine),
380
+ * If there is no queue (for usage with LLMEngine),
381
return a list of RequestOutput objects.
382
383
NOTE FOR DEVELOPERS
384
385
vLLM V1 minimizes the number of python loops over the full
386
- batch to ensure system overheads are minimized. This is the
386
+ batch to ensure system overheads are minimized. This is the
387
only function that should loop over EngineCoreOutputs.
388
389
If you need to touch every element of the batch, do it from