A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 10:15 UTC
[responsesAPI] fix incomplete_messages for simple/parsable context (#31836)
Signed-off-by: Andrew Xia <axia@fb.com> Co-authored-by: Andrew Xia <axia@fb.com>
Andrew Xia committed
Jan 9, 2026 at 16:00 UTC
1f8b7c536be40975573eeebf36204286cfb4e4e9
4 files changed
+44
tests/entrypoints/openai/test_response_api_parsable_context.py
+15
@@ -58,6 +58,7 @@ async def test_basic(client: OpenAI, model_name: str):
58
assert response is not None
59
print("response: ", response)
60
assert response.status == "completed"
61
+ assert response.incomplete_details is None
62
63
64
@pytest.mark.asyncio
@@ -184,3 +185,17 @@ async def test_mcp_tool_call(client: OpenAI, model_name: str):
185
assert len(response.input_messages) == 1
186
assert len(response.output_messages) == 3
187
assert "312" in response.output_messages[2]["message"]
188
+
189
+
190
+@pytest.mark.asyncio
191
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
192
+async def test_max_tokens(client: OpenAI, model_name: str):
193
+ response = await client.responses.create(
194
+ model=model_name,
195
+ input="What is the first paragraph of Moby Dick?",
196
+ reasoning={"effort": "low"},
197
+ max_output_tokens=30,
198
+ )
199
+ assert response is not None
200
+ assert response.status == "incomplete"
201
+ assert response.incomplete_details.reason == "max_output_tokens"
tests/entrypoints/openai/test_response_api_simple.py
+15
@@ -40,6 +40,7 @@ async def test_basic(client: OpenAI, model_name: str):
40
assert response is not None
41
print("response: ", response)
42
assert response.status == "completed"
43
+ assert response.incomplete_details is None
44
45
46
@pytest.mark.asyncio
@@ -132,3 +133,17 @@ async def test_streaming_output_consistency(client: OpenAI, model_name: str):
133
f"Streaming: {streaming_text!r}\n"
134
f"Final: {final_output_text!r}"
135
)
136
+
137
+
138
+@pytest.mark.asyncio
139
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
140
+async def test_max_tokens(client: OpenAI, model_name: str):
141
+ response = await client.responses.create(
142
+ model=model_name,
143
+ input="What is the first paragraph of Moby Dick?",
144
+ reasoning={"effort": "low"},
145
+ max_output_tokens=30,
146
+ )
147
+ assert response is not None
148
+ assert response.status == "incomplete"
149
+ assert response.incomplete_details.reason == "max_output_tokens"
vllm/entrypoints/openai/parser/responses_parser.py
+6
@@ -51,7 +51,13 @@ class ResponsesParser:
51
if tool_parser_cls is not None:
52
self.tool_parser_instance = tool_parser_cls(tokenizer)
53
54
+ # Store the last finish_reason to determine response status
55
+ self.finish_reason: str | None = None
56
+
57
def process(self, output: CompletionOutput) -> "ResponsesParser":
58
+ # Store the finish_reason from the output
59
+ self.finish_reason = output.finish_reason
60
+
61
reasoning_content, content = self.reasoning_parser_instance.extract_reasoning(
62
output.text, request=self.request
63
)
vllm/entrypoints/openai/serving_responses.py
+8
@@ -692,6 +692,10 @@ class OpenAIServingResponses(OpenAIServing):
692
# TODO: Calculate usage.
693
# assert final_res.prompt_token_ids is not None
694
num_tool_output_tokens = 0
695
+
696
+ # Check finish reason from the parser
697
+ if context.parser.finish_reason == "length":
698
+ status = "incomplete"
699
else:
700
assert isinstance(context, SimpleContext)
701
# Use final_output which has accumulated text/token_ids/logprobs
@@ -703,6 +707,10 @@ class OpenAIServingResponses(OpenAIServing):
707
# finish_reason='error' indicates retryable internal error
708
self._raise_if_error(final_output.finish_reason, request.request_id)
709
710
+ # Check if generation was stopped due to max_tokens
711
+ if final_output.finish_reason == "length":
712
+ status = "incomplete"
713
+
714
output = self._make_response_output_items(request, final_output, tokenizer)
715
716
if request.enable_response_messages: