A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Frontend] Responses API messages out, just harmony for now (#24985)
Signed-off-by: Alec Solder <alecs@fb.com> Co-authored-by: Alec Solder <alecs@fb.com> Co-authored-by: Ye (Charlotte) Qi <yeq@meta.com>
Alec S committed
Sep 19, 2025 at 17:40 UTC
e69e0b8b5fc5ef2958f7b1fc159119e9c4c0e2d2
3 files changed
+44
-1
tests/entrypoints/openai/test_response_api_with_harmony.py
+15
@@ -744,3 +744,18 @@ async def test_function_calling_full_history(client: OpenAI, model_name: str):
744
assert response_2 is not None
745
assert response_2.status == "completed"
746
assert response_2.output_text is not None
747
+
748
+
749
+@pytest.mark.asyncio
750
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
751
+async def test_output_messages_enabled(client: OpenAI, model_name: str,
752
+ server):
753
+ response = await client.responses.create(
754
+ model=model_name,
755
+ input="What is the capital of South Korea?",
756
+ extra_body={"enable_response_messages": True})
757
+
758
+ assert response is not None
759
+ assert response.status == "completed"
760
+ assert len(response.input_messages) > 0
761
+ assert len(response.output_messages) > 0
vllm/entrypoints/openai/protocol.py
+16
-1
@@ -328,6 +328,13 @@ class ResponsesRequest(OpenAIBaseModel):
328
"access by 3rd parties, and long enough to be "
329
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
330
"to 256 bit). Not supported by vLLM engine V0."))
331
+
332
+ enable_response_messages: bool = Field(
333
+ default=False,
334
+ description=(
335
+ "Dictates whether or not to return messages as part of the "
336
+ "response object. Currently only supported for non-streaming "
337
+ "non-background and gpt-oss only. "))
338
# --8<-- [end:responses-extra-params]
339
340
_DEFAULT_SAMPLING_PARAMS = {
@@ -1831,6 +1838,11 @@ class ResponsesResponse(OpenAIBaseModel):
1838
model: str
1839
object: Literal["response"] = "response"
1840
output: list[ResponseOutputItem]
1841
+ # These are populated when enable_response_messages is set to True
1842
+ # TODO: Currently an issue where content of harmony messages
1843
+ # is not available when these are serialized. Metadata is available
1844
+ input_messages: Optional[list[ChatCompletionMessageParam]] = None
1845
+ output_messages: Optional[list[ChatCompletionMessageParam]] = None
1846
parallel_tool_calls: bool
1847
temperature: float
1848
tool_choice: ToolChoice
@@ -1860,6 +1872,8 @@ class ResponsesResponse(OpenAIBaseModel):
1872
output: list[ResponseOutputItem],
1873
status: ResponseStatus,
1874
usage: Optional[ResponseUsage] = None,
1875
+ input_messages: Optional[list[ChatCompletionMessageParam]] = None,
1876
+ output_messages: Optional[list[ChatCompletionMessageParam]] = None,
1877
) -> "ResponsesResponse":
1878
1879
incomplete_details: Optional[IncompleteDetails] = None
@@ -1868,7 +1882,6 @@ class ResponsesResponse(OpenAIBaseModel):
1882
# TODO: implement the other reason for incomplete_details,
1883
# which is content_filter
1884
# incomplete_details = IncompleteDetails(reason='content_filter')
1871
-
1885
return cls(
1886
id=request.request_id,
1887
created_at=created_time,
@@ -1877,6 +1890,8 @@ class ResponsesResponse(OpenAIBaseModel):
1890
metadata=request.metadata,
1891
model=model_name,
1892
output=output,
1893
+ input_messages=input_messages,
1894
+ output_messages=output_messages,
1895
parallel_tool_calls=request.parallel_tool_calls,
1896
temperature=sampling_params.temperature,
1897
tool_choice=request.tool_choice,
vllm/entrypoints/openai/serving_responses.py
+13
@@ -475,9 +475,14 @@ class OpenAIServingResponses(OpenAIServing):
475
# "completed" is implemented as the "catch-all" for now.
476
status: ResponseStatus = "completed"
477
478
+ input_messages = None
479
+ output_messages = None
480
if self.use_harmony:
481
assert isinstance(context, HarmonyContext)
482
output = self._make_response_output_items_with_harmony(context)
483
+ if request.enable_response_messages:
484
+ input_messages = context.messages[:context.num_init_messages]
485
+ output_messages = context.messages[context.num_init_messages:]
486
num_tool_output_tokens = context.num_tool_output_tokens
487
if len(output) > 0:
488
if context.finish_reason == "length":
@@ -496,6 +501,12 @@ class OpenAIServingResponses(OpenAIServing):
501
output = self._make_response_output_items(request, final_output,
502
tokenizer)
503
504
+ # TODO: context for non-gptoss models doesn't use messages
505
+ # so we can't get them out yet
506
+ if request.enable_response_messages:
507
+ raise NotImplementedError(
508
+ "enable_response_messages is currently"
509
+ " only supported for gpt-oss")
510
# Calculate usage.
511
assert final_res.prompt_token_ids is not None
512
num_tool_output_tokens = 0
@@ -519,6 +530,8 @@ class OpenAIServingResponses(OpenAIServing):
530
response = ResponsesResponse.from_request(
531
request,
532
sampling_params,
533
+ input_messages=input_messages,
534
+ output_messages=output_messages,
535
model_name=model_name,
536
created_time=created_time,
537
output=output,