A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 03:30 UTC
[responsesAPI] support input output messages for non harmony models (#29549)
Signed-off-by: Andrew Xia <axia@fb.com> Co-authored-by: Andrew Xia <axia@fb.com>
Andrew Xia committed
Dec 2, 2025 at 23:59 UTC
3a7751485b71ce5ef927e4aa03b28602cb90811c
4 files changed
+64
-11
tests/entrypoints/openai/test_response_api_simple.py
+18
@@ -42,6 +42,24 @@ async def test_basic(client: OpenAI, model_name: str):
42
assert response.status == "completed"
43
44
45
+@pytest.mark.asyncio
46
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
47
+async def test_enable_response_messages(client: OpenAI, model_name: str):
48
+ response = await client.responses.create(
49
+ model=model_name,
50
+ input="Hello?",
51
+ extra_body={"enable_response_messages": True},
52
+ )
53
+ assert response.status == "completed"
54
+ assert response.input_messages[0]["type"] == "raw_message_tokens"
55
+ assert type(response.input_messages[0]["message"]) is str
56
+ assert len(response.input_messages[0]["message"]) > 10
57
+ assert type(response.input_messages[0]["tokens"][0]) is int
58
+ assert type(response.output_messages[0]["message"]) is str
59
+ assert len(response.output_messages[0]["message"]) > 10
60
+ assert type(response.output_messages[0]["tokens"][0]) is int
61
+
62
+
63
@pytest.mark.asyncio
64
@pytest.mark.parametrize("model_name", [MODEL_NAME])
65
async def test_reasoning_item(client: OpenAI, model_name: str):
vllm/entrypoints/context.py
+22
@@ -23,6 +23,7 @@ from vllm.entrypoints.openai.parser.responses_parser import (
23
)
24
from vllm.entrypoints.openai.protocol import (
25
ResponseInputOutputItem,
26
+ ResponseRawMessageAndToken,
27
ResponsesRequest,
28
)
29
from vllm.entrypoints.responses_utils import construct_tool_dicts
@@ -148,6 +149,8 @@ def _create_json_parse_error_messages(
149
150
151
class SimpleContext(ConversationContext):
152
+ """This is a context that cannot handle MCP tool calls"""
153
+
154
def __init__(self):
155
self.last_output = None
156
self.num_prompt_tokens = 0
@@ -158,6 +161,9 @@ class SimpleContext(ConversationContext):
161
# not implemented yet for SimpleContext
162
self.all_turn_metrics = []
163
164
+ self.input_messages: list[ResponseRawMessageAndToken] = []
165
+ self.output_messages: list[ResponseRawMessageAndToken] = []
166
+
167
def append_output(self, output) -> None:
168
self.last_output = output
169
if not isinstance(output, RequestOutput):
@@ -166,6 +172,22 @@ class SimpleContext(ConversationContext):
172
self.num_cached_tokens = output.num_cached_tokens or 0
173
self.num_output_tokens += len(output.outputs[0].token_ids or [])
174
175
+ if len(self.input_messages) == 0:
176
+ output_prompt = output.prompt or ""
177
+ output_prompt_token_ids = output.prompt_token_ids or []
178
+ self.input_messages.append(
179
+ ResponseRawMessageAndToken(
180
+ message=output_prompt,
181
+ tokens=output_prompt_token_ids,
182
+ )
183
+ )
184
+ self.output_messages.append(
185
+ ResponseRawMessageAndToken(
186
+ message=output.outputs[0].text,
187
+ tokens=output.outputs[0].token_ids,
188
+ )
189
+ )
190
+
191
def append_tool_output(self, output) -> None:
192
raise NotImplementedError("Should not be called.")
193
vllm/entrypoints/openai/protocol.py
+18
-4
@@ -1598,6 +1598,20 @@ def serialize_messages(msgs):
1598
return [serialize_message(msg) for msg in msgs] if msgs else None
1599
1600
1601
+class ResponseRawMessageAndToken(OpenAIBaseModel):
1602
+ """Class to show the raw message.
1603
+ If message / tokens diverge, tokens is the source of truth"""
1604
+
1605
+ message: str
1606
+ tokens: list[int]
1607
+ type: Literal["raw_message_tokens"] = "raw_message_tokens"
1608
+
1609
+
1610
+ResponseInputOutputMessage: TypeAlias = (
1611
+ list[ChatCompletionMessageParam] | list[ResponseRawMessageAndToken]
1612
+)
1613
+
1614
+
1615
class ResponsesResponse(OpenAIBaseModel):
1616
id: str = Field(default_factory=lambda: f"resp_{random_uuid()}")
1617
created_at: int = Field(default_factory=lambda: int(time.time()))
@@ -1631,8 +1645,8 @@ class ResponsesResponse(OpenAIBaseModel):
1645
# These are populated when enable_response_messages is set to True
1646
# NOTE: custom serialization is needed
1647
# see serialize_input_messages and serialize_output_messages
1634
- input_messages: list[ChatCompletionMessageParam] | None = None
1635
- output_messages: list[ChatCompletionMessageParam] | None = None
1648
+ input_messages: ResponseInputOutputMessage | None = None
1649
+ output_messages: ResponseInputOutputMessage | None = None
1650
# --8<-- [end:responses-extra-params]
1651
1652
# NOTE: openAI harmony doesn't serialize TextContent properly,
@@ -1658,8 +1672,8 @@ class ResponsesResponse(OpenAIBaseModel):
1672
output: list[ResponseOutputItem],
1673
status: ResponseStatus,
1674
usage: ResponseUsage | None = None,
1661
- input_messages: list[ChatCompletionMessageParam] | None = None,
1662
- output_messages: list[ChatCompletionMessageParam] | None = None,
1675
+ input_messages: ResponseInputOutputMessage | None = None,
1676
+ output_messages: ResponseInputOutputMessage | None = None,
1677
) -> "ResponsesResponse":
1678
incomplete_details: IncompleteDetails | None = None
1679
if status == "incomplete":
vllm/entrypoints/openai/serving_responses.py
+6
-7
@@ -86,6 +86,7 @@ from vllm.entrypoints.openai.protocol import (
86
ResponseCompletedEvent,
87
ResponseCreatedEvent,
88
ResponseInProgressEvent,
89
+ ResponseInputOutputMessage,
90
ResponseReasoningPartAddedEvent,
91
ResponseReasoningPartDoneEvent,
92
ResponsesRequest,
@@ -629,8 +630,8 @@ class OpenAIServingResponses(OpenAIServing):
630
# "completed" is implemented as the "catch-all" for now.
631
status: ResponseStatus = "completed"
632
632
- input_messages = None
633
- output_messages = None
633
+ input_messages: ResponseInputOutputMessage | None = None
634
+ output_messages: ResponseInputOutputMessage | None = None
635
if self.use_harmony:
636
assert isinstance(context, HarmonyContext)
637
output = self._make_response_output_items_with_harmony(context)
@@ -670,12 +671,10 @@ class OpenAIServingResponses(OpenAIServing):
671
672
output = self._make_response_output_items(request, final_output, tokenizer)
673
673
- # TODO: context for non-gptoss models doesn't use messages
674
- # so we can't get them out yet
674
if request.enable_response_messages:
676
- raise NotImplementedError(
677
- "enable_response_messages is currently only supported for gpt-oss"
678
- )
675
+ input_messages = context.input_messages
676
+ output_messages = context.output_messages
677
+
678
# Calculate usage.
679
assert final_res.prompt_token_ids is not None
680
num_tool_output_tokens = 0