A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[responsesAPI][bugfix] serialize harmony messages (#26185)
Signed-off-by: Andrew Xia <axia@meta.com> Co-authored-by: Ye (Charlotte) Qi <yeq@meta.com>
Andrew Xia committed
Oct 7, 2025 at 00:07 UTC
185d8ed44f382fc2b066e4d2eabd3c1b5fcbd8db
3 files changed
+61
-6
tests/entrypoints/openai/test_response_api_with_harmony.py
+16
@@ -8,6 +8,9 @@ import pytest
8
import pytest_asyncio
9
import requests
10
from openai import BadRequestError, NotFoundError, OpenAI
11
+from openai_harmony import (
12
+ Message,
13
+)
14
15
from ...utils import RemoteOpenAIServer
16
@@ -326,6 +329,7 @@ async def test_streaming(client: OpenAI, model_name: str, background: bool):
329
],
330
stream=True,
331
background=background,
332
+ extra_body={"enable_response_messages": True},
333
)
334
335
current_item_id = ""
@@ -334,6 +338,7 @@ async def test_streaming(client: OpenAI, model_name: str, background: bool):
338
events = []
339
current_event_mode = None
340
resp_id = None
341
+ checked_response_completed = False
342
async for event in response:
343
if event.type == "response.created":
344
resp_id = event.response.id
@@ -346,6 +351,16 @@ async def test_streaming(client: OpenAI, model_name: str, background: bool):
351
]:
352
assert "input_messages" in event.response.model_extra
353
assert "output_messages" in event.response.model_extra
354
+ if event.type == "response.completed":
355
+ # make sure the serialization of content works
356
+ for msg in event.response.model_extra["output_messages"]:
357
+ # make sure we can convert the messages back into harmony
358
+ Message.from_dict(msg)
359
+
360
+ for msg in event.response.model_extra["input_messages"]:
361
+ # make sure we can convert the messages back into harmony
362
+ Message.from_dict(msg)
363
+ checked_response_completed = True
364
365
if current_event_mode != event.type:
366
current_event_mode = event.type
@@ -390,6 +405,7 @@ async def test_streaming(client: OpenAI, model_name: str, background: bool):
405
assert len(events) > 0
406
response_completed_event = events[-1]
407
assert len(response_completed_event.response.output) > 0
408
+ assert checked_response_completed
409
410
if background:
411
starting_after = 5
vllm/entrypoints/openai/protocol.py
+44
-5
@@ -63,6 +63,7 @@ from pydantic import (
63
Field,
64
TypeAdapter,
65
ValidationInfo,
66
+ field_serializer,
67
field_validator,
68
model_validator,
69
)
@@ -2078,11 +2079,6 @@ class ResponsesResponse(OpenAIBaseModel):
2079
model: str
2080
object: Literal["response"] = "response"
2081
output: list[ResponseOutputItem]
2081
- # These are populated when enable_response_messages is set to True
2082
- # TODO: Currently an issue where content of harmony messages
2083
- # is not available when these are serialized. Metadata is available
2084
- input_messages: Optional[list[ChatCompletionMessageParam]] = None
2085
- output_messages: Optional[list[ChatCompletionMessageParam]] = None
2082
parallel_tool_calls: bool
2083
temperature: float
2084
tool_choice: ToolChoice
@@ -2102,6 +2098,49 @@ class ResponsesResponse(OpenAIBaseModel):
2098
usage: Optional[ResponseUsage] = None
2099
user: Optional[str] = None
2100
2101
+ # --8<-- [start:responses-extra-params]
2102
+ # These are populated when enable_response_messages is set to True
2103
+ # NOTE: custom serialization is needed
2104
+ # see serialize_input_messages and serialize_output_messages
2105
+ input_messages: Optional[list[ChatCompletionMessageParam]] = None
2106
+ output_messages: Optional[list[ChatCompletionMessageParam]] = None
2107
+ # --8<-- [end:responses-extra-params]
2108
+
2109
+ # NOTE: openAI harmony doesn't serialize TextContent properly,
2110
+ # TODO: this fixes for TextContent, but need to verify for tools etc
2111
+ # https://github.com/openai/harmony/issues/78
2112
+ @field_serializer("output_messages", when_used="json")
2113
+ def serialize_output_messages(self, msgs, _info):
2114
+ if msgs:
2115
+ serialized = []
2116
+ for m in msgs:
2117
+ if isinstance(m, dict):
2118
+ serialized.append(m)
2119
+ elif hasattr(m, "__dict__"):
2120
+ serialized.append(m.to_dict())
2121
+ else:
2122
+ # fallback to pyandic dump
2123
+ serialized.append(m.model_dump_json())
2124
+ return serialized
2125
+ return None
2126
+
2127
+ # NOTE: openAI harmony doesn't serialize TextContent properly, this fixes it
2128
+ # https://github.com/openai/harmony/issues/78
2129
+ @field_serializer("input_messages", when_used="json")
2130
+ def serialize_input_messages(self, msgs, _info):
2131
+ if msgs:
2132
+ serialized = []
2133
+ for m in msgs:
2134
+ if isinstance(m, dict):
2135
+ serialized.append(m)
2136
+ elif hasattr(m, "__dict__"):
2137
+ serialized.append(m.to_dict())
2138
+ else:
2139
+ # fallback to pyandic dump
2140
+ serialized.append(m.model_dump_json())
2141
+ return serialized
2142
+ return None
2143
+
2144
@classmethod
2145
def from_request(
2146
cls,
vllm/entrypoints/openai/serving_responses.py
+1
-1
@@ -1876,6 +1876,6 @@ class OpenAIServingResponses(OpenAIServing):
1876
ResponseCompletedEvent(
1877
type="response.completed",
1878
sequence_number=-1,
1879
- response=final_response.model_dump(),
1879
+ response=final_response,
1880
)
1881
)