A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[gpt-oss] Generate ResponseOutputItem from Harmony Message (#22410)
Signed-off-by: Chen Zhang <zhangch99@outlook.com>
Chen Zhang committed
Aug 7, 2025 at 08:33 UTC
4815b00f5487a070a40c7451c2cfcaef80786220
4 files changed
+292
-82
tests/v1/entrypoints/openai/responses/test_basic.py
+1
-1
@@ -17,7 +17,7 @@ async def test_simple_input(client: openai.AsyncOpenAI):
17
18
# Whether the output contains the reasoning.
19
assert outputs[0].type == "reasoning"
20
- assert outputs[0].text != ""
20
+ assert outputs[0].content[0].text != ""
21
22
23
@pytest.mark.asyncio
vllm/entrypoints/harmony_utils.py
+150
-3
@@ -1,18 +1,25 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import datetime
4
+import json
5
from collections.abc import Iterable, Sequence
6
from typing import Literal, Optional, Union
7
7
-from openai.types.responses import ResponseFunctionToolCall, ResponseOutputItem
8
+from openai.types.responses import (ResponseFunctionToolCall,
9
+ ResponseOutputItem, ResponseOutputMessage,
10
+ ResponseOutputText, ResponseReasoningItem)
11
+from openai.types.responses.response_function_web_search import (
12
+ ActionFind, ActionOpenPage, ActionSearch, ResponseFunctionWebSearch)
13
+from openai.types.responses.response_reasoning_item import (
14
+ Content as ResponseReasoningTextContent)
15
from openai.types.responses.tool import Tool
16
from openai_harmony import (Author, Conversation, DeveloperContent,
17
HarmonyEncodingName, Message, ReasoningEffort,
18
Role, StreamableParser, SystemContent, TextContent,
19
ToolDescription, load_harmony_encoding)
20
14
-from vllm.entrypoints.openai.protocol import (ResponseInputOutputItem,
15
- ResponseReasoningItem)
21
+from vllm.entrypoints.openai.protocol import ResponseInputOutputItem
22
+from vllm.utils import random_uuid
23
24
REASONING_EFFORT = {
25
"high": ReasoningEffort.HIGH,
@@ -160,6 +167,146 @@ def render_for_completion(messages: list[Message]) -> list[int]:
167
return token_ids
168
169
170
+def parse_output_message(message: Message) -> list[ResponseOutputItem]:
171
+ """
172
+ Parse a Harmony message into a list of output response items.
173
+ """
174
+ if message.author.role != "assistant":
175
+ # This is a message from a tool to the assistant (e.g., search result).
176
+ # Don't include it in the final output for now. This aligns with
177
+ # OpenAI's behavior on models like o4-mini.
178
+ return []
179
+
180
+ output_items: list[ResponseOutputItem] = []
181
+ recipient = message.recipient
182
+ if recipient is not None and recipient.startswith("browser."):
183
+ if len(message.content) != 1:
184
+ raise ValueError("Invalid number of contents in browser message")
185
+ content = message.content[0]
186
+ browser_call = json.loads(content.text)
187
+ # TODO: translate to url properly!
188
+ if recipient == "browser.search":
189
+ action = ActionSearch(
190
+ query=f"cursor:{browser_call.get('query', '')}", type="search")
191
+ elif recipient == "browser.open":
192
+ action = ActionOpenPage(
193
+ url=f"cursor:{browser_call.get('url', '')}", type="open_page")
194
+ elif recipient == "browser.find":
195
+ action = ActionFind(pattern=browser_call["pattern"],
196
+ url=f"cursor:{browser_call.get('url', '')}",
197
+ type="find")
198
+ else:
199
+ raise ValueError(f"Unknown browser action: {recipient}")
200
+ web_search_item = ResponseFunctionWebSearch(
201
+ id=f"ws_{random_uuid()}",
202
+ action=action,
203
+ status="completed",
204
+ type="web_search_call",
205
+ )
206
+ output_items.append(web_search_item)
207
+ elif message.channel == "analysis":
208
+ for content in message.content:
209
+ reasoning_item = ResponseReasoningItem(
210
+ id=f"rs_{random_uuid()}",
211
+ summary=[],
212
+ type="reasoning",
213
+ content=[
214
+ ResponseReasoningTextContent(text=content.text,
215
+ type="reasoning_text")
216
+ ],
217
+ status=None,
218
+ )
219
+ output_items.append(reasoning_item)
220
+ elif message.channel == "commentary":
221
+ if message.recipient.startswith("functions."):
222
+ function_name = message.recipient.split(".")[-1]
223
+ for content in message.content:
224
+ random_id = random_uuid()
225
+ response_item = ResponseFunctionToolCall(
226
+ arguments=content.text,
227
+ call_id=f"call_{random_id}",
228
+ type="function_call",
229
+ name=function_name,
230
+ id=f"ft_{random_id}",
231
+ )
232
+ output_items.append(response_item)
233
+ elif message.recipient.startswith(
234
+ "python") or message.recipient.startswith("browser"):
235
+ for content in message.content:
236
+ reasoning_item = ResponseReasoningItem(
237
+ id=f"rs_{random_uuid()}",
238
+ summary=[],
239
+ type="reasoning",
240
+ text=content.text,
241
+ status=None,
242
+ )
243
+ output_items.append(reasoning_item)
244
+ else:
245
+ raise ValueError(f"Unknown recipient: {message.recipient}")
246
+ elif message.channel == "final":
247
+ contents = []
248
+ for content in message.content:
249
+ output_text = ResponseOutputText(
250
+ text=content.text,
251
+ annotations=[], # TODO
252
+ type="output_text",
253
+ logprobs=None, # TODO
254
+ )
255
+ contents.append(output_text)
256
+ text_item = ResponseOutputMessage(
257
+ id=f"msg_{random_uuid()}",
258
+ content=contents,
259
+ role=message.author.role,
260
+ status="completed",
261
+ type="message",
262
+ )
263
+ output_items.append(text_item)
264
+ else:
265
+ raise ValueError(f"Unknown channel: {message.channel}")
266
+ return output_items
267
+
268
+
269
+def parse_remaining_state(
270
+ parser: StreamableParser) -> list[ResponseOutputItem]:
271
+ if not parser.current_content:
272
+ return []
273
+ if parser.current_role != Role.ASSISTANT:
274
+ return []
275
+ current_recipient = parser.current_recipient
276
+ if (current_recipient is not None
277
+ and current_recipient.startswith("browser.")):
278
+ return []
279
+
280
+ if parser.current_channel == "analysis":
281
+ reasoning_item = ResponseReasoningItem(
282
+ id=f"rs_{random_uuid()}",
283
+ summary=[],
284
+ type="reasoning",
285
+ content=[
286
+ ResponseReasoningTextContent(text=parser.current_content,
287
+ type="reasoning_text")
288
+ ],
289
+ status=None,
290
+ )
291
+ return [reasoning_item]
292
+ elif parser.current_channel == "final":
293
+ output_text = ResponseOutputText(
294
+ text=parser.current_content,
295
+ annotations=[], # TODO
296
+ type="output_text",
297
+ logprobs=None, # TODO
298
+ )
299
+ text_item = ResponseOutputMessage(
300
+ id=f"msg_{random_uuid()}",
301
+ content=[output_text],
302
+ role="assistant",
303
+ status="completed",
304
+ type="message",
305
+ )
306
+ return [text_item]
307
+ return []
308
+
309
+
310
def get_stop_tokens_for_assistant_actions() -> list[int]:
311
return get_encoding().stop_tokens_for_assistant_actions()
312
vllm/entrypoints/openai/protocol.py
+19
-12
@@ -19,8 +19,8 @@ from openai.types.chat.chat_completion_message import (
19
# yapf: enable
20
from openai.types.responses import (ResponseFunctionToolCall,
21
ResponseInputItemParam, ResponseOutputItem,
22
- ResponseOutputMessage, ResponsePrompt,
23
- ResponseStatus, ResponseTextConfig)
22
+ ResponsePrompt, ResponseStatus,
23
+ ResponseTextConfig)
24
from openai.types.responses.response import ToolChoice
25
from openai.types.responses.tool import Tool
26
from openai.types.shared import Metadata, Reasoning
@@ -1729,13 +1729,20 @@ class TranscriptionStreamResponse(OpenAIBaseModel):
1729
usage: Optional[UsageInfo] = Field(default=None)
1730
1731
1732
-class ResponseReasoningItem(OpenAIBaseModel):
1733
- id: str = Field(default_factory=lambda: f"rs_{random_uuid()}")
1734
- text: str
1735
- summary: list = Field(default_factory=list)
1736
- type: Literal["reasoning"] = "reasoning"
1737
- encrypted_content: Optional[str] = None
1738
- status: Optional[Literal["in_progress", "completed", "incomplete"]]
1732
+class InputTokensDetails(OpenAIBaseModel):
1733
+ cached_tokens: int
1734
+
1735
+
1736
+class OutputTokensDetails(OpenAIBaseModel):
1737
+ reasoning_tokens: int
1738
+
1739
+
1740
+class ResponseUsage(OpenAIBaseModel):
1741
+ input_tokens: int
1742
+ input_tokens_details: InputTokensDetails
1743
+ output_tokens: int
1744
+ output_tokens_details: OutputTokensDetails
1745
+ total_tokens: int
1746
1747
1748
class ResponsesResponse(OpenAIBaseModel):
@@ -1747,7 +1754,7 @@ class ResponsesResponse(OpenAIBaseModel):
1754
metadata: Optional[Metadata] = None
1755
model: str
1756
object: Literal["response"] = "response"
1750
- output: list[Union[ResponseOutputMessage, ResponseReasoningItem]]
1757
+ output: list[ResponseOutputItem]
1758
parallel_tool_calls: bool
1759
temperature: float
1760
tool_choice: ToolChoice
@@ -1764,7 +1771,7 @@ class ResponsesResponse(OpenAIBaseModel):
1771
text: Optional[ResponseTextConfig] = None
1772
top_logprobs: int
1773
truncation: Literal["auto", "disabled"]
1767
- usage: Optional[UsageInfo] = None
1774
+ usage: Optional[ResponseUsage] = None
1775
user: Optional[str] = None
1776
1777
@classmethod
@@ -1776,7 +1783,7 @@ class ResponsesResponse(OpenAIBaseModel):
1783
created_time: int,
1784
output: list[ResponseOutputItem],
1785
status: ResponseStatus,
1779
- usage: Optional[UsageInfo] = None,
1786
+ usage: Optional[ResponseUsage] = None,
1787
) -> "ResponsesResponse":
1788
return cls(
1789
id=request.request_id,
vllm/entrypoints/openai/serving_responses.py
+122
-66
@@ -6,12 +6,15 @@ import time
6
from collections.abc import AsyncGenerator, AsyncIterator
7
from copy import copy
8
from http import HTTPStatus
9
-from typing import Callable, Final, Optional, Union
9
+from typing import Any, Callable, Final, Optional, Union
10
11
import jinja2
12
from fastapi import Request
13
from openai.types.responses import (ResponseFunctionToolCall,
14
- ResponseOutputMessage, ResponseOutputText)
14
+ ResponseOutputItem, ResponseOutputMessage,
15
+ ResponseOutputText, ResponseReasoningItem)
16
+from openai.types.responses.response_reasoning_item import (
17
+ Content as ResponseReasoningTextContent)
18
from openai_harmony import Message as OpenAIHarmonyMessage
19
20
from vllm import envs
@@ -19,26 +22,28 @@ from vllm.config import ModelConfig
22
from vllm.engine.protocol import EngineClient
23
from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
24
ChatTemplateContentFormatOption)
22
-from vllm.entrypoints.context import ConversationContext, SimpleContext
25
+from vllm.entrypoints.context import (ConversationContext, HarmonyContext,
26
+ SimpleContext, StreamingHarmonyContext)
27
from vllm.entrypoints.harmony_utils import (
28
get_developer_message, get_stop_tokens_for_assistant_actions,
25
- get_system_message, get_user_message, parse_response_input,
26
- render_for_completion)
29
+ get_system_message, get_user_message, parse_output_message,
30
+ parse_remaining_state, parse_response_input, render_for_completion)
31
from vllm.entrypoints.logger import RequestLogger
32
# yapf conflicts with isort for this block
33
# yapf: disable
34
from vllm.entrypoints.openai.protocol import (ErrorResponse,
31
- PromptTokenUsageInfo,
35
+ InputTokensDetails,
36
+ OutputTokensDetails,
37
RequestResponseMetadata,
33
- ResponseReasoningItem,
38
ResponsesRequest,
35
- ResponsesResponse, UsageInfo)
39
+ ResponsesResponse, ResponseUsage)
40
# yapf: enable
41
from vllm.entrypoints.openai.serving_engine import OpenAIServing
42
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
43
from vllm.entrypoints.tool_server import ToolServer
44
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
45
from vllm.logger import init_logger
46
+from vllm.outputs import CompletionOutput
47
from vllm.reasoning import ReasoningParser, ReasoningParserManager
48
from vllm.sampling_params import SamplingParams
49
from vllm.transformers_utils.tokenizer import AnyTokenizer
@@ -222,6 +227,7 @@ class OpenAIServingResponses(OpenAIServing):
227
# Schedule the request and get the result generator.
228
generators: list[AsyncGenerator[ConversationContext, None]] = []
229
try:
230
+ tool_sessions: dict[str, Any] = {}
231
for i, engine_prompt in enumerate(engine_prompts):
232
default_max_tokens = self.max_model_len - len(
233
engine_prompt["prompt_token_ids"])
@@ -231,7 +237,15 @@ class OpenAIServingResponses(OpenAIServing):
237
trace_headers = (None if raw_request is None else await
238
self._get_trace_headers(raw_request.headers))
239
234
- context = SimpleContext()
240
+ context: ConversationContext
241
+ if self.use_harmony:
242
+ if request.stream:
243
+ context = StreamingHarmonyContext(
244
+ messages, tool_sessions)
245
+ else:
246
+ context = HarmonyContext(messages, tool_sessions)
247
+ else:
248
+ context = SimpleContext()
249
generator = self._generate_with_builtin_tools(
250
request_id=request.request_id,
251
request_prompt=request_prompts[i],
@@ -274,6 +288,7 @@ class OpenAIServingResponses(OpenAIServing):
288
request,
289
sampling_params,
290
result_generator,
291
+ context,
292
model_name,
293
tokenizer,
294
request_metadata,
@@ -297,6 +312,7 @@ class OpenAIServingResponses(OpenAIServing):
312
request,
313
sampling_params,
314
result_generator,
315
+ context,
316
model_name,
317
tokenizer,
318
request_metadata,
@@ -344,6 +360,7 @@ class OpenAIServingResponses(OpenAIServing):
360
request: ResponsesRequest,
361
sampling_params: SamplingParams,
362
result_generator: AsyncIterator[ConversationContext],
363
+ context: ConversationContext,
364
model_name: str,
365
tokenizer: AnyTokenizer,
366
request_metadata: RequestResponseMetadata,
@@ -352,9 +369,8 @@ class OpenAIServingResponses(OpenAIServing):
369
if created_time is None:
370
created_time = int(time.time())
371
355
- context: Optional[ConversationContext] = None
372
try:
357
- async for context in result_generator:
373
+ async for _ in result_generator:
374
pass
375
except asyncio.CancelledError:
376
return self.create_error_response("Client disconnected")
@@ -362,64 +378,40 @@ class OpenAIServingResponses(OpenAIServing):
378
# TODO: Use a vllm-specific Validation Error
379
return self.create_error_response(str(e))
380
365
- assert context is not None
366
- assert isinstance(context, SimpleContext)
367
- final_res = context.last_output
368
- assert final_res is not None
369
- assert len(final_res.outputs) == 1
370
- final_output = final_res.outputs[0]
371
-
372
- if self.reasoning_parser:
373
- try:
374
- reasoning_parser = self.reasoning_parser(tokenizer)
375
- except RuntimeError as e:
376
- logger.exception("Error in reasoning parser creation.")
377
- return self.create_error_response(str(e))
378
-
379
- reasoning_content, content = (
380
- reasoning_parser.extract_reasoning_content(final_output.text,
381
- request=request))
381
+ if self.use_harmony:
382
+ assert isinstance(context, HarmonyContext)
383
+ output = self._make_response_output_items_with_harmony(context)
384
+ # TODO: these are all 0 for now!
385
+ num_prompt_tokens = context.num_prompt_tokens
386
+ num_generated_tokens = context.num_output_tokens
387
+ num_cached_tokens = context.num_cached_tokens
388
+ num_reasoning_tokens = context.num_reasoning_tokens
389
else:
383
- reasoning_content = None
384
- content = final_output.text
385
-
386
- output = []
387
- if reasoning_content:
388
- reasoning_item = ResponseReasoningItem(
389
- text=reasoning_content,
390
- status=None, # NOTE: Only the last output item has status.
391
- )
392
- output.append(reasoning_item)
393
- if content:
394
- output_text = ResponseOutputText(
395
- text=content,
396
- annotations=[], # TODO
397
- type="output_text",
398
- logprobs=None, # TODO
399
- )
400
- message = ResponseOutputMessage(
401
- id=f"msg_{random_uuid()}",
402
- content=[output_text],
403
- role="assistant",
404
- status="completed",
405
- type="message",
406
- )
407
- output.append(message)
408
-
409
- # Calculate usage.
410
- assert final_res.prompt_token_ids is not None
411
- num_prompt_tokens = len(final_res.prompt_token_ids)
412
- num_generated_tokens = len(final_output.token_ids)
413
- usage = UsageInfo(
414
- prompt_tokens=num_prompt_tokens,
415
- completion_tokens=num_generated_tokens,
390
+ assert isinstance(context, SimpleContext)
391
+ final_res = context.last_output
392
+ assert final_res is not None
393
+ assert len(final_res.outputs) == 1
394
+ final_output = final_res.outputs[0]
395
+
396
+ output = self._make_response_output_items(request, final_output,
397
+ tokenizer)
398
+
399
+ # Calculate usage.
400
+ assert final_res.prompt_token_ids is not None
401
+ num_prompt_tokens = len(final_res.prompt_token_ids)
402
+ num_generated_tokens = len(final_output.token_ids)
403
+ num_cached_tokens = final_res.num_cached_tokens
404
+ num_reasoning_tokens = 0
405
+
406
+ usage = ResponseUsage(
407
+ input_tokens=num_prompt_tokens,
408
+ output_tokens=num_generated_tokens,
409
total_tokens=num_prompt_tokens + num_generated_tokens,
410
+ input_tokens_details=InputTokensDetails(
411
+ cached_tokens=num_cached_tokens),
412
+ output_tokens_details=OutputTokensDetails(
413
+ reasoning_tokens=num_reasoning_tokens),
414
)
418
- if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
419
- usage.prompt_tokens_details = PromptTokenUsageInfo(
420
- cached_tokens=final_res.num_cached_tokens)
421
- request_metadata.final_usage_info = usage
422
-
415
response = ResponsesResponse.from_request(
416
request,
417
sampling_params,
@@ -457,6 +449,70 @@ class OpenAIServingResponses(OpenAIServing):
449
self.response_store[response.id] = response
450
return response
451
452
+ def _make_response_output_items(
453
+ self,
454
+ request: ResponsesRequest,
455
+ final_output: CompletionOutput,
456
+ tokenizer: AnyTokenizer,
457
+ ) -> list[ResponseOutputItem]:
458
+ if self.reasoning_parser:
459
+ try:
460
+ reasoning_parser = self.reasoning_parser(tokenizer)
461
+ except RuntimeError as e:
462
+ logger.exception("Error in reasoning parser creation.")
463
+ raise e
464
+
465
+ reasoning_content, content = (
466
+ reasoning_parser.extract_reasoning_content(final_output.text,
467
+ request=request))
468
+ else:
469
+ reasoning_content = None
470
+ content = final_output.text
471
+
472
+ output = []
473
+ if reasoning_content:
474
+ reasoning_item = ResponseReasoningItem(
475
+ id=f"rs_{random_uuid()}",
476
+ summary=[],
477
+ type="reasoning",
478
+ content=[
479
+ ResponseReasoningTextContent(text=reasoning_content,
480
+ type="reasoning_text")
481
+ ],
482
+ status=None, # NOTE: Only the last output item has status.
483
+ )
484
+ output.append(reasoning_item)
485
+ if content:
486
+ output_text = ResponseOutputText(
487
+ text=content,
488
+ annotations=[], # TODO
489
+ type="output_text",
490
+ logprobs=None, # TODO
491
+ )
492
+ message = ResponseOutputMessage(
493
+ id=f"msg_{random_uuid()}",
494
+ content=[output_text],
495
+ role="assistant",
496
+ status="completed",
497
+ type="message",
498
+ )
499
+ output.append(message)
500
+ return output
501
+
502
+ def _make_response_output_items_with_harmony(
503
+ self,
504
+ context: HarmonyContext,
505
+ ) -> list[ResponseOutputItem]:
506
+ output_items = []
507
+ num_init_messages = context.num_init_messages
508
+ for msg in context.messages[num_init_messages:]:
509
+ output_items.extend(parse_output_message(msg))
510
+ # Handle the generation stopped in the middle (if any).
511
+ last_items = parse_remaining_state(context.parser)
512
+ if last_items:
513
+ output_items.extend(last_items)
514
+ return output_items
515
+
516
def _construct_input_messages(
517
self,
518
request: ResponsesRequest,