@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8f423e5f43

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Feature][Response API] Add streaming support for non-harmony (#23741)

Signed-off-by: Kebe <mail@kebe7jun.com>

Kebe committed Sep 4, 2025 at 18:49 UTC 8f423e5f43eca54c9b0e5583ab388fd6d5cc8672

3 files changed +407 -77

tests/v1/entrypoints/openai/responses/test_basic.py

+16

@@ -2,6 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 import openai # use the official client for correctness check

5 +import openai.types.responses as openai_responses_types

6 import pytest

@@ -86,3 +87,18 @@ async def test_logprobs(client: openai.AsyncOpenAI):

87 outputs = response.output

88 assert outputs[-1].content[-1].logprobs

89 assert len(outputs[-1].content[-1].logprobs[0].top_logprobs) == 5

90 +

91 +

92 +@pytest.mark.asyncio

93 +async def test_streaming(client: openai.AsyncOpenAI):

94 + stream = await client.responses.create(

95 + input="What is 13 * 24?",

96 + stream=True,

97 + )

98 + events = [event async for event in stream]

99 + assert isinstance(events[0], openai_responses_types.ResponseCreatedEvent)

100 + assert any(

101 + isinstance(event, openai_responses_types.ResponseTextDeltaEvent)

102 + for event in events)

103 + assert isinstance(events[-1],

104 + openai_responses_types.ResponseCompletedEvent)

vllm/entrypoints/context.py

+10

@@ -49,9 +49,19 @@ class SimpleContext(ConversationContext):

50 def __init__(self):

51 self.last_output = None

52 + self.num_prompt_tokens = 0

53 + self.num_output_tokens = 0

54 + self.num_cached_tokens = 0

55 + # todo num_reasoning_tokens is not implemented yet.

56 + self.num_reasoning_tokens = 0

58 def append_output(self, output) -> None:

59 self.last_output = output

60 + if not isinstance(output, RequestOutput):

61 + raise ValueError("SimpleContext only supports RequestOutput.")

62 + self.num_prompt_tokens = len(output.prompt_token_ids or [])

63 + self.num_cached_tokens = output.num_cached_tokens or 0

64 + self.num_output_tokens += len(output.outputs[0].token_ids or [])

66 def need_builtin_tool_call(self) -> bool:

67 return False

vllm/entrypoints/openai/serving_responses.py

+381 -77

@@ -4,6 +4,7 @@

4 import asyncio

5 import json

6 import time

7 +import uuid

8 from collections import deque

9 from collections.abc import AsyncGenerator, AsyncIterator, Sequence

10 from contextlib import AsyncExitStack

@@ -25,7 +26,8 @@ from openai.types.responses import (ResponseCreatedEvent,

26 ResponseOutputMessage, ResponseOutputText,

27 ResponseReasoningItem,

28 ResponseReasoningTextDeltaEvent,

28 - ResponseReasoningTextDoneEvent)

29 + ResponseReasoningTextDoneEvent,

30 + response_text_delta_event)

31 from openai.types.responses.response_output_text import (Logprob,

32 LogprobTopLogprob)

33 # yapf: enable

@@ -47,7 +49,7 @@ from vllm.entrypoints.harmony_utils import (

49 from vllm.entrypoints.logger import RequestLogger

50 # yapf conflicts with isort for this block

51 # yapf: disable

50 -from vllm.entrypoints.openai.protocol import (ErrorResponse,

52 +from vllm.entrypoints.openai.protocol import (DeltaMessage, ErrorResponse,

53 InputTokensDetails,

54 OutputTokensDetails,

55 RequestResponseMetadata,

@@ -459,10 +461,6 @@ class OpenAIServingResponses(OpenAIServing):

461 assert isinstance(context, HarmonyContext)

462 output = self._make_response_output_items_with_harmony(context)

463 # TODO: these are all 0 for now!

462 - num_prompt_tokens = context.num_prompt_tokens

463 - num_generated_tokens = context.num_output_tokens

464 - num_cached_tokens = context.num_cached_tokens

465 - num_reasoning_tokens = context.num_reasoning_tokens

464 else:

465 assert isinstance(context, SimpleContext)

466 final_res = context.last_output

@@ -475,10 +473,11 @@ class OpenAIServingResponses(OpenAIServing):

473

474 # Calculate usage.

475 assert final_res.prompt_token_ids is not None

478 - num_prompt_tokens = len(final_res.prompt_token_ids)

479 - num_generated_tokens = len(final_output.token_ids)

480 - num_cached_tokens = final_res.num_cached_tokens

481 - num_reasoning_tokens = 0

476 + assert isinstance(context, (SimpleContext, HarmonyContext))

477 + num_prompt_tokens = context.num_prompt_tokens

478 + num_generated_tokens = context.num_output_tokens

479 + num_cached_tokens = context.num_cached_tokens

480 + num_reasoning_tokens = context.num_reasoning_tokens

481

482 usage = ResponseUsage(

483 input_tokens=num_prompt_tokens,

@@ -553,6 +552,28 @@ class OpenAIServingResponses(OpenAIServing):

552 ))

553 return out

554

555 + def _create_stream_response_logprobs(

556 + self,

557 + token_ids: Sequence[int],

558 + logprobs: Optional[SampleLogprobs],

559 + tokenizer: AnyTokenizer,

560 + top_logprobs: Optional[int] = None

561 + ) -> list[response_text_delta_event.Logprob]:

562 + lgs = self._create_response_logprobs(token_ids=token_ids,

563 + logprobs=logprobs,

564 + tokenizer=tokenizer,

565 + top_logprobs=top_logprobs)

566 + return [

567 + response_text_delta_event.Logprob(

568 + token=lg.token,

569 + logprob=lg.logprob,

570 + top_logprobs=[

571 + response_text_delta_event.LogprobTopLogprob(

572 + token=tl.token, logprob=tl.logprob)

573 + for tl in lg.top_logprobs

574 + ]) for lg in lgs

575 + ]

576 +

577 def _make_response_output_items(

578 self,

579 request: ResponsesRequest,

@@ -912,7 +933,7 @@ class OpenAIServingResponses(OpenAIServing):

933 status_code=HTTPStatus.BAD_REQUEST,

934 )

935

915 - async def _process_streaming_events(

936 + async def _process_simple_streaming_events(

937 self,

938 request: ResponsesRequest,

939 sampling_params: SamplingParams,

@@ -922,47 +943,292 @@ class OpenAIServingResponses(OpenAIServing):

943 tokenizer: AnyTokenizer,

944 request_metadata: RequestResponseMetadata,

945 created_time: int,

946 + _send_event: Callable[[BaseModel], str],

947 ) -> AsyncGenerator[str, None]:

926 - sequence_number = 0

927 -

928 - def _send_event(event: BaseModel):

929 - nonlocal sequence_number

930 - # Set sequence_number if the event has this attribute

931 - if hasattr(event, 'sequence_number'):

932 - event.sequence_number = sequence_number

933 - sequence_number += 1

934 - # Get event type from the event's type field if it exists

935 - event_type = getattr(event, 'type', 'unknown')

936 - return (f"event: {event_type}\n"

937 - f"data: {event.model_dump_json(indent=None)}\n\n")

948 + current_content_index = 0

949 + current_output_index = 0

950 + current_item_id = ""

951 + reasoning_parser = None

952 + if self.reasoning_parser:

953 + reasoning_parser = self.reasoning_parser(tokenizer)

954 + previous_text = ""

955 + previous_token_ids: list[int] = []

956 + first_delta_sent = False

957 + previous_delta_messages: list[DeltaMessage] = []

958 + async for ctx in result_generator:

959 + assert isinstance(ctx, SimpleContext)

960 + if ctx.last_output is None:

961 + continue

962 + if ctx.last_output.outputs:

963 + output = ctx.last_output.outputs[0]

964 + if reasoning_parser:

965 + delta_message = \

966 + reasoning_parser.extract_reasoning_content_streaming(

967 + previous_text=previous_text,

968 + current_text=previous_text + output.text,

969 + delta_text=output.text,

970 + previous_token_ids=previous_token_ids,

971 + current_token_ids=previous_token_ids +

972 + output.token_ids,

973 + delta_token_ids=output.token_ids,

974 + )

975 + else:

976 + delta_message = DeltaMessage(content=output.text, )

977 + previous_text += output.text

978 + previous_token_ids += output.token_ids

979 + if not delta_message:

980 + continue

981 + if not first_delta_sent:

982 + current_item_id = str(uuid.uuid4())

983 + if delta_message.reasoning_content:

984 + yield _send_event(

985 + openai_responses_types.

986 + ResponseOutputItemAddedEvent(

987 + type="response.output_item.added",

988 + sequence_number=-1,

989 + output_index=current_output_index,

990 + item=openai_responses_types.

991 + ResponseReasoningItem(

992 + type="reasoning",

993 + id=current_item_id,

994 + summary=[],

995 + status="in_progress",

996 + ),

997 + ))

998 + else:

999 + yield _send_event(

1000 + openai_responses_types.

1001 + ResponseOutputItemAddedEvent(

1002 + type="response.output_item.added",

1003 + sequence_number=-1,

1004 + output_index=current_output_index,

1005 + item=openai_responses_types.

1006 + ResponseOutputMessage(

1007 + id=current_item_id,

1008 + type="message",

1009 + role="assistant",

1010 + content=[],

1011 + status="in_progress",

1012 + ),

1013 + ))

1014 + yield _send_event(

1015 + openai_responses_types.ResponseContentPartAddedEvent(

1016 + type="response.content_part.added",

1017 + sequence_number=-1,

1018 + output_index=current_output_index,

1019 + item_id=current_item_id,

1020 + content_index=current_content_index,

1021 + part=openai_responses_types.ResponseOutputText(

1022 + type="output_text",

1023 + text="",

1024 + annotations=[],

1025 + logprobs=[],

1026 + ),

1027 + ))

1028 + current_content_index += 1

1029 + first_delta_sent = True

1030 + # todo(kebe7jun) tool call support

1031 +

1032 + # check delta message and previous delta message are

1033 + # same as content or reasoning content

1034 + if (previous_delta_messages

1035 + and previous_delta_messages[-1].reasoning_content

1036 + is not None and delta_message.content is not None):

1037 + # from reasoning to normal content, send done

1038 + # event for reasoning

1039 + reason_content = ''.join(

1040 + pm.reasoning_content for pm in previous_delta_messages

1041 + if pm.reasoning_content is not None)

1042 + yield _send_event(

1043 + ResponseReasoningTextDoneEvent(

1044 + type="response.reasoning_text.done",

1045 + item_id=current_item_id,

1046 + sequence_number=-1,

1047 + output_index=current_output_index,

1048 + content_index=current_content_index,

1049 + text=reason_content,

1050 + ))

1051 + current_content_index = 0

1052 + reasoning_item = ResponseReasoningItem(

1053 + type="reasoning",

1054 + content=[

1055 + ResponseReasoningTextContent(

1056 + text=reason_content,

1057 + type="reasoning_text",

1058 + ),

1059 + ],

1060 + status="completed",

1061 + id=current_item_id,

1062 + summary=[],

1063 + )

1064 + yield _send_event(

1065 + ResponseOutputItemDoneEvent(

1066 + type="response.output_item.done",

1067 + sequence_number=-1,

1068 + output_index=current_output_index,

1069 + item=reasoning_item,

1070 + ))

1071 + yield _send_event(

1072 + openai_responses_types.ResponseOutputItemAddedEvent(

1073 + type="response.output_item.added",

1074 + sequence_number=-1,

1075 + output_index=current_output_index,

1076 + item=openai_responses_types.ResponseOutputMessage(

1077 + id=current_item_id,

1078 + type="message",

1079 + role="assistant",

1080 + content=[],

1081 + status="in_progress",

1082 + ),

1083 + ))

1084 + current_output_index += 1

1085 + current_item_id = str(uuid.uuid4())

1086 + yield _send_event(

1087 + openai_responses_types.ResponseContentPartAddedEvent(

1088 + type="response.content_part.added",

1089 + sequence_number=-1,

1090 + output_index=current_output_index,

1091 + item_id=current_item_id,

1092 + content_index=current_content_index,

1093 + part=openai_responses_types.ResponseOutputText(

1094 + type="output_text",

1095 + text="",

1096 + annotations=[],

1097 + logprobs=[],

1098 + ),

1099 + ))

1100 + current_content_index += 1

1101 + # reset previous delta messages

1102 + previous_delta_messages = []

1103

1104 + if delta_message.reasoning_content is not None:

1105 + yield _send_event(

1106 + ResponseReasoningTextDeltaEvent(

1107 + type="response.reasoning_text.delta",

1108 + sequence_number=-1,

1109 + content_index=current_content_index,

1110 + output_index=current_output_index,

1111 + item_id=current_item_id,

1112 + delta=delta_message.reasoning_content,

1113 + ))

1114 + elif delta_message.content is not None:

1115 + yield _send_event(

1116 + openai_responses_types.ResponseTextDeltaEvent(

1117 + type="response.output_text.delta",

1118 + sequence_number=-1,

1119 + content_index=current_content_index,

1120 + output_index=current_output_index,

1121 + item_id=current_item_id,

1122 + delta=delta_message.content,

1123 + logprobs=self._create_stream_response_logprobs(

1124 + token_ids=output.token_ids,

1125 + logprobs=output.logprobs,

1126 + tokenizer=tokenizer,

1127 + top_logprobs=request.top_logprobs,

1128 + ) if request.is_include_output_logprobs() else [],

1129 + ))

1130 + current_content_index += 1

1131 +

1132 + previous_delta_messages.append(delta_message)

1133 + if previous_delta_messages:

1134 + if previous_delta_messages[-1].reasoning_content is not None:

1135 + reason_content = ''.join(pm.reasoning_content

1136 + for pm in previous_delta_messages

1137 + if pm.reasoning_content is not None)

1138 + yield _send_event(

1139 + ResponseReasoningTextDoneEvent(

1140 + type="response.reasoning_text.done",

1141 + item_id=current_item_id,

1142 + sequence_number=-1,

1143 + output_index=current_output_index,

1144 + content_index=current_content_index,

1145 + text=reason_content,

1146 + ))

1147 + current_content_index += 1

1148 + reasoning_item = ResponseReasoningItem(

1149 + type="reasoning",

1150 + content=[

1151 + ResponseReasoningTextContent(

1152 + text=reason_content,

1153 + type="reasoning_text",

1154 + ),

1155 + ],

1156 + status="completed",

1157 + id=current_item_id,

1158 + summary=[],

1159 + )

1160 + yield _send_event(

1161 + ResponseOutputItemDoneEvent(

1162 + type="response.output_item.done",

1163 + sequence_number=-1,

1164 + output_index=current_output_index,

1165 + item=reasoning_item,

1166 + ))

1167 + elif previous_delta_messages[-1].content is not None:

1168 + final_content = ''.join(pm.content

1169 + for pm in previous_delta_messages

1170 + if pm.content is not None)

1171 + yield _send_event(

1172 + openai_responses_types.ResponseTextDoneEvent(

1173 + type="response.output_text.done",

1174 + sequence_number=-1,

1175 + output_index=current_output_index,

1176 + content_index=current_content_index,

1177 + text=final_content,

1178 + logprobs=[],

1179 + item_id=current_item_id,

1180 + ))

1181 + current_content_index += 1

1182 + part = ResponseOutputText(

1183 + text=final_content,

1184 + type="output_text",

1185 + annotations=[],

1186 + )

1187 + yield _send_event(

1188 + openai_responses_types.ResponseContentPartDoneEvent(

1189 + type="response.content_part.done",

1190 + sequence_number=-1,

1191 + item_id=current_item_id,

1192 + output_index=current_output_index,

1193 + content_index=current_content_index,

1194 + part=part,

1195 + ))

1196 + current_content_index += 1

1197 + item = ResponseOutputMessage(

1198 + type="message",

1199 + role="assistant",

1200 + content=[

1201 + part,

1202 + ],

1203 + status="completed",

1204 + id=current_item_id,

1205 + summary=[],

1206 + )

1207 + yield _send_event(

1208 + ResponseOutputItemDoneEvent(

1209 + type="response.output_item.done",

1210 + sequence_number=-1,

1211 + output_index=current_output_index,

1212 + item=item,

1213 + ))

1214 +

1215 + async def _process_harmony_streaming_events(

1216 + self,

1217 + request: ResponsesRequest,

1218 + sampling_params: SamplingParams,

1219 + result_generator: AsyncIterator[Optional[ConversationContext]],

1220 + context: ConversationContext,

1221 + model_name: str,

1222 + tokenizer: AnyTokenizer,

1223 + request_metadata: RequestResponseMetadata,

1224 + created_time: int,

1225 + _send_event: Callable[[BaseModel], str],

1226 + ) -> AsyncGenerator[str, None]:

1227 current_content_index = 0 # FIXME: this number is never changed

1228 current_output_index = 0

1229 current_item_id = "" # FIXME: this number is never changed

1230 sent_output_item_added = False

1231

944 - initial_response = ResponsesResponse.from_request(

945 - request,

946 - sampling_params,

947 - model_name=model_name,

948 - created_time=created_time,

949 - output=[],

950 - status="in_progress",

951 - usage=None,

952 - ).model_dump()

953 - yield _send_event(

954 - ResponseCreatedEvent(

955 - type="response.created",

956 - sequence_number=-1,

957 - response=initial_response,

958 - ))

959 - yield _send_event(

960 - ResponseInProgressEvent(

961 - type="response.in_progress",

962 - sequence_number=-1,

963 - response=initial_response,

964 - ))

965 -

1232 async for ctx in result_generator:

1233

1234 assert isinstance(ctx, StreamingHarmonyContext)

@@ -1312,29 +1578,6 @@ class OpenAIServingResponses(OpenAIServing):

1578 ),

1579 ))

1580

1315 - async def empty_async_generator():

1316 - # A hack to trick Python to think this is a generator but in fact

1317 - # it immediately returns.

1318 - if False:

1319 - yield

1320 -

1321 - final_response = await self.responses_full_generator(

1322 - request,

1323 - sampling_params,

1324 - empty_async_generator(),

1325 - context,

1326 - model_name,

1327 - tokenizer,

1328 - request_metadata,

1329 - created_time=created_time,

1330 - )

1331 - yield _send_event(

1332 - openai_responses_types.ResponseCompletedEvent(

1333 - type="response.completed",

1334 - sequence_number=-1,

1335 - response=final_response.model_dump(),

1336 - ))

1337 -

1581 async def responses_stream_generator(

1582 self,

1583 request: ResponsesRequest,

@@ -1349,16 +1592,77 @@ class OpenAIServingResponses(OpenAIServing):

1592 # TODO:

1593 # 1. Handle disconnect

1594

1352 - if not isinstance(context, StreamingHarmonyContext):

1353 - raise NotImplementedError(

1354 - "Streaming is not supported for responses API without Harmony."

1355 - )

1356 -

1595 created_time = created_time or int(time.time())

1596

1597 + sequence_number = 0

1598 +

1599 + def _send_event(event: BaseModel):

1600 + nonlocal sequence_number

1601 + # Set sequence_number if the event has this attribute

1602 + if hasattr(event, 'sequence_number'):

1603 + event.sequence_number = sequence_number

1604 + sequence_number += 1

1605 + # Get event type from the event's type field if it exists

1606 + event_type = getattr(event, 'type', 'unknown')

1607 + return (f"event: {event_type}\n"

1608 + f"data: {event.model_dump_json(indent=None)}\n\n")

1609 +

1610 async with AsyncExitStack() as exit_stack:

1360 - await context.init_tool_sessions(self.tool_server, exit_stack)

1361 - async for event_data in self._process_streaming_events(

1362 - request, sampling_params, result_generator, context,

1363 - model_name, tokenizer, request_metadata, created_time):

1611 + processer = None

1612 + if self.use_harmony:

1613 + await context.init_tool_sessions(self.tool_server, exit_stack)

1614 + processer = self._process_harmony_streaming_events

1615 + else:

1616 + processer = self._process_simple_streaming_events

1617 +

1618 + initial_response = ResponsesResponse.from_request(

1619 + request,

1620 + sampling_params,

1621 + model_name=model_name,

1622 + created_time=created_time,

1623 + output=[],

1624 + status="in_progress",

1625 + usage=None,

1626 + ).model_dump()

1627 + yield _send_event(

1628 + ResponseCreatedEvent(

1629 + type="response.created",

1630 + sequence_number=-1,

1631 + response=initial_response,

1632 + ))

1633 + yield _send_event(

1634 + ResponseInProgressEvent(

1635 + type="response.in_progress",

1636 + sequence_number=-1,

1637 + response=initial_response,

1638 + ))

1639 +

1640 + async for event_data in processer(request, sampling_params,

1641 + result_generator, context,

1642 + model_name, tokenizer,

1643 + request_metadata, created_time,

1644 + _send_event):

1645 yield event_data

1646 +

1647 + async def empty_async_generator():

1648 + # A hack to trick Python to think this is a generator but

1649 + # in fact it immediately returns.

1650 + if False:

1651 + yield

1652 +

1653 + final_response = await self.responses_full_generator(

1654 + request,

1655 + sampling_params,

1656 + empty_async_generator(),

1657 + context,

1658 + model_name,

1659 + tokenizer,

1660 + request_metadata,

1661 + created_time=created_time,

1662 + )

1663 + yield _send_event(

1664 + openai_responses_types.ResponseCompletedEvent(

1665 + type="response.completed",

1666 + sequence_number=-1,

1667 + response=final_response.model_dump(),

1668 + ))