@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 73df49ef3a

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[gpt-oss][1a] create_responses stream outputs BaseModel type, api server is SSE still (#24759)

Signed-off-by: Andrew Xia <axia@meta.com>

Andrew Xia committed Sep 15, 2025 at 13:08 UTC 73df49ef3a220c79abfffc36bdfb4e8dee61226b

2 files changed +90 -71

vllm/entrypoints/openai/api_server.py

+21 -6

@@ -15,7 +15,7 @@ import socket

15 import tempfile

16 import uuid

17 from argparse import Namespace

18 -from collections.abc import AsyncIterator, Awaitable

18 +from collections.abc import AsyncGenerator, AsyncIterator, Awaitable

19 from contextlib import asynccontextmanager

20 from functools import partial

21 from http import HTTPStatus

@@ -29,6 +29,7 @@ from fastapi import APIRouter, Depends, FastAPI, Form, HTTPException, Request

29 from fastapi.exceptions import RequestValidationError

30 from fastapi.middleware.cors import CORSMiddleware

31 from fastapi.responses import JSONResponse, Response, StreamingResponse

32 +from openai import BaseModel

33 from prometheus_client import make_asgi_app

34 from prometheus_fastapi_instrumentator import Instrumentator

35 from starlette.concurrency import iterate_in_threadpool

@@ -577,6 +578,18 @@ async def show_version():

578 return JSONResponse(content=ver)

579

580

581 +async def _convert_stream_to_sse_events(

582 + generator: AsyncGenerator[BaseModel,

583 + None]) -> AsyncGenerator[str, None]:

584 + """Convert the generator to a stream of events in SSE format"""

585 + async for event in generator:

586 + event_type = getattr(event, 'type', 'unknown')

587 + # https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#event_stream_format

588 + event_data = (f"event: {event_type}\n"

589 + f"data: {event.model_dump_json(indent=None)}\n\n")

590 + yield event_data

591 +

592 +

593 @router.post("/v1/responses",

594 dependencies=[Depends(validate_json_request)],

595 responses={

@@ -612,7 +625,9 @@ async def create_responses(request: ResponsesRequest, raw_request: Request):

625 status_code=generator.error.code)

626 elif isinstance(generator, ResponsesResponse):

627 return JSONResponse(content=generator.model_dump())

615 - return StreamingResponse(content=generator, media_type="text/event-stream")

628 +

629 + return StreamingResponse(content=_convert_stream_to_sse_events(generator),

630 + media_type="text/event-stream")

631

632

633 @router.get("/v1/responses/{response_id}")

@@ -640,10 +655,10 @@ async def retrieve_responses(

655 if isinstance(response, ErrorResponse):

656 return JSONResponse(content=response.model_dump(),

657 status_code=response.error.code)

643 - elif stream:

644 - return StreamingResponse(content=response,

645 - media_type="text/event-stream")

646 - return JSONResponse(content=response.model_dump())

658 + elif isinstance(response, ResponsesResponse):

659 + return JSONResponse(content=response.model_dump())

660 + return StreamingResponse(content=_convert_stream_to_sse_events(response),

661 + media_type="text/event-stream")

662

663

664 @router.post("/v1/responses/{response_id}/cancel")

vllm/entrypoints/openai/serving_responses.py

+69 -65

@@ -10,7 +10,7 @@ from collections.abc import AsyncGenerator, AsyncIterator, Sequence

10 from contextlib import AsyncExitStack

11 from copy import copy

12 from http import HTTPStatus

13 -from typing import Callable, Final, Optional, Union

13 +from typing import Callable, Final, Optional, TypeVar, Union

15 import jinja2

16 import openai.types.responses as openai_responses_types

@@ -175,7 +175,8 @@ class OpenAIServingResponses(OpenAIServing):

175 # HACK(wuhang): This is a hack. We should use a better store.

176 # FIXME: If enable_store=True, this may cause a memory leak since we

177 # never remove events from the store.

178 - self.event_store: dict[str, tuple[deque[str], asyncio.Event]] = {}

178 + self.event_store: dict[str, tuple[deque[BaseModel],

179 + asyncio.Event]] = {}

180

181 self.background_tasks: dict[str, asyncio.Task] = {}

182

@@ -185,7 +186,8 @@ class OpenAIServingResponses(OpenAIServing):

186 self,

187 request: ResponsesRequest,

188 raw_request: Optional[Request] = None,

188 - ) -> Union[AsyncGenerator[str, None], ResponsesResponse, ErrorResponse]:

189 + ) -> Union[AsyncGenerator[BaseModel, None], ResponsesResponse,

190 + ErrorResponse]:

191 error_check_ret = await self._check_model(request)

192 if error_check_ret is not None:

193 logger.error("Error with model %s", error_check_ret)

@@ -812,7 +814,7 @@ class OpenAIServingResponses(OpenAIServing):

814 *args,

815 **kwargs,

816 ):

815 - event_deque: deque[str] = deque()

817 + event_deque: deque[BaseModel] = deque()

818 new_event_signal = asyncio.Event()

819 self.event_store[request.request_id] = (event_deque, new_event_signal)

820 response = None

@@ -827,8 +829,6 @@ class OpenAIServingResponses(OpenAIServing):

829 request.request_id)

830 response = self.create_error_response(str(e))

831 finally:

830 - # Mark as finished with a special marker

831 - event_deque.append("__STREAM_END__")

832 new_event_signal.set()

833

834 if response is not None and isinstance(response, ErrorResponse):

@@ -867,7 +867,7 @@ class OpenAIServingResponses(OpenAIServing):

867 self,

868 response_id: str,

869 starting_after: Optional[int] = None,

870 - ):

870 + ) -> AsyncGenerator[BaseModel, None]:

871 if response_id not in self.event_store:

872 raise ValueError(f"Unknown response_id: {response_id}")

873

@@ -881,9 +881,9 @@ class OpenAIServingResponses(OpenAIServing):

881 # Yield existing events from start_index

882 while current_index < len(event_deque):

883 event = event_deque[current_index]

884 - if event == "__STREAM_END__":

885 - return

884 yield event

885 + if getattr(event, 'type', 'unknown') == "response.completed":

886 + return

887 current_index += 1

888

889 await new_event_signal.wait()

@@ -893,7 +893,8 @@ class OpenAIServingResponses(OpenAIServing):

893 response_id: str,

894 starting_after: Optional[int],

895 stream: Optional[bool],

896 - ) -> Union[ErrorResponse, ResponsesResponse]:

896 + ) -> Union[ErrorResponse, ResponsesResponse, AsyncGenerator[BaseModel,

897 + None]]:

898 if not response_id.startswith("resp_"):

899 return self._make_invalid_id_error(response_id)

900

@@ -976,8 +977,9 @@ class OpenAIServingResponses(OpenAIServing):

977 tokenizer: AnyTokenizer,

978 request_metadata: RequestResponseMetadata,

979 created_time: int,

979 - _send_event: Callable[[BaseModel], str],

980 - ) -> AsyncGenerator[str, None]:

980 + _increment_sequence_number_and_return: Callable[[BaseModel],

981 + BaseModel],

982 + ) -> AsyncGenerator[BaseModel, None]:

983 current_content_index = 0

984 current_output_index = 0

985 current_item_id = ""

@@ -1014,7 +1016,7 @@ class OpenAIServingResponses(OpenAIServing):

1016 if not first_delta_sent:

1017 current_item_id = str(uuid.uuid4())

1018 if delta_message.reasoning_content:

1017 - yield _send_event(

1019 + yield _increment_sequence_number_and_return(

1020 openai_responses_types.

1021 ResponseOutputItemAddedEvent(

1022 type="response.output_item.added",

@@ -1029,7 +1031,7 @@ class OpenAIServingResponses(OpenAIServing):

1031 ),

1032 ))

1033 else:

1032 - yield _send_event(

1034 + yield _increment_sequence_number_and_return(

1035 openai_responses_types.

1036 ResponseOutputItemAddedEvent(

1037 type="response.output_item.added",

@@ -1044,7 +1046,7 @@ class OpenAIServingResponses(OpenAIServing):

1046 status="in_progress",

1047 ),

1048 ))

1047 - yield _send_event(

1049 + yield _increment_sequence_number_and_return(

1050 openai_responses_types.ResponseContentPartAddedEvent(

1051 type="response.content_part.added",

1052 sequence_number=-1,

@@ -1072,7 +1074,7 @@ class OpenAIServingResponses(OpenAIServing):

1074 reason_content = ''.join(

1075 pm.reasoning_content for pm in previous_delta_messages

1076 if pm.reasoning_content is not None)

1075 - yield _send_event(

1077 + yield _increment_sequence_number_and_return(

1078 ResponseReasoningTextDoneEvent(

1079 type="response.reasoning_text.done",

1080 item_id=current_item_id,

@@ -1094,14 +1096,14 @@ class OpenAIServingResponses(OpenAIServing):

1096 id=current_item_id,

1097 summary=[],

1098 )

1097 - yield _send_event(

1099 + yield _increment_sequence_number_and_return(

1100 ResponseOutputItemDoneEvent(

1101 type="response.output_item.done",

1102 sequence_number=-1,

1103 output_index=current_output_index,

1104 item=reasoning_item,

1105 ))

1104 - yield _send_event(

1106 + yield _increment_sequence_number_and_return(

1107 openai_responses_types.ResponseOutputItemAddedEvent(

1108 type="response.output_item.added",

1109 sequence_number=-1,

@@ -1116,7 +1118,7 @@ class OpenAIServingResponses(OpenAIServing):

1118 ))

1119 current_output_index += 1

1120 current_item_id = str(uuid.uuid4())

1119 - yield _send_event(

1121 + yield _increment_sequence_number_and_return(

1122 openai_responses_types.ResponseContentPartAddedEvent(

1123 type="response.content_part.added",

1124 sequence_number=-1,

@@ -1135,7 +1137,7 @@ class OpenAIServingResponses(OpenAIServing):

1137 previous_delta_messages = []

1138

1139 if delta_message.reasoning_content is not None:

1138 - yield _send_event(

1140 + yield _increment_sequence_number_and_return(

1141 ResponseReasoningTextDeltaEvent(

1142 type="response.reasoning_text.delta",

1143 sequence_number=-1,

@@ -1145,7 +1147,7 @@ class OpenAIServingResponses(OpenAIServing):

1147 delta=delta_message.reasoning_content,

1148 ))

1149 elif delta_message.content is not None:

1148 - yield _send_event(

1150 + yield _increment_sequence_number_and_return(

1151 openai_responses_types.ResponseTextDeltaEvent(

1152 type="response.output_text.delta",

1153 sequence_number=-1,

@@ -1168,7 +1170,7 @@ class OpenAIServingResponses(OpenAIServing):

1170 reason_content = ''.join(pm.reasoning_content

1171 for pm in previous_delta_messages

1172 if pm.reasoning_content is not None)

1171 - yield _send_event(

1173 + yield _increment_sequence_number_and_return(

1174 ResponseReasoningTextDoneEvent(

1175 type="response.reasoning_text.done",

1176 item_id=current_item_id,

@@ -1190,7 +1192,7 @@ class OpenAIServingResponses(OpenAIServing):

1192 id=current_item_id,

1193 summary=[],

1194 )

1193 - yield _send_event(

1195 + yield _increment_sequence_number_and_return(

1196 ResponseOutputItemDoneEvent(

1197 type="response.output_item.done",

1198 sequence_number=-1,

@@ -1201,7 +1203,7 @@ class OpenAIServingResponses(OpenAIServing):

1203 final_content = ''.join(pm.content

1204 for pm in previous_delta_messages

1205 if pm.content is not None)

1204 - yield _send_event(

1206 + yield _increment_sequence_number_and_return(

1207 openai_responses_types.ResponseTextDoneEvent(

1208 type="response.output_text.done",

1209 sequence_number=-1,

@@ -1217,7 +1219,7 @@ class OpenAIServingResponses(OpenAIServing):

1219 type="output_text",

1220 annotations=[],

1221 )

1220 - yield _send_event(

1222 + yield _increment_sequence_number_and_return(

1223 openai_responses_types.ResponseContentPartDoneEvent(

1224 type="response.content_part.done",

1225 sequence_number=-1,

@@ -1237,7 +1239,7 @@ class OpenAIServingResponses(OpenAIServing):

1239 id=current_item_id,

1240 summary=[],

1241 )

1240 - yield _send_event(

1242 + yield _increment_sequence_number_and_return(

1243 ResponseOutputItemDoneEvent(

1244 type="response.output_item.done",

1245 sequence_number=-1,

@@ -1255,8 +1257,9 @@ class OpenAIServingResponses(OpenAIServing):

1257 tokenizer: AnyTokenizer,

1258 request_metadata: RequestResponseMetadata,

1259 created_time: int,

1258 - _send_event: Callable[[BaseModel], str],

1259 - ) -> AsyncGenerator[str, None]:

1260 + _increment_sequence_number_and_return: Callable[[BaseModel],

1261 + BaseModel],

1262 + ) -> AsyncGenerator[BaseModel, None]:

1263 current_content_index = 0 # FIXME: this number is never changed

1264 current_output_index = 0

1265 current_item_id = "" # FIXME: this number is never changed

@@ -1288,7 +1291,7 @@ class OpenAIServingResponses(OpenAIServing):

1291 id=current_item_id,

1292 summary=[],

1293 )

1291 - yield _send_event(

1294 + yield _increment_sequence_number_and_return(

1295 ResponseReasoningTextDoneEvent(

1296 type="response.reasoning_text.done",

1297 item_id=current_item_id,

@@ -1297,7 +1300,7 @@ class OpenAIServingResponses(OpenAIServing):

1300 content_index=current_content_index,

1301 text=previous_item.content[0].text,

1302 ))

1300 - yield _send_event(

1303 + yield _increment_sequence_number_and_return(

1304 ResponseOutputItemDoneEvent(

1305 type="response.output_item.done",

1306 sequence_number=-1,

@@ -1310,7 +1313,7 @@ class OpenAIServingResponses(OpenAIServing):

1313 text=previous_item.content[0].text,

1314 annotations=[],

1315 )

1313 - yield _send_event(

1316 + yield _increment_sequence_number_and_return(

1317 openai_responses_types.ResponseTextDoneEvent(

1318 type="response.output_text.done",

1319 sequence_number=-1,

@@ -1320,7 +1323,7 @@ class OpenAIServingResponses(OpenAIServing):

1323 logprobs=[],

1324 item_id=current_item_id,

1325 ))

1323 - yield _send_event(

1326 + yield _increment_sequence_number_and_return(

1327 openai_responses_types.

1328 ResponseContentPartDoneEvent(

1329 type="response.content_part.done",

@@ -1330,7 +1333,7 @@ class OpenAIServingResponses(OpenAIServing):

1333 content_index=current_content_index,

1334 part=text_content,

1335 ))

1333 - yield _send_event(

1336 + yield _increment_sequence_number_and_return(

1337 openai_responses_types.ResponseOutputItemDoneEvent(

1338 type="response.output_item.done",

1339 sequence_number=-1,

@@ -1344,12 +1347,13 @@ class OpenAIServingResponses(OpenAIServing):

1347 ),

1348 ))

1349

1350 + # stream the output of a harmony message

1351 if ctx.parser.last_content_delta:

1352 if (ctx.parser.current_channel == "final"

1353 and ctx.parser.current_recipient is None):

1354 if not sent_output_item_added:

1355 sent_output_item_added = True

1352 - yield _send_event(

1356 + yield _increment_sequence_number_and_return(

1357 openai_responses_types.

1358 ResponseOutputItemAddedEvent(

1359 type="response.output_item.added",

@@ -1364,7 +1368,7 @@ class OpenAIServingResponses(OpenAIServing):

1368 status="in_progress",

1369 ),

1370 ))

1367 - yield _send_event(

1371 + yield _increment_sequence_number_and_return(

1372 openai_responses_types.

1373 ResponseContentPartAddedEvent(

1374 type="response.content_part.added",

@@ -1379,7 +1383,7 @@ class OpenAIServingResponses(OpenAIServing):

1383 logprobs=[],

1384 ),

1385 ))

1382 - yield _send_event(

1386 + yield _increment_sequence_number_and_return(

1387 openai_responses_types.ResponseTextDeltaEvent(

1388 type="response.output_text.delta",

1389 sequence_number=-1,

@@ -1394,7 +1398,7 @@ class OpenAIServingResponses(OpenAIServing):

1398 and ctx.parser.current_recipient is None):

1399 if not sent_output_item_added:

1400 sent_output_item_added = True

1397 - yield _send_event(

1401 + yield _increment_sequence_number_and_return(

1402 openai_responses_types.

1403 ResponseOutputItemAddedEvent(

1404 type="response.output_item.added",

@@ -1408,7 +1412,7 @@ class OpenAIServingResponses(OpenAIServing):

1412 status="in_progress",

1413 ),

1414 ))

1411 - yield _send_event(

1415 + yield _increment_sequence_number_and_return(

1416 openai_responses_types.

1417 ResponseContentPartAddedEvent(

1418 type="response.content_part.added",

@@ -1423,7 +1427,7 @@ class OpenAIServingResponses(OpenAIServing):

1427 logprobs=[],

1428 ),

1429 ))

1426 - yield _send_event(

1430 + yield _increment_sequence_number_and_return(

1431 ResponseReasoningTextDeltaEvent(

1432 type="response.reasoning_text.delta",

1433 item_id=current_item_id,

@@ -1440,7 +1444,7 @@ class OpenAIServingResponses(OpenAIServing):

1444 ) and ctx.parser.current_recipient == "python":

1445 if not sent_output_item_added:

1446 sent_output_item_added = True

1443 - yield _send_event(

1447 + yield _increment_sequence_number_and_return(

1448 openai_responses_types.

1449 ResponseOutputItemAddedEvent(

1450 type="response.output_item.added",

@@ -1456,7 +1460,7 @@ class OpenAIServingResponses(OpenAIServing):

1460 status="in_progress",

1461 ),

1462 ))

1459 - yield _send_event(

1463 + yield _increment_sequence_number_and_return(

1464 openai_responses_types.

1465 ResponseCodeInterpreterCallInProgressEvent(

1466 type=

@@ -1465,7 +1469,7 @@ class OpenAIServingResponses(OpenAIServing):

1469 output_index=current_output_index,

1470 item_id=current_item_id,

1471 ))

1468 - yield _send_event(

1472 + yield _increment_sequence_number_and_return(

1473 openai_responses_types.

1474 ResponseCodeInterpreterCallCodeDeltaEvent(

1475 type="response.code_interpreter_call_code.delta",

@@ -1474,6 +1478,8 @@ class OpenAIServingResponses(OpenAIServing):

1478 item_id=current_item_id,

1479 delta=ctx.parser.last_content_delta,

1480 ))

1481 +

1482 + # stream tool call outputs

1483 if ctx.is_assistant_action_turn() and len(ctx.parser.messages) > 0:

1484 previous_item = ctx.parser.messages[-1]

1485 if (self.tool_server is not None

@@ -1510,7 +1516,7 @@ class OpenAIServingResponses(OpenAIServing):

1516 raise ValueError(

1517 f"Unknown function name: {function_name}")

1518

1513 - yield _send_event(

1519 + yield _increment_sequence_number_and_return(

1520 openai_responses_types.ResponseOutputItemAddedEvent(

1521 type="response.output_item.added",

1522 sequence_number=-1,

@@ -1525,7 +1531,7 @@ class OpenAIServingResponses(OpenAIServing):

1531 status="in_progress",

1532 ),

1533 ))

1528 - yield _send_event(

1534 + yield _increment_sequence_number_and_return(

1535 openai_responses_types.

1536 ResponseWebSearchCallInProgressEvent(

1537 type="response.web_search_call.in_progress",

@@ -1533,7 +1539,7 @@ class OpenAIServingResponses(OpenAIServing):

1539 output_index=current_output_index,

1540 item_id=current_item_id,

1541 ))

1536 - yield _send_event(

1542 + yield _increment_sequence_number_and_return(

1543 openai_responses_types.

1544 ResponseWebSearchCallSearchingEvent(

1545 type="response.web_search_call.searching",

@@ -1543,7 +1549,7 @@ class OpenAIServingResponses(OpenAIServing):

1549 ))

1550

1551 # enqueue

1546 - yield _send_event(

1552 + yield _increment_sequence_number_and_return(

1553 openai_responses_types.

1554 ResponseWebSearchCallCompletedEvent(

1555 type="response.web_search_call.completed",

@@ -1551,7 +1557,7 @@ class OpenAIServingResponses(OpenAIServing):

1557 output_index=current_output_index,

1558 item_id=current_item_id,

1559 ))

1554 - yield _send_event(

1560 + yield _increment_sequence_number_and_return(

1561 openai_responses_types.ResponseOutputItemDoneEvent(

1562 type="response.output_item.done",

1563 sequence_number=-1,

@@ -1569,7 +1575,7 @@ class OpenAIServingResponses(OpenAIServing):

1575 and self.tool_server.has_tool("python")

1576 and previous_item.recipient is not None

1577 and previous_item.recipient.startswith("python")):

1572 - yield _send_event(

1578 + yield _increment_sequence_number_and_return(

1579 openai_responses_types.

1580 ResponseCodeInterpreterCallCodeDoneEvent(

1581 type="response.code_interpreter_call_code.done",

@@ -1578,7 +1584,7 @@ class OpenAIServingResponses(OpenAIServing):

1584 item_id=current_item_id,

1585 code=previous_item.content[0].text,

1586 ))

1581 - yield _send_event(

1587 + yield _increment_sequence_number_and_return(

1588 openai_responses_types.

1589 ResponseCodeInterpreterCallInterpretingEvent(

1590 type="response.code_interpreter_call.interpreting",

@@ -1586,7 +1592,7 @@ class OpenAIServingResponses(OpenAIServing):

1592 output_index=current_output_index,

1593 item_id=current_item_id,

1594 ))

1589 - yield _send_event(

1595 + yield _increment_sequence_number_and_return(

1596 openai_responses_types.

1597 ResponseCodeInterpreterCallCompletedEvent(

1598 type="response.code_interpreter_call.completed",

@@ -1594,7 +1600,7 @@ class OpenAIServingResponses(OpenAIServing):

1600 output_index=current_output_index,

1601 item_id=current_item_id,

1602 ))

1597 - yield _send_event(

1603 + yield _increment_sequence_number_and_return(

1604 openai_responses_types.ResponseOutputItemDoneEvent(

1605 type="response.output_item.done",

1606 sequence_number=-1,

@@ -1621,7 +1627,7 @@ class OpenAIServingResponses(OpenAIServing):

1627 tokenizer: AnyTokenizer,

1628 request_metadata: RequestResponseMetadata,

1629 created_time: Optional[int] = None,

1624 - ) -> AsyncGenerator[str, None]:

1630 + ) -> AsyncGenerator[BaseModel, None]:

1631 # TODO:

1632 # 1. Handle disconnect

1633

@@ -1629,16 +1635,15 @@ class OpenAIServingResponses(OpenAIServing):

1635

1636 sequence_number = 0

1637

1632 - def _send_event(event: BaseModel):

1638 + T = TypeVar("T", bound=BaseModel)

1639 +

1640 + def _increment_sequence_number_and_return(event: T) -> T:

1641 nonlocal sequence_number

1642 # Set sequence_number if the event has this attribute

1643 if hasattr(event, 'sequence_number'):

1644 event.sequence_number = sequence_number

1645 sequence_number += 1

1638 - # Get event type from the event's type field if it exists

1639 - event_type = getattr(event, 'type', 'unknown')

1640 - return (f"event: {event_type}\n"

1641 - f"data: {event.model_dump_json(indent=None)}\n\n")

1646 + return event

1647

1648 async with AsyncExitStack() as exit_stack:

1649 processer = None

@@ -1658,24 +1663,23 @@ class OpenAIServingResponses(OpenAIServing):

1663 status="in_progress",

1664 usage=None,

1665 ).model_dump()

1661 - yield _send_event(

1666 + yield _increment_sequence_number_and_return(

1667 ResponseCreatedEvent(

1668 type="response.created",

1669 sequence_number=-1,

1670 response=initial_response,

1671 ))

1667 - yield _send_event(

1672 + yield _increment_sequence_number_and_return(

1673 ResponseInProgressEvent(

1674 type="response.in_progress",

1675 sequence_number=-1,

1676 response=initial_response,

1677 ))

1678

1674 - async for event_data in processer(request, sampling_params,

1675 - result_generator, context,

1676 - model_name, tokenizer,

1677 - request_metadata, created_time,

1678 - _send_event):

1679 + async for event_data in processer(

1680 + request, sampling_params, result_generator, context,

1681 + model_name, tokenizer, request_metadata, created_time,

1682 + _increment_sequence_number_and_return):

1683 yield event_data

1684

1685 async def empty_async_generator():

@@ -1694,7 +1698,7 @@ class OpenAIServingResponses(OpenAIServing):

1698 request_metadata,

1699 created_time=created_time,

1700 )

1697 - yield _send_event(

1701 + yield _increment_sequence_number_and_return(

1702 openai_responses_types.ResponseCompletedEvent(

1703 type="response.completed",

1704 sequence_number=-1,