@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bff2e5f1d6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[gpt-oss][2] fix types for streaming (#24556)

Signed-off-by: Andrew Xia <axia@meta.com>

Andrew Xia committed Sep 17, 2025 at 15:04 UTC bff2e5f1d6201c817d29f49581e6c15724a4e186
3 files changed +104 -96
vllm/entrypoints/openai/api_server.py
+5 -4
@@ -27,7 +27,6 @@ from fastapi import APIRouter, Depends, FastAPI, Form, HTTPException, Request
27 from fastapi.exceptions import RequestValidationError
28 from fastapi.middleware.cors import CORSMiddleware
29 from fastapi.responses import JSONResponse, Response, StreamingResponse
30 -from openai import BaseModel
30 from prometheus_client import make_asgi_app
31 from prometheus_fastapi_instrumentator import Instrumentator
32 from starlette.concurrency import iterate_in_threadpool
@@ -67,7 +66,9 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
66 RerankRequest, RerankResponse,
67 ResponsesRequest,
68 ResponsesResponse, ScoreRequest,
70 - ScoreResponse, TokenizeRequest,
69 + ScoreResponse,
70 + StreamingResponsesResponse,
71 + TokenizeRequest,
72 TokenizeResponse,
73 TranscriptionRequest,
74 TranscriptionResponse,
@@ -481,8 +482,8 @@ async def show_version():
482
483
484 async def _convert_stream_to_sse_events(
484 - generator: AsyncGenerator[BaseModel,
485 - None]) -> AsyncGenerator[str, None]:
485 + generator: AsyncGenerator[StreamingResponsesResponse, None]
486 +) -> AsyncGenerator[str, None]:
487 """Convert the generator to a stream of events in SSE format"""
488 async for event in generator:
489 event_type = getattr(event, 'type', 'unknown')
vllm/entrypoints/openai/protocol.py
+33 -4
@@ -18,10 +18,19 @@ from openai.types.chat.chat_completion_audio import (
18 from openai.types.chat.chat_completion_message import (
19 Annotation as OpenAIAnnotation)
20 # yapf: enable
21 -from openai.types.responses import (ResponseFunctionToolCall,
22 - ResponseInputItemParam, ResponseOutputItem,
23 - ResponsePrompt, ResponseReasoningItem,
24 - ResponseStatus)
21 +from openai.types.responses import (
22 + ResponseCodeInterpreterCallCodeDeltaEvent,
23 + ResponseCodeInterpreterCallCodeDoneEvent,
24 + ResponseCodeInterpreterCallCompletedEvent,
25 + ResponseCodeInterpreterCallInProgressEvent,
26 + ResponseCodeInterpreterCallInterpretingEvent, ResponseCompletedEvent,
27 + ResponseContentPartAddedEvent, ResponseContentPartDoneEvent,
28 + ResponseCreatedEvent, ResponseFunctionToolCall, ResponseInProgressEvent,
29 + ResponseInputItemParam, ResponseOutputItem, ResponseOutputItemAddedEvent,
30 + ResponseOutputItemDoneEvent, ResponsePrompt, ResponseReasoningItem,
31 + ResponseReasoningTextDeltaEvent, ResponseReasoningTextDoneEvent,
32 + ResponseStatus, ResponseWebSearchCallCompletedEvent,
33 + ResponseWebSearchCallInProgressEvent, ResponseWebSearchCallSearchingEvent)
34
35 # Backward compatibility for OpenAI client versions
36 try: # For older openai versions (< 1.100.0)
@@ -251,6 +260,26 @@ ResponseInputOutputItem: TypeAlias = Union[ResponseInputItemParam,
260 ResponseReasoningItem,
261 ResponseFunctionToolCall]
262
263 +StreamingResponsesResponse: TypeAlias = Union[
264 + ResponseCreatedEvent,
265 + ResponseInProgressEvent,
266 + ResponseCompletedEvent,
267 + ResponseOutputItemAddedEvent,
268 + ResponseOutputItemDoneEvent,
269 + ResponseContentPartAddedEvent,
270 + ResponseContentPartDoneEvent,
271 + ResponseReasoningTextDeltaEvent,
272 + ResponseReasoningTextDoneEvent,
273 + ResponseCodeInterpreterCallInProgressEvent,
274 + ResponseCodeInterpreterCallCodeDeltaEvent,
275 + ResponseWebSearchCallInProgressEvent,
276 + ResponseWebSearchCallSearchingEvent,
277 + ResponseWebSearchCallCompletedEvent,
278 + ResponseCodeInterpreterCallCodeDoneEvent,
279 + ResponseCodeInterpreterCallInterpretingEvent,
280 + ResponseCodeInterpreterCallCompletedEvent,
281 +]
282 +
283
284 class ResponsesRequest(OpenAIBaseModel):
285 # Ordered by official OpenAI API documentation
vllm/entrypoints/openai/serving_responses.py
+66 -88
@@ -10,24 +10,28 @@ from collections.abc import AsyncGenerator, AsyncIterator, Sequence
10 from contextlib import AsyncExitStack
11 from copy import copy
12 from http import HTTPStatus
13 -from typing import Callable, Final, Optional, TypeVar, Union
13 +from typing import Callable, Final, Optional, Union
14
15 import jinja2
16 -import openai.types.responses as openai_responses_types
16 from fastapi import Request
18 -from openai import BaseModel
17 # yapf conflicts with isort for this block
18 # yapf: disable
21 -from openai.types.responses import (ResponseCreatedEvent,
22 - ResponseFunctionToolCall,
23 - ResponseInProgressEvent,
24 - ResponseOutputItem,
25 - ResponseOutputItemDoneEvent,
26 - ResponseOutputMessage, ResponseOutputText,
27 - ResponseReasoningItem,
28 - ResponseReasoningTextDeltaEvent,
29 - ResponseReasoningTextDoneEvent,
30 - ResponseStatus, response_text_delta_event)
19 +from openai.types.responses import (
20 + ResponseCodeInterpreterCallCodeDeltaEvent,
21 + ResponseCodeInterpreterCallCodeDoneEvent,
22 + ResponseCodeInterpreterCallCompletedEvent,
23 + ResponseCodeInterpreterCallInProgressEvent,
24 + ResponseCodeInterpreterCallInterpretingEvent,
25 + ResponseCodeInterpreterToolCallParam, ResponseCompletedEvent,
26 + ResponseContentPartAddedEvent, ResponseContentPartDoneEvent,
27 + ResponseCreatedEvent, ResponseFunctionToolCall, ResponseFunctionWebSearch,
28 + ResponseInProgressEvent, ResponseOutputItem, ResponseOutputItemAddedEvent,
29 + ResponseOutputItemDoneEvent, ResponseOutputMessage, ResponseOutputText,
30 + ResponseReasoningItem, ResponseReasoningTextDeltaEvent,
31 + ResponseReasoningTextDoneEvent, ResponseStatus, ResponseTextDeltaEvent,
32 + ResponseTextDoneEvent, ResponseWebSearchCallCompletedEvent,
33 + ResponseWebSearchCallInProgressEvent, ResponseWebSearchCallSearchingEvent,
34 + response_function_web_search, response_text_delta_event)
35 from openai.types.responses.response_output_text import (Logprob,
36 LogprobTopLogprob)
37 # yapf: enable
@@ -55,7 +59,8 @@ from vllm.entrypoints.openai.protocol import (DeltaMessage, ErrorResponse,
59 OutputTokensDetails,
60 RequestResponseMetadata,
61 ResponsesRequest,
58 - ResponsesResponse, ResponseUsage)
62 + ResponsesResponse, ResponseUsage,
63 + StreamingResponsesResponse)
64 # yapf: enable
65 from vllm.entrypoints.openai.serving_engine import OpenAIServing
66 from vllm.entrypoints.openai.serving_models import OpenAIServingModels
@@ -175,7 +180,7 @@ class OpenAIServingResponses(OpenAIServing):
180 # HACK(wuhang): This is a hack. We should use a better store.
181 # FIXME: If enable_store=True, this may cause a memory leak since we
182 # never remove events from the store.
178 - self.event_store: dict[str, tuple[deque[BaseModel],
183 + self.event_store: dict[str, tuple[deque[StreamingResponsesResponse],
184 asyncio.Event]] = {}
185
186 self.background_tasks: dict[str, asyncio.Task] = {}
@@ -186,8 +191,8 @@ class OpenAIServingResponses(OpenAIServing):
191 self,
192 request: ResponsesRequest,
193 raw_request: Optional[Request] = None,
189 - ) -> Union[AsyncGenerator[BaseModel, None], ResponsesResponse,
190 - ErrorResponse]:
194 + ) -> Union[AsyncGenerator[StreamingResponsesResponse, None],
195 + ResponsesResponse, ErrorResponse]:
196 error_check_ret = await self._check_model(request)
197 if error_check_ret is not None:
198 logger.error("Error with model %s", error_check_ret)
@@ -814,7 +819,7 @@ class OpenAIServingResponses(OpenAIServing):
819 *args,
820 **kwargs,
821 ):
817 - event_deque: deque[BaseModel] = deque()
822 + event_deque: deque[StreamingResponsesResponse] = deque()
823 new_event_signal = asyncio.Event()
824 self.event_store[request.request_id] = (event_deque, new_event_signal)
825 response = None
@@ -867,7 +872,7 @@ class OpenAIServingResponses(OpenAIServing):
872 self,
873 response_id: str,
874 starting_after: Optional[int] = None,
870 - ) -> AsyncGenerator[BaseModel, None]:
875 + ) -> AsyncGenerator[StreamingResponsesResponse, None]:
876 if response_id not in self.event_store:
877 raise ValueError(f"Unknown response_id: {response_id}")
878
@@ -893,8 +898,8 @@ class OpenAIServingResponses(OpenAIServing):
898 response_id: str,
899 starting_after: Optional[int],
900 stream: Optional[bool],
896 - ) -> Union[ErrorResponse, ResponsesResponse, AsyncGenerator[BaseModel,
897 - None]]:
901 + ) -> Union[ErrorResponse, ResponsesResponse, AsyncGenerator[
902 + StreamingResponsesResponse, None]]:
903 if not response_id.startswith("resp_"):
904 return self._make_invalid_id_error(response_id)
905
@@ -977,9 +982,9 @@ class OpenAIServingResponses(OpenAIServing):
982 tokenizer: AnyTokenizer,
983 request_metadata: RequestResponseMetadata,
984 created_time: int,
980 - _increment_sequence_number_and_return: Callable[[BaseModel],
981 - BaseModel],
982 - ) -> AsyncGenerator[BaseModel, None]:
985 + _increment_sequence_number_and_return: Callable[
986 + [StreamingResponsesResponse], StreamingResponsesResponse],
987 + ) -> AsyncGenerator[StreamingResponsesResponse, None]:
988 current_content_index = 0
989 current_output_index = 0
990 current_item_id = ""
@@ -1017,13 +1022,11 @@ class OpenAIServingResponses(OpenAIServing):
1022 current_item_id = str(uuid.uuid4())
1023 if delta_message.reasoning_content:
1024 yield _increment_sequence_number_and_return(
1020 - openai_responses_types.
1025 ResponseOutputItemAddedEvent(
1026 type="response.output_item.added",
1027 sequence_number=-1,
1028 output_index=current_output_index,
1025 - item=openai_responses_types.
1026 - ResponseReasoningItem(
1029 + item=ResponseReasoningItem(
1030 type="reasoning",
1031 id=current_item_id,
1032 summary=[],
@@ -1032,13 +1035,11 @@ class OpenAIServingResponses(OpenAIServing):
1035 ))
1036 else:
1037 yield _increment_sequence_number_and_return(
1035 - openai_responses_types.
1038 ResponseOutputItemAddedEvent(
1039 type="response.output_item.added",
1040 sequence_number=-1,
1041 output_index=current_output_index,
1040 - item=openai_responses_types.
1041 - ResponseOutputMessage(
1042 + item=ResponseOutputMessage(
1043 id=current_item_id,
1044 type="message",
1045 role="assistant",
@@ -1047,13 +1048,13 @@ class OpenAIServingResponses(OpenAIServing):
1048 ),
1049 ))
1050 yield _increment_sequence_number_and_return(
1050 - openai_responses_types.ResponseContentPartAddedEvent(
1051 + ResponseContentPartAddedEvent(
1052 type="response.content_part.added",
1053 sequence_number=-1,
1054 output_index=current_output_index,
1055 item_id=current_item_id,
1056 content_index=current_content_index,
1056 - part=openai_responses_types.ResponseOutputText(
1057 + part=ResponseOutputText(
1058 type="output_text",
1059 text="",
1060 annotations=[],
@@ -1104,11 +1105,11 @@ class OpenAIServingResponses(OpenAIServing):
1105 item=reasoning_item,
1106 ))
1107 yield _increment_sequence_number_and_return(
1107 - openai_responses_types.ResponseOutputItemAddedEvent(
1108 + ResponseOutputItemAddedEvent(
1109 type="response.output_item.added",
1110 sequence_number=-1,
1111 output_index=current_output_index,
1111 - item=openai_responses_types.ResponseOutputMessage(
1112 + item=ResponseOutputMessage(
1113 id=current_item_id,
1114 type="message",
1115 role="assistant",
@@ -1119,13 +1120,13 @@ class OpenAIServingResponses(OpenAIServing):
1120 current_output_index += 1
1121 current_item_id = str(uuid.uuid4())
1122 yield _increment_sequence_number_and_return(
1122 - openai_responses_types.ResponseContentPartAddedEvent(
1123 + ResponseContentPartAddedEvent(
1124 type="response.content_part.added",
1125 sequence_number=-1,
1126 output_index=current_output_index,
1127 item_id=current_item_id,
1128 content_index=current_content_index,
1128 - part=openai_responses_types.ResponseOutputText(
1129 + part=ResponseOutputText(
1130 type="output_text",
1131 text="",
1132 annotations=[],
@@ -1148,7 +1149,7 @@ class OpenAIServingResponses(OpenAIServing):
1149 ))
1150 elif delta_message.content is not None:
1151 yield _increment_sequence_number_and_return(
1151 - openai_responses_types.ResponseTextDeltaEvent(
1152 + ResponseTextDeltaEvent(
1153 type="response.output_text.delta",
1154 sequence_number=-1,
1155 content_index=current_content_index,
@@ -1204,7 +1205,7 @@ class OpenAIServingResponses(OpenAIServing):
1205 for pm in previous_delta_messages
1206 if pm.content is not None)
1207 yield _increment_sequence_number_and_return(
1207 - openai_responses_types.ResponseTextDoneEvent(
1208 + ResponseTextDoneEvent(
1209 type="response.output_text.done",
1210 sequence_number=-1,
1211 output_index=current_output_index,
@@ -1220,7 +1221,7 @@ class OpenAIServingResponses(OpenAIServing):
1221 annotations=[],
1222 )
1223 yield _increment_sequence_number_and_return(
1223 - openai_responses_types.ResponseContentPartDoneEvent(
1224 + ResponseContentPartDoneEvent(
1225 type="response.content_part.done",
1226 sequence_number=-1,
1227 item_id=current_item_id,
@@ -1257,9 +1258,9 @@ class OpenAIServingResponses(OpenAIServing):
1258 tokenizer: AnyTokenizer,
1259 request_metadata: RequestResponseMetadata,
1260 created_time: int,
1260 - _increment_sequence_number_and_return: Callable[[BaseModel],
1261 - BaseModel],
1262 - ) -> AsyncGenerator[BaseModel, None]:
1261 + _increment_sequence_number_and_return: Callable[
1262 + [StreamingResponsesResponse], StreamingResponsesResponse],
1263 + ) -> AsyncGenerator[StreamingResponsesResponse, None]:
1264 current_content_index = -1
1265 current_output_index = 0
1266 current_item_id: str = ""
@@ -1314,7 +1315,7 @@ class OpenAIServingResponses(OpenAIServing):
1315 annotations=[],
1316 )
1317 yield _increment_sequence_number_and_return(
1317 - openai_responses_types.ResponseTextDoneEvent(
1318 + ResponseTextDoneEvent(
1319 type="response.output_text.done",
1320 sequence_number=-1,
1321 output_index=current_output_index,
@@ -1324,7 +1325,6 @@ class OpenAIServingResponses(OpenAIServing):
1325 item_id=current_item_id,
1326 ))
1327 yield _increment_sequence_number_and_return(
1327 - openai_responses_types.
1328 ResponseContentPartDoneEvent(
1329 type="response.content_part.done",
1330 sequence_number=-1,
@@ -1334,7 +1334,7 @@ class OpenAIServingResponses(OpenAIServing):
1334 part=text_content,
1335 ))
1336 yield _increment_sequence_number_and_return(
1337 - openai_responses_types.ResponseOutputItemDoneEvent(
1337 + ResponseOutputItemDoneEvent(
1338 type="response.output_item.done",
1339 sequence_number=-1,
1340 output_index=current_output_index,
@@ -1355,13 +1355,11 @@ class OpenAIServingResponses(OpenAIServing):
1355 sent_output_item_added = True
1356 current_item_id = f"msg_{random_uuid()}"
1357 yield _increment_sequence_number_and_return(
1358 - openai_responses_types.
1358 ResponseOutputItemAddedEvent(
1359 type="response.output_item.added",
1360 sequence_number=-1,
1361 output_index=current_output_index,
1363 - item=openai_responses_types.
1364 - ResponseOutputMessage(
1362 + item=ResponseOutputMessage(
1363 id=current_item_id,
1364 type="message",
1365 role="assistant",
@@ -1371,14 +1369,13 @@ class OpenAIServingResponses(OpenAIServing):
1369 ))
1370 current_content_index += 1
1371 yield _increment_sequence_number_and_return(
1374 - openai_responses_types.
1372 ResponseContentPartAddedEvent(
1373 type="response.content_part.added",
1374 sequence_number=-1,
1375 output_index=current_output_index,
1376 item_id=current_item_id,
1377 content_index=current_content_index,
1381 - part=openai_responses_types.ResponseOutputText(
1378 + part=ResponseOutputText(
1379 type="output_text",
1380 text="",
1381 annotations=[],
@@ -1386,7 +1383,7 @@ class OpenAIServingResponses(OpenAIServing):
1383 ),
1384 ))
1385 yield _increment_sequence_number_and_return(
1389 - openai_responses_types.ResponseTextDeltaEvent(
1386 + ResponseTextDeltaEvent(
1387 type="response.output_text.delta",
1388 sequence_number=-1,
1389 content_index=current_content_index,
@@ -1402,13 +1399,11 @@ class OpenAIServingResponses(OpenAIServing):
1399 sent_output_item_added = True
1400 current_item_id = f"msg_{random_uuid()}"
1401 yield _increment_sequence_number_and_return(
1405 - openai_responses_types.
1402 ResponseOutputItemAddedEvent(
1403 type="response.output_item.added",
1404 sequence_number=-1,
1405 output_index=current_output_index,
1410 - item=openai_responses_types.
1411 - ResponseReasoningItem(
1406 + item=ResponseReasoningItem(
1407 type="reasoning",
1408 id=current_item_id,
1409 summary=[],
@@ -1417,14 +1412,13 @@ class OpenAIServingResponses(OpenAIServing):
1412 ))
1413 current_content_index += 1
1414 yield _increment_sequence_number_and_return(
1420 - openai_responses_types.
1415 ResponseContentPartAddedEvent(
1416 type="response.content_part.added",
1417 sequence_number=-1,
1418 output_index=current_output_index,
1419 item_id=current_item_id,
1420 content_index=current_content_index,
1427 - part=openai_responses_types.ResponseOutputText(
1421 + part=ResponseOutputText(
1422 type="output_text",
1423 text="",
1424 annotations=[],
@@ -1450,13 +1444,11 @@ class OpenAIServingResponses(OpenAIServing):
1444 sent_output_item_added = True
1445 current_item_id = f"tool_{random_uuid()}"
1446 yield _increment_sequence_number_and_return(
1453 - openai_responses_types.
1447 ResponseOutputItemAddedEvent(
1448 type="response.output_item.added",
1449 sequence_number=-1,
1450 output_index=current_output_index,
1458 - item=openai_responses_types.
1459 - ResponseCodeInterpreterToolCallParam(
1451 + item=ResponseCodeInterpreterToolCallParam(
1452 type="code_interpreter_call",
1453 id=current_item_id,
1454 code=None,
@@ -1466,7 +1458,6 @@ class OpenAIServingResponses(OpenAIServing):
1458 ),
1459 ))
1460 yield _increment_sequence_number_and_return(
1469 - openai_responses_types.
1461 ResponseCodeInterpreterCallInProgressEvent(
1462 type=
1463 "response.code_interpreter_call.in_progress",
@@ -1475,7 +1466,6 @@ class OpenAIServingResponses(OpenAIServing):
1466 item_id=current_item_id,
1467 ))
1468 yield _increment_sequence_number_and_return(
1478 - openai_responses_types.
1469 ResponseCodeInterpreterCallCodeDeltaEvent(
1470 type="response.code_interpreter_call_code.delta",
1471 sequence_number=-1,
@@ -1495,14 +1485,12 @@ class OpenAIServingResponses(OpenAIServing):
1485 action = None
1486 parsed_args = json.loads(previous_item.content[0].text)
1487 if function_name == "search":
1498 - action = (openai_responses_types.
1499 - response_function_web_search.ActionSearch(
1500 - type="search",
1501 - query=parsed_args["query"],
1502 - ))
1488 + action = (response_function_web_search.ActionSearch(
1489 + type="search",
1490 + query=parsed_args["query"],
1491 + ))
1492 elif function_name == "open":
1493 action = (
1505 - openai_responses_types.
1494 response_function_web_search.ActionOpenPage(
1495 type="open_page",
1496 # TODO: translate to url
@@ -1510,7 +1498,6 @@ class OpenAIServingResponses(OpenAIServing):
1498 ))
1499 elif function_name == "find":
1500 action = (
1513 - openai_responses_types.
1501 response_function_web_search.ActionFind(
1502 type="find",
1503 pattern=parsed_args["pattern"],
@@ -1523,12 +1510,11 @@ class OpenAIServingResponses(OpenAIServing):
1510
1511 current_item_id = f"tool_{random_uuid()}"
1512 yield _increment_sequence_number_and_return(
1526 - openai_responses_types.ResponseOutputItemAddedEvent(
1513 + ResponseOutputItemAddedEvent(
1514 type="response.output_item.added",
1515 sequence_number=-1,
1516 output_index=current_output_index,
1530 - item=openai_responses_types.
1531 - response_function_web_search.
1517 + item=response_function_web_search.
1518 ResponseFunctionWebSearch(
1519 # TODO: generate a unique id for web search call
1520 type="web_search_call",
@@ -1538,7 +1524,6 @@ class OpenAIServingResponses(OpenAIServing):
1524 ),
1525 ))
1526 yield _increment_sequence_number_and_return(
1541 - openai_responses_types.
1527 ResponseWebSearchCallInProgressEvent(
1528 type="response.web_search_call.in_progress",
1529 sequence_number=-1,
@@ -1546,7 +1531,6 @@ class OpenAIServingResponses(OpenAIServing):
1531 item_id=current_item_id,
1532 ))
1533 yield _increment_sequence_number_and_return(
1549 - openai_responses_types.
1534 ResponseWebSearchCallSearchingEvent(
1535 type="response.web_search_call.searching",
1536 sequence_number=-1,
@@ -1556,7 +1540,6 @@ class OpenAIServingResponses(OpenAIServing):
1540
1541 # enqueue
1542 yield _increment_sequence_number_and_return(
1559 - openai_responses_types.
1543 ResponseWebSearchCallCompletedEvent(
1544 type="response.web_search_call.completed",
1545 sequence_number=-1,
@@ -1564,12 +1547,11 @@ class OpenAIServingResponses(OpenAIServing):
1547 item_id=current_item_id,
1548 ))
1549 yield _increment_sequence_number_and_return(
1567 - openai_responses_types.ResponseOutputItemDoneEvent(
1550 + ResponseOutputItemDoneEvent(
1551 type="response.output_item.done",
1552 sequence_number=-1,
1553 output_index=current_output_index,
1571 - item=openai_responses_types.
1572 - ResponseFunctionWebSearch(
1554 + item=ResponseFunctionWebSearch(
1555 type="web_search_call",
1556 id=current_item_id,
1557 action=action,
@@ -1582,7 +1564,6 @@ class OpenAIServingResponses(OpenAIServing):
1564 and previous_item.recipient is not None
1565 and previous_item.recipient.startswith("python")):
1566 yield _increment_sequence_number_and_return(
1585 - openai_responses_types.
1567 ResponseCodeInterpreterCallCodeDoneEvent(
1568 type="response.code_interpreter_call_code.done",
1569 sequence_number=-1,
@@ -1591,7 +1572,6 @@ class OpenAIServingResponses(OpenAIServing):
1572 code=previous_item.content[0].text,
1573 ))
1574 yield _increment_sequence_number_and_return(
1594 - openai_responses_types.
1575 ResponseCodeInterpreterCallInterpretingEvent(
1576 type="response.code_interpreter_call.interpreting",
1577 sequence_number=-1,
@@ -1599,7 +1579,6 @@ class OpenAIServingResponses(OpenAIServing):
1579 item_id=current_item_id,
1580 ))
1581 yield _increment_sequence_number_and_return(
1602 - openai_responses_types.
1582 ResponseCodeInterpreterCallCompletedEvent(
1583 type="response.code_interpreter_call.completed",
1584 sequence_number=-1,
@@ -1607,12 +1586,11 @@ class OpenAIServingResponses(OpenAIServing):
1586 item_id=current_item_id,
1587 ))
1588 yield _increment_sequence_number_and_return(
1610 - openai_responses_types.ResponseOutputItemDoneEvent(
1589 + ResponseOutputItemDoneEvent(
1590 type="response.output_item.done",
1591 sequence_number=-1,
1592 output_index=current_output_index,
1614 - item=openai_responses_types.
1615 - ResponseCodeInterpreterToolCallParam(
1593 + item=ResponseCodeInterpreterToolCallParam(
1594 type="code_interpreter_call",
1595 id=current_item_id,
1596 code=previous_item.content[0].text,
@@ -1633,7 +1611,7 @@ class OpenAIServingResponses(OpenAIServing):
1611 tokenizer: AnyTokenizer,
1612 request_metadata: RequestResponseMetadata,
1613 created_time: Optional[int] = None,
1636 - ) -> AsyncGenerator[BaseModel, None]:
1614 + ) -> AsyncGenerator[StreamingResponsesResponse, None]:
1615 # TODO:
1616 # 1. Handle disconnect
1617
@@ -1641,9 +1619,9 @@ class OpenAIServingResponses(OpenAIServing):
1619
1620 sequence_number = 0
1621
1644 - T = TypeVar("T", bound=BaseModel)
1645 -
1646 - def _increment_sequence_number_and_return(event: T) -> T:
1622 + def _increment_sequence_number_and_return(
1623 + event: StreamingResponsesResponse
1624 + ) -> StreamingResponsesResponse:
1625 nonlocal sequence_number
1626 # Set sequence_number if the event has this attribute
1627 if hasattr(event, 'sequence_number'):
@@ -1705,7 +1683,7 @@ class OpenAIServingResponses(OpenAIServing):
1683 created_time=created_time,
1684 )
1685 yield _increment_sequence_number_and_return(
1708 - openai_responses_types.ResponseCompletedEvent(
1686 + ResponseCompletedEvent(
1687 type="response.completed",
1688 sequence_number=-1,
1689 response=final_response.model_dump(),