@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 462b269280
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 03:30 UTC

Implement OpenAI Responses API [1/N] (#20504)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Jul 6, 2025 at 18:32 UTC 462b26928051742c4dcac259fd0f04cc56fe581d
12 files changed +1106 -8
tests/entrypoints/openai/test_openai_schema.py
+4
@@ -95,6 +95,10 @@ def test_openapi_stateless(case: schemathesis.Case):
95 case.operation.method.upper(),
96 case.operation.path,
97 )
98 + if case.operation.path.startswith("/v1/responses"):
99 + # Skip responses API as it is meant to be stateful.
100 + return
101 +
102 timeout = {
103 # requires a longer timeout
104 ("POST", "/v1/chat/completions"):
tests/v1/entrypoints/openai/responses/__init__.py
tests/v1/entrypoints/openai/responses/conftest.py new
+32
@@ -0,0 +1,32 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import pytest
4 +import pytest_asyncio
5 +
6 +from tests.utils import RemoteOpenAIServer
7 +
8 +# Use a small reasoning model to test the responses API.
9 +MODEL_NAME = "Qwen/Qwen3-0.6B"
10 +
11 +
12 +@pytest.fixture(scope="module")
13 +def default_server_args():
14 + return [
15 + "--max-model-len",
16 + "8192",
17 + "--enforce-eager", # For faster startup.
18 + "--reasoning-parser",
19 + "deepseek_r1",
20 + ]
21 +
22 +
23 +@pytest.fixture(scope="module")
24 +def server(default_server_args):
25 + with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
26 + yield remote_server
27 +
28 +
29 +@pytest_asyncio.fixture
30 +async def client(server):
31 + async with server.get_async_client() as async_client:
32 + yield async_client
tests/v1/entrypoints/openai/responses/test_basic.py new
+75
@@ -0,0 +1,75 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import openai # use the official client for correctness check
5 +import pytest
6 +
7 +
8 +@pytest.mark.asyncio
9 +async def test_simple_input(client: openai.AsyncOpenAI):
10 + response = await client.responses.create(input="What is 13 * 24?")
11 + print(response)
12 +
13 + outputs = response.output
14 + # Whether the output contains the answer.
15 + assert outputs[-1].type == "message"
16 + assert "312" in outputs[-1].content[0].text
17 +
18 + # Whether the output contains the reasoning.
19 + assert outputs[0].type == "reasoning"
20 + assert outputs[0].text != ""
21 +
22 +
23 +@pytest.mark.asyncio
24 +async def test_instructions(client: openai.AsyncOpenAI):
25 + response = await client.responses.create(
26 + instructions="Finish the answer with QED.",
27 + input="What is 13 * 24?",
28 + )
29 + print(response)
30 +
31 + output_text = response.output[-1].content[0].text
32 + assert "312" in output_text
33 + assert "QED" in output_text
34 +
35 +
36 +@pytest.mark.asyncio
37 +async def test_chat(client: openai.AsyncOpenAI):
38 + response = await client.responses.create(input=[
39 + {
40 + "role": "system",
41 + "content": "Finish the answer with QED."
42 + },
43 + {
44 + "role": "user",
45 + "content": "What is 5 * 3?"
46 + },
47 + {
48 + "role": "assistant",
49 + "content": "15. QED."
50 + },
51 + {
52 + "role": "user",
53 + "content": "Multiply the result by 2."
54 + },
55 + ], )
56 + print(response)
57 +
58 + output_text = response.output[-1].content[0].text
59 + assert "30" in output_text
60 + assert "QED" in output_text
61 +
62 +
63 +@pytest.mark.asyncio
64 +async def test_chat_with_input_type(client: openai.AsyncOpenAI):
65 + response = await client.responses.create(input=[
66 + {
67 + "role": "user",
68 + "content": [{
69 + "type": "input_text",
70 + "text": "Hello!"
71 + }],
72 + },
73 + ], )
74 + print(response)
75 + assert response.status == "completed"
tests/v1/entrypoints/openai/responses/test_stateful.py new
+137
@@ -0,0 +1,137 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import asyncio
4 +
5 +import openai
6 +import pytest
7 +
8 +
9 +@pytest.mark.asyncio
10 +async def test_store(client: openai.AsyncOpenAI):
11 + # By default, store is True.
12 + response = await client.responses.create(input="Hello!")
13 + assert response.status == "completed"
14 +
15 + # Retrieve the response.
16 + response = await client.responses.retrieve(response.id)
17 + assert response.status == "completed"
18 +
19 + # Test store=False.
20 + response = await client.responses.create(
21 + input="Hello!",
22 + store=False,
23 + )
24 + assert response.status == "completed"
25 +
26 + # The response should not be found.
27 + with pytest.raises(openai.NotFoundError,
28 + match="Response with id .* not found."):
29 + await client.responses.retrieve(response.id)
30 +
31 +
32 +@pytest.mark.asyncio
33 +async def test_background(client: openai.AsyncOpenAI):
34 + # NOTE: This query should be easy enough for the model to answer
35 + # within the 10 seconds.
36 + response = await client.responses.create(
37 + input="Hello!",
38 + background=True,
39 + )
40 + assert response.status == "queued"
41 +
42 + max_retries = 10
43 + for _ in range(max_retries):
44 + await asyncio.sleep(1)
45 + response = await client.responses.retrieve(response.id)
46 + if response.status != "queued":
47 + break
48 + print(response)
49 +
50 + assert response.status == "completed"
51 +
52 +
53 +@pytest.mark.asyncio
54 +async def test_background_error(client: openai.AsyncOpenAI):
55 + with pytest.raises(
56 + openai.BadRequestError,
57 + match="background can only be used when `store` is true"):
58 + _ = await client.responses.create(
59 + input="What is 13 * 24?",
60 + background=True,
61 + store=False,
62 + )
63 +
64 +
65 +@pytest.mark.asyncio
66 +async def test_background_cancel(client: openai.AsyncOpenAI):
67 + response = await client.responses.create(
68 + input="Write a long story about a cat.",
69 + background=True,
70 + )
71 + assert response.status == "queued"
72 +
73 + # Cancel the response before it is completed.
74 + # FIXME: This test can be flaky.
75 + await asyncio.sleep(0.5)
76 + response = await client.responses.cancel(response.id)
77 + assert response.status == "cancelled"
78 +
79 + # Make sure the response status remains unchanged.
80 + await asyncio.sleep(5)
81 + response = await client.responses.retrieve(response.id)
82 + assert response.status == "cancelled"
83 +
84 +
85 +@pytest.mark.asyncio
86 +async def test_cancel_completed(client: openai.AsyncOpenAI):
87 + response = await client.responses.create(input="Hello")
88 + assert response.status == "completed"
89 +
90 + with pytest.raises(openai.BadRequestError,
91 + match="Cannot cancel a synchronous response."):
92 + await client.responses.cancel(response.id)
93 +
94 +
95 +@pytest.mark.asyncio
96 +async def test_previous_response_id(client: openai.AsyncOpenAI):
97 + response1 = await client.responses.create(
98 + instructions="You are tested on your ability to retrieve the correct "
99 + "information from the previous response.",
100 + input="Hello, my name is John.")
101 +
102 + response2 = await client.responses.create(
103 + input="Actually, my name is not John. My real name is Mark.",
104 + previous_response_id=response1.id,
105 + )
106 +
107 + response3 = await client.responses.create(
108 + input="What is my real name again? Answer in one word.",
109 + previous_response_id=response2.id,
110 + )
111 + print(response3)
112 + assert "Mark" in response3.output[-1].content[0].text
113 + assert "John" not in response3.output[-1].content[0].text
114 +
115 +
116 +@pytest.mark.asyncio
117 +async def test_two_responses_with_same_prev_id(client: openai.AsyncOpenAI):
118 + response1 = await client.responses.create(
119 + instructions="You are tested on your ability to retrieve the correct "
120 + "information from the previous response.",
121 + input="Hello, my name is John.")
122 +
123 + # Both response 2 and 3 use response 1 as the previous response.
124 + response2 = client.responses.create(
125 + input="Actually, my name is not John. My name is Mark.",
126 + previous_response_id=response1.id,
127 + )
128 + response3 = client.responses.create(
129 + input="What is my name again? Answer in one word.",
130 + previous_response_id=response1.id,
131 + )
132 +
133 + _ = await response2
134 + response3_result = await response3
135 + print(response3_result)
136 + assert "John" in response3_result.output[-1].content[0].text
137 + assert "Mark" not in response3_result.output[-1].content[0].text
tests/v1/entrypoints/openai/responses/test_structured_output.py new
+92
@@ -0,0 +1,92 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import json
4 +
5 +import openai
6 +import pytest
7 +from pydantic import BaseModel
8 +
9 +
10 +@pytest.mark.asyncio
11 +async def test_structured_output(client: openai.AsyncOpenAI):
12 + response = await client.responses.create(
13 + input=[
14 + {
15 + "role": "system",
16 + "content": "Extract the event information."
17 + },
18 + {
19 + "role": "user",
20 + "content":
21 + "Alice and Bob are going to a science fair on Friday.",
22 + },
23 + ],
24 + text={
25 + "format": {
26 + "type": "json_schema",
27 + "name": "calendar_event",
28 + "schema": {
29 + "type": "object",
30 + "properties": {
31 + "event_name": {
32 + "type": "string"
33 + },
34 + "date": {
35 + "type": "string"
36 + },
37 + "participants": {
38 + "type": "array",
39 + "items": {
40 + "type": "string"
41 + }
42 + },
43 + },
44 + "required": ["event_name", "date", "participants"],
45 + "additionalProperties": False,
46 + },
47 + "description": "A calendar event.",
48 + "strict": True,
49 + }
50 + },
51 + )
52 + print(response)
53 +
54 + # NOTE: The JSON schema is applied to the output text, not reasoning.
55 + output_text = response.output[-1].content[0].text
56 + event = json.loads(output_text)
57 +
58 + assert event["event_name"].lower() == "science fair"
59 + assert event["date"] == "Friday"
60 + participants = event["participants"]
61 + assert len(participants) == 2
62 + assert participants[0] == "Alice"
63 + assert participants[1] == "Bob"
64 +
65 +
66 +@pytest.mark.asyncio
67 +async def test_structured_output_with_parse(client: openai.AsyncOpenAI):
68 +
69 + class CalendarEvent(BaseModel):
70 + event_name: str
71 + date: str
72 + participants: list[str]
73 +
74 + response = await client.responses.parse(
75 + model=None,
76 + instructions="Extract the event information.",
77 + input="Alice and Bob are going to a science fair on Friday.",
78 + text_format=CalendarEvent,
79 + )
80 + print(response)
81 +
82 + # The output is successfully parsed.
83 + event = response.output_parsed
84 + assert event is not None
85 +
86 + # The output is correct.
87 + assert event.event_name.lower() == "science fair"
88 + assert event.date == "Friday"
89 + participants = event.participants
90 + assert len(participants) == 2
91 + assert participants[0] == "Alice"
92 + assert participants[1] == "Bob"
vllm/entrypoints/chat_utils.py
+3 -1
@@ -902,6 +902,8 @@ MM_PARSER_MAP: dict[
902 ] = {
903 "text":
904 lambda part: _TextParser(part).get("text", None),
905 + "input_text":
906 + lambda part: _TextParser(part).get("text", None),
907 "image_url":
908 lambda part: _ImageParser(part).get("image_url", {}).get("url", None),
909 "image_embeds":
@@ -1040,7 +1042,7 @@ def _parse_chat_message_content_part(
1042 "with empty / unparsable content.", part, part_type)
1043 return None
1044
1043 - if part_type in ("text", "refusal"):
1045 + if part_type in ("text", "input_text", "refusal"):
1046 str_content = cast(str, content)
1047 if wrap_dicts:
1048 return {'type': 'text', 'text': str_content}
vllm/entrypoints/openai/api_server.py
+89 -2
@@ -69,8 +69,9 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
69 PoolingCompletionRequest,
70 PoolingRequest, PoolingResponse,
71 RerankRequest, RerankResponse,
72 - ScoreRequest, ScoreResponse,
73 - TokenizeRequest,
72 + ResponsesRequest,
73 + ResponsesResponse, ScoreRequest,
74 + ScoreResponse, TokenizeRequest,
75 TokenizeResponse,
76 TranscriptionRequest,
77 TranscriptionResponse,
@@ -87,6 +88,7 @@ from vllm.entrypoints.openai.serving_engine import OpenAIServing
88 from vllm.entrypoints.openai.serving_models import (BaseModelPath,
89 OpenAIServingModels)
90 from vllm.entrypoints.openai.serving_pooling import OpenAIServingPooling
91 +from vllm.entrypoints.openai.serving_responses import OpenAIServingResponses
92 from vllm.entrypoints.openai.serving_score import ServingScores
93 from vllm.entrypoints.openai.serving_tokenization import (
94 OpenAIServingTokenization)
@@ -368,6 +370,10 @@ def models(request: Request) -> OpenAIServingModels:
370 return request.app.state.openai_serving_models
371
372
373 +def responses(request: Request) -> Optional[OpenAIServingResponses]:
374 + return request.app.state.openai_serving_responses
375 +
376 +
377 def chat(request: Request) -> Optional[OpenAIServingChat]:
378 return request.app.state.openai_serving_chat
379
@@ -531,6 +537,71 @@ async def show_version():
537 return JSONResponse(content=ver)
538
539
540 +@router.post("/v1/responses",
541 + dependencies=[Depends(validate_json_request)],
542 + responses={
543 + HTTPStatus.OK.value: {
544 + "content": {
545 + "text/event-stream": {}
546 + }
547 + },
548 + HTTPStatus.BAD_REQUEST.value: {
549 + "model": ErrorResponse
550 + },
551 + HTTPStatus.NOT_FOUND.value: {
552 + "model": ErrorResponse
553 + },
554 + HTTPStatus.INTERNAL_SERVER_ERROR.value: {
555 + "model": ErrorResponse
556 + },
557 + })
558 +@with_cancellation
559 +async def create_responses(request: ResponsesRequest, raw_request: Request):
560 + handler = responses(raw_request)
561 + if handler is None:
562 + return base(raw_request).create_error_response(
563 + message="The model does not support Responses API")
564 +
565 + generator = await handler.create_responses(request, raw_request)
566 +
567 + if isinstance(generator, ErrorResponse):
568 + return JSONResponse(content=generator.model_dump(),
569 + status_code=generator.code)
570 + elif isinstance(generator, ResponsesResponse):
571 + return JSONResponse(content=generator.model_dump())
572 + return StreamingResponse(content=generator, media_type="text/event-stream")
573 +
574 +
575 +@router.get("/v1/responses/{response_id}")
576 +async def retrieve_responses(response_id: str, raw_request: Request):
577 + handler = responses(raw_request)
578 + if handler is None:
579 + return base(raw_request).create_error_response(
580 + message="The model does not support Responses API")
581 +
582 + response = await handler.retrieve_responses(response_id)
583 +
584 + if isinstance(response, ErrorResponse):
585 + return JSONResponse(content=response.model_dump(),
586 + status_code=response.code)
587 + return JSONResponse(content=response.model_dump())
588 +
589 +
590 +@router.post("/v1/responses/{response_id}/cancel")
591 +async def cancel_responses(response_id: str, raw_request: Request):
592 + handler = responses(raw_request)
593 + if handler is None:
594 + return base(raw_request).create_error_response(
595 + message="The model does not support Responses API")
596 +
597 + response = await handler.cancel_responses(response_id)
598 +
599 + if isinstance(response, ErrorResponse):
600 + return JSONResponse(content=response.model_dump(),
601 + status_code=response.code)
602 + return JSONResponse(content=response.model_dump())
603 +
604 +
605 @router.post("/v1/chat/completions",
606 dependencies=[Depends(validate_json_request)],
607 responses={
@@ -1272,6 +1343,22 @@ async def init_app_state(
1343 prompt_adapters=args.prompt_adapters,
1344 )
1345 await state.openai_serving_models.init_static_loras()
1346 + state.openai_serving_responses = OpenAIServingResponses(
1347 + engine_client,
1348 + model_config,
1349 + state.openai_serving_models,
1350 + request_logger=request_logger,
1351 + chat_template=resolved_chat_template,
1352 + chat_template_content_format=args.chat_template_content_format,
1353 + return_tokens_as_token_ids=args.return_tokens_as_token_ids,
1354 + enable_auto_tools=args.enable_auto_tool_choice,
1355 + expand_tools_even_if_tool_choice_none=args.
1356 + expand_tools_even_if_tool_choice_none,
1357 + tool_parser=args.tool_call_parser,
1358 + reasoning_parser=args.reasoning_parser,
1359 + enable_prompt_tokens_details=args.enable_prompt_tokens_details,
1360 + enable_force_include_usage=args.enable_force_include_usage,
1361 + ) if model_config.runner_type == "generate" else None
1362 state.openai_serving_chat = OpenAIServingChat(
1363 engine_client,
1364 model_config,
vllm/entrypoints/openai/protocol.py
+201
@@ -11,6 +11,12 @@ from typing import Annotated, Any, ClassVar, Literal, Optional, Union
11 import regex as re
12 import torch
13 from fastapi import HTTPException, UploadFile
14 +from openai.types.responses import (ResponseInputParam, ResponseOutputItem,
15 + ResponseOutputMessage, ResponsePrompt,
16 + ResponseStatus, ResponseTextConfig)
17 +from openai.types.responses.response import ToolChoice
18 +from openai.types.responses.tool import Tool
19 +from openai.types.shared import Metadata, Reasoning
20 from pydantic import (BaseModel, ConfigDict, Field, TypeAdapter,
21 ValidationInfo, field_validator, model_validator)
22 from typing_extensions import TypeAlias
@@ -220,6 +226,124 @@ def get_logits_processors(processors: Optional[LogitsProcessors],
226 return None
227
228
229 +class ResponsesRequest(OpenAIBaseModel):
230 + # Ordered by official OpenAI API documentation
231 + # https://platform.openai.com/docs/api-reference/responses/create
232 + background: Optional[bool] = False
233 + include: Optional[list[
234 + Literal[
235 + "code_interpreter_call.outputs",
236 + "computer_call_output.output.image_url",
237 + "file_search_call.results",
238 + "message.input_image.image_url",
239 + "message.output_text.logprobs",
240 + "reasoning.encrypted_content",
241 + ],
242 + ]] = None
243 + input: Union[str, ResponseInputParam]
244 + instructions: Optional[str] = None
245 + max_output_tokens: Optional[int] = None
246 + max_tool_calls: Optional[int] = None
247 + metadata: Optional[Metadata] = None
248 + model: Optional[str] = None
249 + parallel_tool_calls: Optional[bool] = True
250 + previous_response_id: Optional[str] = None
251 + prompt: Optional[ResponsePrompt] = None
252 + reasoning: Optional[Reasoning] = None
253 + service_tier: Literal["auto", "default", "flex", "scale",
254 + "priority"] = "auto"
255 + store: Optional[bool] = True
256 + stream: Optional[bool] = False
257 + temperature: Optional[float] = None
258 + text: Optional[ResponseTextConfig] = None
259 + tool_choice: ToolChoice = "auto"
260 + tools: list[Tool] = Field(default_factory=list)
261 + top_logprobs: Optional[int] = 0
262 + top_p: Optional[float] = None
263 + truncation: Optional[Literal["auto", "disabled"]] = "disabled"
264 + user: Optional[str] = None
265 +
266 + # --8<-- [start:responses-extra-params]
267 + request_id: str = Field(
268 + default_factory=lambda: f"resp_{random_uuid()}",
269 + description=(
270 + "The request_id related to this request. If the caller does "
271 + "not set it, a random_uuid will be generated. This id is used "
272 + "through out the inference process and return in response."),
273 + )
274 + mm_processor_kwargs: Optional[dict[str, Any]] = Field(
275 + default=None,
276 + description=("Additional kwargs to pass to the HF processor."),
277 + )
278 + priority: int = Field(
279 + default=0,
280 + description=(
281 + "The priority of the request (lower means earlier handling; "
282 + "default: 0). Any priority other than 0 will raise an error "
283 + "if the served model does not use priority scheduling."),
284 + )
285 + # --8<-- [end:responses-extra-params]
286 +
287 + _DEFAULT_SAMPLING_PARAMS = {
288 + "temperature": 1.0,
289 + "top_p": 1.0,
290 + }
291 +
292 + def to_sampling_params(
293 + self,
294 + default_max_tokens: int,
295 + default_sampling_params: Optional[dict] = None,
296 + ) -> SamplingParams:
297 + if self.max_output_tokens is None:
298 + max_tokens = default_max_tokens
299 + else:
300 + max_tokens = min(self.max_output_tokens, default_max_tokens)
301 +
302 + default_sampling_params = default_sampling_params or {}
303 + if (temperature := self.temperature) is None:
304 + temperature = default_sampling_params.get(
305 + "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"])
306 + if (top_p := self.top_p) is None:
307 + top_p = default_sampling_params.get(
308 + "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"])
309 +
310 + # Structured output
311 + guided_decoding = None
312 + if self.text is not None and self.text.format is not None:
313 + response_format = self.text.format
314 + if response_format.type == "json_schema":
315 + guided_decoding = GuidedDecodingParams.from_optional(
316 + json=response_format.schema_)
317 + elif response_format.type == "json_object":
318 + raise NotImplementedError("json_object is not supported")
319 +
320 + # TODO: add more parameters
321 + return SamplingParams.from_optional(
322 + temperature=temperature,
323 + top_p=top_p,
324 + max_tokens=max_tokens,
325 + logprobs=self.top_logprobs,
326 + output_kind=(RequestOutputKind.DELTA
327 + if self.stream else RequestOutputKind.FINAL_ONLY),
328 + guided_decoding=guided_decoding,
329 + )
330 +
331 + @model_validator(mode="before")
332 + def validate_background(cls, data):
333 + if not data.get("background"):
334 + return data
335 + if not data.get("store", True):
336 + raise ValueError(
337 + "background can only be used when `store` is true")
338 + return data
339 +
340 + @model_validator(mode="before")
341 + def validate_prompt(cls, data):
342 + if data.get("prompt") is not None:
343 + raise ValueError("prompt template is not supported")
344 + return data
345 +
346 +
347 class ChatCompletionRequest(OpenAIBaseModel):
348 # Ordered by official OpenAI API documentation
349 # https://platform.openai.com/docs/api-reference/chat/create
@@ -1473,6 +1597,83 @@ class TranscriptionStreamResponse(OpenAIBaseModel):
1597 usage: Optional[UsageInfo] = Field(default=None)
1598
1599
1600 +class ResponseReasoningItem(OpenAIBaseModel):
1601 + id: str = Field(default_factory=lambda: f"rs_{random_uuid()}")
1602 + text: str
1603 + summary: list = Field(default_factory=list)
1604 + type: Literal["reasoning"] = "reasoning"
1605 + encrypted_content: Optional[str] = None
1606 + status: Optional[Literal["in_progress", "completed", "incomplete"]]
1607 +
1608 +
1609 +class ResponsesResponse(OpenAIBaseModel):
1610 + id: str = Field(default_factory=lambda: f"resp_{random_uuid()}")
1611 + created_at: int = Field(default_factory=lambda: int(time.time()))
1612 + # error: Optional[ResponseError] = None
1613 + # incomplete_details: Optional[IncompleteDetails] = None
1614 + instructions: Optional[str] = None
1615 + metadata: Optional[Metadata] = None
1616 + model: str
1617 + object: Literal["response"] = "response"
1618 + output: list[Union[ResponseOutputMessage, ResponseReasoningItem]]
1619 + parallel_tool_calls: bool
1620 + temperature: float
1621 + tool_choice: ToolChoice
1622 + tools: list[Tool]
1623 + top_p: float
1624 + background: bool
1625 + max_output_tokens: int
1626 + max_tool_calls: Optional[int] = None
1627 + previous_response_id: Optional[str] = None
1628 + prompt: Optional[ResponsePrompt] = None
1629 + reasoning: Optional[Reasoning] = None
1630 + service_tier: Literal["auto", "default", "flex", "scale", "priority"]
1631 + status: ResponseStatus
1632 + text: Optional[ResponseTextConfig] = None
1633 + top_logprobs: int
1634 + truncation: Literal["auto", "disabled"]
1635 + usage: Optional[UsageInfo] = None
1636 + user: Optional[str] = None
1637 +
1638 + @classmethod
1639 + def from_request(
1640 + cls,
1641 + request: ResponsesRequest,
1642 + sampling_params: SamplingParams,
1643 + model_name: str,
1644 + created_time: int,
1645 + output: list[ResponseOutputItem],
1646 + status: ResponseStatus,
1647 + usage: Optional[UsageInfo] = None,
1648 + ) -> "ResponsesResponse":
1649 + return cls(
1650 + id=request.request_id,
1651 + created_at=created_time,
1652 + instructions=request.instructions,
1653 + metadata=request.metadata,
1654 + model=model_name,
1655 + output=output,
1656 + parallel_tool_calls=request.parallel_tool_calls,
1657 + temperature=sampling_params.temperature,
1658 + tool_choice=request.tool_choice,
1659 + tools=request.tools,
1660 + top_p=sampling_params.top_p,
1661 + background=request.background,
1662 + max_output_tokens=sampling_params.max_tokens,
1663 + max_tool_calls=request.max_tool_calls,
1664 + previous_response_id=request.previous_response_id,
1665 + prompt=request.prompt,
1666 + reasoning=request.reasoning,
1667 + service_tier=request.service_tier,
1668 + status=status,
1669 + text=request.text,
1670 + top_logprobs=sampling_params.logprobs,
1671 + truncation=request.truncation,
1672 + user=request.user,
1673 + usage=usage,
1674 + )
1675 +
1676 +
1677 BatchRequestInputBody = Union[ChatCompletionRequest, EmbeddingRequest,
1678 ScoreRequest, RerankRequest]
1679
vllm/entrypoints/openai/serving_engine.py
+5 -3
@@ -53,7 +53,8 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
53 EmbeddingRequest,
54 EmbeddingResponse, ErrorResponse,
55 PoolingResponse, RerankRequest,
56 - ScoreRequest, ScoreResponse,
56 + ResponsesRequest, ScoreRequest,
57 + ScoreResponse,
58 TokenizeChatRequest,
59 TokenizeCompletionRequest,
60 TokenizeResponse,
@@ -91,7 +92,8 @@ CompletionLikeRequest = Union[CompletionRequest, DetokenizeRequest,
92 ChatLikeRequest = Union[ChatCompletionRequest, EmbeddingChatRequest,
93 TokenizeChatRequest]
94 SpeechToTextRequest = Union[TranscriptionRequest, TranslationRequest]
94 -AnyRequest = Union[CompletionLikeRequest, ChatLikeRequest, SpeechToTextRequest]
95 +AnyRequest = Union[CompletionLikeRequest, ChatLikeRequest, SpeechToTextRequest,
96 + ResponsesRequest]
97
98 AnyResponse = Union[
99 CompletionResponse,
@@ -762,7 +764,7 @@ class OpenAIServing:
764
765 async def _preprocess_chat(
766 self,
765 - request: ChatLikeRequest,
767 + request: Union[ChatLikeRequest, ResponsesRequest],
768 tokenizer: AnyTokenizer,
769 messages: list[ChatCompletionMessageParam],
770 chat_template: Optional[str],
vllm/entrypoints/openai/serving_responses.py new
+464
@@ -0,0 +1,464 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import asyncio
5 +import time
6 +from collections.abc import AsyncGenerator, AsyncIterator
7 +from http import HTTPStatus
8 +from typing import Callable, Final, Optional, Union
9 +
10 +import jinja2
11 +from fastapi import Request
12 +from openai.types.responses import ResponseOutputMessage, ResponseOutputText
13 +
14 +from vllm.config import ModelConfig
15 +from vllm.engine.protocol import EngineClient
16 +from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
17 + ChatTemplateContentFormatOption)
18 +from vllm.entrypoints.logger import RequestLogger
19 +# yapf conflicts with isort for this block
20 +# yapf: disable
21 +from vllm.entrypoints.openai.protocol import (ErrorResponse,
22 + PromptTokenUsageInfo,
23 + RequestResponseMetadata,
24 + ResponseReasoningItem,
25 + ResponsesRequest,
26 + ResponsesResponse, UsageInfo)
27 +# yapf: enable
28 +from vllm.entrypoints.openai.serving_engine import OpenAIServing
29 +from vllm.entrypoints.openai.serving_models import OpenAIServingModels
30 +from vllm.logger import init_logger
31 +from vllm.outputs import RequestOutput
32 +from vllm.reasoning import ReasoningParser, ReasoningParserManager
33 +from vllm.sampling_params import SamplingParams
34 +from vllm.transformers_utils.tokenizer import AnyTokenizer
35 +from vllm.utils import random_uuid
36 +
37 +logger = init_logger(__name__)
38 +
39 +
40 +class OpenAIServingResponses(OpenAIServing):
41 +
42 + def __init__(
43 + self,
44 + engine_client: EngineClient,
45 + model_config: ModelConfig,
46 + models: OpenAIServingModels,
47 + *,
48 + request_logger: Optional[RequestLogger],
49 + chat_template: Optional[str],
50 + chat_template_content_format: ChatTemplateContentFormatOption,
51 + return_tokens_as_token_ids: bool = False,
52 + reasoning_parser: str = "",
53 + enable_auto_tools: bool = False,
54 + expand_tools_even_if_tool_choice_none: bool = False,
55 + tool_parser: Optional[str] = None,
56 + enable_prompt_tokens_details: bool = False,
57 + enable_force_include_usage: bool = False,
58 + ) -> None:
59 + super().__init__(
60 + engine_client=engine_client,
61 + model_config=model_config,
62 + models=models,
63 + request_logger=request_logger,
64 + return_tokens_as_token_ids=return_tokens_as_token_ids,
65 + enable_force_include_usage=enable_force_include_usage,
66 + )
67 +
68 + self.chat_template = chat_template
69 + self.chat_template_content_format: Final = chat_template_content_format
70 +
71 + self.reasoning_parser: Optional[Callable[[AnyTokenizer],
72 + ReasoningParser]] = None
73 + if reasoning_parser:
74 + try:
75 + self.reasoning_parser = (
76 + ReasoningParserManager.get_reasoning_parser(
77 + reasoning_parser))
78 + assert self.reasoning_parser is not None
79 + except Exception as e:
80 + raise TypeError(
81 + f"{reasoning_parser=} has not been registered") from e
82 +
83 + self.enable_prompt_tokens_details = enable_prompt_tokens_details
84 + self.enable_force_include_usage = enable_force_include_usage
85 + self.default_sampling_params = (
86 + self.model_config.get_diff_sampling_param())
87 + if self.default_sampling_params:
88 + source = self.model_config.generation_config
89 + source = "model" if source == "auto" else source
90 + logger.info("Using default chat sampling params from %s: %s",
91 + source, self.default_sampling_params)
92 +
93 + # HACK(woosuk): This is a hack. We should use a better store.
94 + # FIXME: This causes a memory leak since we never remove responses
95 + # from the store.
96 + self.response_store: dict[str, ResponsesResponse] = {}
97 + self.response_store_lock = asyncio.Lock()
98 +
99 + # HACK(woosuk): This is a hack. We should use a better store.
100 + # FIXME: This causes a memory leak since we never remove messages
101 + # from the store.
102 + self.msg_store: dict[str, list[ChatCompletionMessageParam]] = {}
103 +
104 + self.background_tasks: dict[str, asyncio.Task] = {}
105 +
106 + async def create_responses(
107 + self,
108 + request: ResponsesRequest,
109 + raw_request: Optional[Request] = None,
110 + ) -> Union[AsyncGenerator[str, None], ResponsesResponse, ErrorResponse]:
111 + error_check_ret = await self._check_model(request)
112 + if error_check_ret is not None:
113 + logger.error("Error with model %s", error_check_ret)
114 + return error_check_ret
115 +
116 + # If the engine is dead, raise the engine's DEAD_ERROR.
117 + # This is required for the streaming case, where we return a
118 + # success status before we actually start generating text :).
119 + if self.engine_client.errored:
120 + raise self.engine_client.dead_error
121 +
122 + # Handle the previous response ID.
123 + prev_response_id = request.previous_response_id
124 + if prev_response_id is not None:
125 + if not prev_response_id.startswith("resp_"):
126 + return self._make_invalid_id_error(prev_response_id)
127 + async with self.response_store_lock:
128 + prev_response = self.response_store.get(prev_response_id)
129 + if prev_response is None:
130 + return self._make_not_found_error(prev_response_id)
131 + else:
132 + prev_response = None
133 + # Construct the input messages.
134 + messages = self._construct_input_messages(request, prev_response)
135 +
136 + try:
137 + (
138 + lora_request,
139 + prompt_adapter_request,
140 + ) = self._maybe_get_adapters(request)
141 + model_name = self._get_model_name(request.model, lora_request)
142 + tokenizer = await self.engine_client.get_tokenizer(lora_request)
143 +
144 + _, request_prompts, engine_prompts = await self._preprocess_chat(
145 + request,
146 + tokenizer,
147 + messages,
148 + chat_template=self.chat_template,
149 + chat_template_content_format=self.chat_template_content_format,
150 + )
151 + except (ValueError, TypeError, RuntimeError,
152 + jinja2.TemplateError) as e:
153 + logger.exception("Error in preprocessing prompt inputs")
154 + return self.create_error_response(f"{e} {e.__cause__}")
155 +
156 + request_metadata = RequestResponseMetadata(
157 + request_id=request.request_id)
158 + if raw_request:
159 + raw_request.state.request_metadata = request_metadata
160 +
161 + # Schedule the request and get the result generator.
162 + generators: list[AsyncGenerator[RequestOutput, None]] = []
163 + try:
164 + for i, engine_prompt in enumerate(engine_prompts):
165 + default_max_tokens = self.max_model_len - len(
166 + engine_prompt["prompt_token_ids"])
167 + sampling_params = request.to_sampling_params(
168 + default_max_tokens, self.default_sampling_params)
169 +
170 + self._log_inputs(request.request_id,
171 + request_prompts[i],
172 + params=sampling_params,
173 + lora_request=lora_request,
174 + prompt_adapter_request=prompt_adapter_request)
175 +
176 + trace_headers = (None if raw_request is None else await
177 + self._get_trace_headers(raw_request.headers))
178 +
179 + generator = self.engine_client.generate(
180 + engine_prompt,
181 + sampling_params,
182 + request.request_id,
183 + lora_request=lora_request,
184 + trace_headers=trace_headers,
185 + prompt_adapter_request=prompt_adapter_request,
186 + priority=request.priority,
187 + )
188 + generators.append(generator)
189 + except ValueError as e:
190 + # TODO: Use a vllm-specific Validation Error
191 + return self.create_error_response(str(e))
192 +
193 + assert len(generators) == 1
194 + result_generator, = generators
195 +
196 + # Store the input messages.
197 + if request.store:
198 + self.msg_store[request.request_id] = messages
199 +
200 + if request.background:
201 + created_time = int(time.time())
202 + response = ResponsesResponse.from_request(
203 + request,
204 + sampling_params,
205 + model_name=model_name,
206 + created_time=created_time,
207 + output=[],
208 + status="queued",
209 + usage=None,
210 + )
211 + async with self.response_store_lock:
212 + self.response_store[response.id] = response
213 +
214 + # Run the request in the background.
215 + task = asyncio.create_task(
216 + self._run_background_request(
217 + request,
218 + sampling_params,
219 + result_generator,
220 + model_name,
221 + tokenizer,
222 + request_metadata,
223 + created_time,
224 + ),
225 + name=f"create_{response.id}",
226 + )
227 +
228 + # For cleanup.
229 + response_id = response.id
230 + self.background_tasks[response_id] = task
231 + task.add_done_callback(
232 + lambda _: self.background_tasks.pop(response_id, None))
233 + return response
234 +
235 + if request.stream:
236 + raise NotImplementedError("Streaming responses are not supported")
237 +
238 + try:
239 + return await self.responses_full_generator(
240 + request,
241 + sampling_params,
242 + result_generator,
243 + model_name,
244 + tokenizer,
245 + request_metadata,
246 + )
247 + except Exception as e:
248 + return self.create_error_response(str(e))
249 +
250 + async def responses_full_generator(
251 + self,
252 + request: ResponsesRequest,
253 + sampling_params: SamplingParams,
254 + result_generator: AsyncIterator[RequestOutput],
255 + model_name: str,
256 + tokenizer: AnyTokenizer,
257 + request_metadata: RequestResponseMetadata,
258 + created_time: Optional[int] = None,
259 + ) -> Union[ErrorResponse, ResponsesResponse]:
260 + if created_time is None:
261 + created_time = int(time.time())
262 + final_res: Optional[RequestOutput] = None
263 +
264 + try:
265 + async for res in result_generator:
266 + final_res = res
267 + except asyncio.CancelledError:
268 + return self.create_error_response("Client disconnected")
269 + except ValueError as e:
270 + # TODO: Use a vllm-specific Validation Error
271 + return self.create_error_response(str(e))
272 +
273 + assert final_res is not None
274 + assert len(final_res.outputs) == 1
275 + final_output = final_res.outputs[0]
276 +
277 + if self.reasoning_parser:
278 + try:
279 + reasoning_parser = self.reasoning_parser(tokenizer)
280 + except RuntimeError as e:
281 + logger.exception("Error in reasoning parser creation.")
282 + return self.create_error_response(str(e))
283 +
284 + reasoning_content, content = (
285 + reasoning_parser.extract_reasoning_content(final_output.text,
286 + request=request))
287 + else:
288 + reasoning_content = None
289 + content = final_output.text
290 +
291 + output = []
292 + if reasoning_content:
293 + reasoning_item = ResponseReasoningItem(
294 + text=reasoning_content,
295 + status=None, # NOTE: Only the last output item has status.
296 + )
297 + output.append(reasoning_item)
298 + if content:
299 + output_text = ResponseOutputText(
300 + text=content,
301 + annotations=[], # TODO
302 + type="output_text",
303 + logprobs=None, # TODO
304 + )
305 + message = ResponseOutputMessage(
306 + id=f"msg_{random_uuid()}",
307 + content=[output_text],
308 + role="assistant",
309 + status="completed",
310 + type="message",
311 + )
312 + output.append(message)
313 +
314 + # Calculate usage.
315 + assert final_res.prompt_token_ids is not None
316 + num_prompt_tokens = len(final_res.prompt_token_ids)
317 + num_generated_tokens = len(final_output.token_ids)
318 + usage = UsageInfo(
319 + prompt_tokens=num_prompt_tokens,
320 + completion_tokens=num_generated_tokens,
321 + total_tokens=num_prompt_tokens + num_generated_tokens,
322 + )
323 + if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
324 + usage.prompt_tokens_details = PromptTokenUsageInfo(
325 + cached_tokens=final_res.num_cached_tokens)
326 + request_metadata.final_usage_info = usage
327 +
328 + response = ResponsesResponse.from_request(
329 + request,
330 + sampling_params,
331 + model_name=model_name,
332 + created_time=created_time,
333 + output=output,
334 + status="completed",
335 + usage=usage,
336 + )
337 +
338 + if request.store:
339 + async with self.response_store_lock:
340 + stored_response = self.response_store.get(response.id)
341 + # If the response is already cancelled, don't update it.
342 + if (stored_response is None
343 + or stored_response.status != "cancelled"):
344 + self.response_store[response.id] = response
345 + return response
346 +
347 + def _construct_input_messages(
348 + self,
349 + request: ResponsesRequest,
350 + prev_response: Optional[ResponsesResponse] = None,
351 + ) -> list[ChatCompletionMessageParam]:
352 + messages: list[ChatCompletionMessageParam] = []
353 + if request.instructions:
354 + messages.append({
355 + "role": "system",
356 + "content": request.instructions,
357 + })
358 +
359 + # Prepend the conversation history.
360 + if prev_response is not None:
361 + # Add the previous messages.
362 + prev_msg = self.msg_store[prev_response.id]
363 + messages.extend(prev_msg)
364 +
365 + # Add the previous output.
366 + for output_item in prev_response.output:
367 + # NOTE: We skip the reasoning output.
368 + if isinstance(output_item, ResponseOutputMessage):
369 + for content in output_item.content:
370 + messages.append({
371 + "role": "assistant",
372 + "content": content.text,
373 + })
374 +
375 + # Append the new input.
376 + # Reponses API supports simple text inputs without chat format.
377 + if isinstance(request.input, str):
378 + messages.append({"role": "user", "content": request.input})
379 + else:
380 + messages.extend(request.input) # type: ignore
381 + return messages
382 +
383 + async def _run_background_request(
384 + self,
385 + request: ResponsesRequest,
386 + *args,
387 + **kwargs,
388 + ):
389 + try:
390 + response = await self.responses_full_generator(
391 + request, *args, **kwargs)
392 + except Exception as e:
393 + logger.exception("Background request failed for %s",
394 + request.request_id)
395 + response = self.create_error_response(str(e))
396 +
397 + if isinstance(response, ErrorResponse):
398 + # If the request has failed, update the status to "failed".
399 + response_id = request.request_id
400 + async with self.response_store_lock:
401 + stored_response = self.response_store.get(response_id)
402 + assert stored_response is not None
403 + if stored_response.status not in ("completed", "cancelled"):
404 + stored_response.status = "failed"
405 +
406 + async def retrieve_responses(
407 + self,
408 + response_id: str,
409 + ) -> Union[ErrorResponse, ResponsesResponse]:
410 + if not response_id.startswith("resp_"):
411 + return self._make_invalid_id_error(response_id)
412 +
413 + async with self.response_store_lock:
414 + response = self.response_store.get(response_id)
415 +
416 + if response is None:
417 + return self._make_not_found_error(response_id)
418 + return response
419 +
420 + async def cancel_responses(
421 + self,
422 + response_id: str,
423 + ) -> Union[ErrorResponse, ResponsesResponse]:
424 + if not response_id.startswith("resp_"):
425 + return self._make_invalid_id_error(response_id)
426 +
427 + async with self.response_store_lock:
428 + response = self.response_store.get(response_id)
429 + if response is None:
430 + return self._make_not_found_error(response_id)
431 +
432 + prev_status = response.status
433 + if prev_status not in ("queued", "in_progress"):
434 + return self.create_error_response(
435 + err_type="invalid_request_error",
436 + message="Cannot cancel a synchronous response.",
437 + )
438 +
439 + # Update the status to "cancelled".
440 + response.status = "cancelled"
441 +
442 + # Abort the request.
443 + if (task := self.background_tasks.get(response_id)):
444 + task.cancel()
445 + try:
446 + await task
447 + except asyncio.CancelledError:
448 + logger.exception("Background task for %s was cancelled",
449 + response_id)
450 + return response
451 +
452 + def _make_invalid_id_error(self, response_id: str) -> ErrorResponse:
453 + return self.create_error_response(
454 + err_type="invalid_request_error",
455 + message=(f"Invalid 'response_id': '{response_id}'. "
456 + "Expected an ID that begins with 'resp'."),
457 + )
458 +
459 + def _make_not_found_error(self, response_id: str) -> ErrorResponse:
460 + return self.create_error_response(
461 + err_type="invalid_request_error",
462 + message=f"Response with id '{response_id}' not found.",
463 + status_code=HTTPStatus.NOT_FOUND,
464 + )
vllm/reasoning/abs_reasoning_parsers.py
+4 -2
@@ -10,7 +10,7 @@ from functools import cached_property
10 from typing import Callable, Optional, Union
11
12 from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
13 - DeltaMessage)
13 + DeltaMessage, ResponsesRequest)
14 from vllm.logger import init_logger
15 from vllm.transformers_utils.tokenizer import AnyTokenizer
16 from vllm.utils import import_from_path, is_list_of
@@ -66,7 +66,9 @@ class ReasoningParser:
66
67 @abstractmethod
68 def extract_reasoning_content(
69 - self, model_output: str, request: ChatCompletionRequest
69 + self,
70 + model_output: str,
71 + request: Union[ChatCompletionRequest, ResponsesRequest],
72 ) -> tuple[Optional[str], Optional[str]]:
73 """
74 Extract reasoning content from a complete model-generated string.