A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 04:00 UTC
[Refactor] [8/N] to simplify the vLLM openai responsesapi_serving architecture (#32260)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Jan 14, 2026 at 15:26 UTC
9312a6c03a87e100fdfc2679900221f9c6489a63
21 files changed
+754
-674
tests/entrypoints/openai/responses/test_function_call_parsing.py
+2
-2
@@ -7,7 +7,7 @@ import json
7
import pytest
8
from openai.types.responses import ResponseFunctionToolCall
9
10
-from vllm.entrypoints.openai.engine.protocol import ResponsesRequest
10
+from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
11
12
13
def test_function_call_dict_converted_to_object():
@@ -253,7 +253,7 @@ def test_function_call_validation_failure_logs_debug(caplog):
253
}
254
255
# Mock the logger to verify debug was called
256
- with patch("vllm.entrypoints.openai.engine.protocol.logger") as mock_logger:
256
+ with patch("vllm.entrypoints.openai.responses.protocol.logger") as mock_logger:
257
with pytest.raises(ValueError):
258
ResponsesRequest(**request_data)
259
tests/entrypoints/openai/test_protocol.py
+1
-1
@@ -4,7 +4,7 @@ from openai_harmony import (
4
Message,
5
)
6
7
-from vllm.entrypoints.openai.engine.protocol import (
7
+from vllm.entrypoints.openai.responses.protocol import (
8
serialize_message,
9
serialize_messages,
10
)
tests/entrypoints/openai/test_serving_responses.py
+3
-2
@@ -14,8 +14,9 @@ from openai.types.responses.tool import (
14
)
15
16
from vllm.entrypoints.context import ConversationContext
17
-from vllm.entrypoints.openai.engine.protocol import ErrorResponse, ResponsesRequest
18
-from vllm.entrypoints.openai.serving_responses import (
17
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
18
+from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
19
+from vllm.entrypoints.openai.responses.serving import (
20
OpenAIServingResponses,
21
_extract_allowed_tools_from_mcp_requests,
22
extract_tool_types,
vllm/entrypoints/context.py
+5
-3
@@ -24,9 +24,6 @@ from vllm.entrypoints.chat_utils import (
24
from vllm.entrypoints.constants import MCP_PREFIX
25
from vllm.entrypoints.openai.engine.protocol import (
26
FunctionCall,
27
- ResponseInputOutputItem,
28
- ResponseRawMessageAndToken,
29
- ResponsesRequest,
27
)
28
from vllm.entrypoints.openai.parser.harmony_utils import (
29
get_encoding,
@@ -36,6 +33,11 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
33
from vllm.entrypoints.openai.parser.responses_parser import (
34
get_responses_parser_for_simple_context,
35
)
36
+from vllm.entrypoints.openai.responses.protocol import (
37
+ ResponseInputOutputItem,
38
+ ResponseRawMessageAndToken,
39
+ ResponsesRequest,
40
+)
41
from vllm.entrypoints.responses_utils import construct_tool_dicts
42
from vllm.entrypoints.tool import Tool
43
from vllm.entrypoints.tool_server import ToolServer
vllm/entrypoints/openai/api_server.py
+8
-111
@@ -14,7 +14,7 @@ import socket
14
import tempfile
15
import uuid
16
from argparse import Namespace
17
-from collections.abc import AsyncGenerator, AsyncIterator, Awaitable
17
+from collections.abc import AsyncIterator, Awaitable
18
from contextlib import asynccontextmanager
19
from http import HTTPStatus
20
from typing import Annotated, Any
@@ -49,9 +49,6 @@ from vllm.entrypoints.openai.engine.protocol import (
49
CompletionResponse,
50
ErrorInfo,
51
ErrorResponse,
52
- ResponsesRequest,
53
- ResponsesResponse,
54
- StreamingResponsesResponse,
52
TranscriptionRequest,
53
TranscriptionResponseVariant,
54
TranslationRequest,
@@ -59,12 +56,12 @@ from vllm.entrypoints.openai.engine.protocol import (
56
)
57
from vllm.entrypoints.openai.engine.serving import OpenAIServing
58
from vllm.entrypoints.openai.orca_metrics import metrics_header
59
+from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
60
from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
61
from vllm.entrypoints.openai.serving_models import (
62
BaseModelPath,
63
OpenAIServingModels,
64
)
67
-from vllm.entrypoints.openai.serving_responses import OpenAIServingResponses
65
from vllm.entrypoints.openai.serving_transcription import (
66
OpenAIServingTranscription,
67
OpenAIServingTranslation,
@@ -311,112 +308,6 @@ async def show_version():
308
return JSONResponse(content=ver)
309
310
314
-async def _convert_stream_to_sse_events(
315
- generator: AsyncGenerator[StreamingResponsesResponse, None],
316
-) -> AsyncGenerator[str, None]:
317
- """Convert the generator to a stream of events in SSE format"""
318
- async for event in generator:
319
- event_type = getattr(event, "type", "unknown")
320
- # https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#event_stream_format
321
- event_data = (
322
- f"event: {event_type}\ndata: {event.model_dump_json(indent=None)}\n\n"
323
- )
324
- yield event_data
325
-
326
-
327
-@router.post(
328
- "/v1/responses",
329
- dependencies=[Depends(validate_json_request)],
330
- responses={
331
- HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
332
- HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
333
- HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
334
- HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
335
- },
336
-)
337
-@with_cancellation
338
-async def create_responses(request: ResponsesRequest, raw_request: Request):
339
- handler = responses(raw_request)
340
- if handler is None:
341
- return base(raw_request).create_error_response(
342
- message="The model does not support Responses API"
343
- )
344
- try:
345
- generator = await handler.create_responses(request, raw_request)
346
- except Exception as e:
347
- raise HTTPException(
348
- status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
349
- ) from e
350
-
351
- if isinstance(generator, ErrorResponse):
352
- return JSONResponse(
353
- content=generator.model_dump(), status_code=generator.error.code
354
- )
355
- elif isinstance(generator, ResponsesResponse):
356
- return JSONResponse(content=generator.model_dump())
357
-
358
- return StreamingResponse(
359
- content=_convert_stream_to_sse_events(generator), media_type="text/event-stream"
360
- )
361
-
362
-
363
-@router.get("/v1/responses/{response_id}")
364
-async def retrieve_responses(
365
- response_id: str,
366
- raw_request: Request,
367
- starting_after: int | None = None,
368
- stream: bool | None = False,
369
-):
370
- handler = responses(raw_request)
371
- if handler is None:
372
- return base(raw_request).create_error_response(
373
- message="The model does not support Responses API"
374
- )
375
-
376
- try:
377
- response = await handler.retrieve_responses(
378
- response_id,
379
- starting_after=starting_after,
380
- stream=stream,
381
- )
382
- except Exception as e:
383
- raise HTTPException(
384
- status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
385
- ) from e
386
-
387
- if isinstance(response, ErrorResponse):
388
- return JSONResponse(
389
- content=response.model_dump(), status_code=response.error.code
390
- )
391
- elif isinstance(response, ResponsesResponse):
392
- return JSONResponse(content=response.model_dump())
393
- return StreamingResponse(
394
- content=_convert_stream_to_sse_events(response), media_type="text/event-stream"
395
- )
396
-
397
-
398
-@router.post("/v1/responses/{response_id}/cancel")
399
-async def cancel_responses(response_id: str, raw_request: Request):
400
- handler = responses(raw_request)
401
- if handler is None:
402
- return base(raw_request).create_error_response(
403
- message="The model does not support Responses API"
404
- )
405
-
406
- try:
407
- response = await handler.cancel_responses(response_id)
408
- except Exception as e:
409
- raise HTTPException(
410
- status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
411
- ) from e
412
-
413
- if isinstance(response, ErrorResponse):
414
- return JSONResponse(
415
- content=response.model_dump(), status_code=response.error.code
416
- )
417
- return JSONResponse(content=response.model_dump())
418
-
419
-
311
@router.post(
312
"/v1/messages",
313
dependencies=[Depends(validate_json_request)],
@@ -844,6 +735,12 @@ def build_app(args: Namespace) -> FastAPI:
735
)
736
737
register_chat_api_router(app)
738
+
739
+ from vllm.entrypoints.openai.responses.api_router import (
740
+ attach_router as register_responses_api_router,
741
+ )
742
+
743
+ register_responses_api_router(app)
744
from vllm.entrypoints.sagemaker.routes import register_sagemaker_routes
745
746
register_sagemaker_routes(router)
vllm/entrypoints/openai/engine/protocol.py
+1
-531
@@ -11,63 +11,14 @@ from typing import Annotated, Any, ClassVar, Literal, TypeAlias
11
import regex as re
12
import torch
13
from fastapi import HTTPException, UploadFile
14
-from openai.types.responses import (
15
- ResponseCodeInterpreterCallCodeDeltaEvent,
16
- ResponseCodeInterpreterCallCodeDoneEvent,
17
- ResponseCodeInterpreterCallCompletedEvent,
18
- ResponseCodeInterpreterCallInProgressEvent,
19
- ResponseCodeInterpreterCallInterpretingEvent,
20
- ResponseContentPartAddedEvent,
21
- ResponseContentPartDoneEvent,
22
- ResponseFunctionToolCall,
23
- ResponseInputItemParam,
24
- ResponseMcpCallArgumentsDeltaEvent,
25
- ResponseMcpCallArgumentsDoneEvent,
26
- ResponseMcpCallCompletedEvent,
27
- ResponseMcpCallInProgressEvent,
28
- ResponseOutputItem,
29
- ResponseOutputItemAddedEvent,
30
- ResponseOutputItemDoneEvent,
31
- ResponsePrompt,
32
- ResponseReasoningTextDeltaEvent,
33
- ResponseReasoningTextDoneEvent,
34
- ResponseStatus,
35
- ResponseWebSearchCallCompletedEvent,
36
- ResponseWebSearchCallInProgressEvent,
37
- ResponseWebSearchCallSearchingEvent,
38
-)
39
-from openai.types.responses import (
40
- ResponseCompletedEvent as OpenAIResponseCompletedEvent,
41
-)
42
-from openai.types.responses import ResponseCreatedEvent as OpenAIResponseCreatedEvent
43
-from openai.types.responses import (
44
- ResponseInProgressEvent as OpenAIResponseInProgressEvent,
45
-)
46
-from openai.types.responses.response_reasoning_item import (
47
- Content as ResponseReasoningTextContent,
48
-)
49
-from openai_harmony import Message as OpenAIHarmonyMessage
50
-
51
-# Backward compatibility for OpenAI client versions
52
-try: # For older openai versions (< 1.100.0)
53
- from openai.types.responses import ResponseTextConfig
54
-except ImportError: # For newer openai versions (>= 1.100.0)
55
- from openai.types.responses import ResponseFormatTextConfig as ResponseTextConfig
56
-
57
-
58
-from openai.types.responses.response import IncompleteDetails, ToolChoice
59
-from openai.types.responses.tool import Tool
60
-from openai.types.shared import Metadata, Reasoning
14
from pydantic import (
15
BaseModel,
16
ConfigDict,
17
Field,
65
- ValidationError,
66
- field_serializer,
18
model_validator,
19
)
20
70
-from vllm.entrypoints.chat_utils import ChatCompletionMessageParam, make_tool_call_id
21
+from vllm.entrypoints.chat_utils import make_tool_call_id
22
from vllm.exceptions import VLLMValidationError
23
from vllm.logger import init_logger
24
from vllm.logprobs import Logprob
@@ -277,237 +228,6 @@ def get_logits_processors(
228
return None
229
230
280
-ResponseInputOutputItem: TypeAlias = ResponseInputItemParam | ResponseOutputItem
281
-
282
-
283
-class ResponsesRequest(OpenAIBaseModel):
284
- # Ordered by official OpenAI API documentation
285
- # https://platform.openai.com/docs/api-reference/responses/create
286
- background: bool | None = False
287
- include: (
288
- list[
289
- Literal[
290
- "code_interpreter_call.outputs",
291
- "computer_call_output.output.image_url",
292
- "file_search_call.results",
293
- "message.input_image.image_url",
294
- "message.output_text.logprobs",
295
- "reasoning.encrypted_content",
296
- ],
297
- ]
298
- | None
299
- ) = None
300
- input: str | list[ResponseInputOutputItem]
301
- instructions: str | None = None
302
- max_output_tokens: int | None = None
303
- max_tool_calls: int | None = None
304
- metadata: Metadata | None = None
305
- model: str | None = None
306
- logit_bias: dict[str, float] | None = None
307
- parallel_tool_calls: bool | None = True
308
- previous_response_id: str | None = None
309
- prompt: ResponsePrompt | None = None
310
- reasoning: Reasoning | None = None
311
- service_tier: Literal["auto", "default", "flex", "scale", "priority"] = "auto"
312
- store: bool | None = True
313
- stream: bool | None = False
314
- temperature: float | None = None
315
- text: ResponseTextConfig | None = None
316
- tool_choice: ToolChoice = "auto"
317
- tools: list[Tool] = Field(default_factory=list)
318
- top_logprobs: int | None = 0
319
- top_p: float | None = None
320
- top_k: int | None = None
321
- truncation: Literal["auto", "disabled"] | None = "disabled"
322
- user: str | None = None
323
-
324
- # --8<-- [start:responses-extra-params]
325
- request_id: str = Field(
326
- default_factory=lambda: f"resp_{random_uuid()}",
327
- description=(
328
- "The request_id related to this request. If the caller does "
329
- "not set it, a random_uuid will be generated. This id is used "
330
- "through out the inference process and return in response."
331
- ),
332
- )
333
- mm_processor_kwargs: dict[str, Any] | None = Field(
334
- default=None,
335
- description=("Additional kwargs to pass to the HF processor."),
336
- )
337
- priority: int = Field(
338
- default=0,
339
- description=(
340
- "The priority of the request (lower means earlier handling; "
341
- "default: 0). Any priority other than 0 will raise an error "
342
- "if the served model does not use priority scheduling."
343
- ),
344
- )
345
- cache_salt: str | None = Field(
346
- default=None,
347
- description=(
348
- "If specified, the prefix cache will be salted with the provided "
349
- "string to prevent an attacker to guess prompts in multi-user "
350
- "environments. The salt should be random, protected from "
351
- "access by 3rd parties, and long enough to be "
352
- "unpredictable (e.g., 43 characters base64-encoded, corresponding "
353
- "to 256 bit)."
354
- ),
355
- )
356
-
357
- enable_response_messages: bool = Field(
358
- default=False,
359
- description=(
360
- "Dictates whether or not to return messages as part of the "
361
- "response object. Currently only supported for"
362
- "non-background and gpt-oss only. "
363
- ),
364
- )
365
- # similar to input_messages / output_messages in ResponsesResponse
366
- # we take in previous_input_messages (ie in harmony format)
367
- # this cannot be used in conjunction with previous_response_id
368
- # TODO: consider supporting non harmony messages as well
369
- previous_input_messages: list[OpenAIHarmonyMessage | dict] | None = None
370
- # --8<-- [end:responses-extra-params]
371
-
372
- _DEFAULT_SAMPLING_PARAMS = {
373
- "temperature": 1.0,
374
- "top_p": 1.0,
375
- "top_k": 0,
376
- }
377
-
378
- def to_sampling_params(
379
- self,
380
- default_max_tokens: int,
381
- default_sampling_params: dict | None = None,
382
- ) -> SamplingParams:
383
- if self.max_output_tokens is None:
384
- max_tokens = default_max_tokens
385
- else:
386
- max_tokens = min(self.max_output_tokens, default_max_tokens)
387
-
388
- default_sampling_params = default_sampling_params or {}
389
- if (temperature := self.temperature) is None:
390
- temperature = default_sampling_params.get(
391
- "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
392
- )
393
- if (top_p := self.top_p) is None:
394
- top_p = default_sampling_params.get(
395
- "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
396
- )
397
- if (top_k := self.top_k) is None:
398
- top_k = default_sampling_params.get(
399
- "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
400
- )
401
- stop_token_ids = default_sampling_params.get("stop_token_ids")
402
-
403
- # Structured output
404
- structured_outputs = None
405
- if self.text is not None and self.text.format is not None:
406
- response_format = self.text.format
407
- if (
408
- response_format.type == "json_schema"
409
- and response_format.schema_ is not None
410
- ):
411
- structured_outputs = StructuredOutputsParams(
412
- json=response_format.schema_
413
- )
414
- elif response_format.type == "json_object":
415
- raise NotImplementedError("json_object is not supported")
416
-
417
- # TODO: add more parameters
418
- return SamplingParams.from_optional(
419
- temperature=temperature,
420
- top_p=top_p,
421
- top_k=top_k,
422
- max_tokens=max_tokens,
423
- logprobs=self.top_logprobs if self.is_include_output_logprobs() else None,
424
- stop_token_ids=stop_token_ids,
425
- output_kind=(
426
- RequestOutputKind.DELTA if self.stream else RequestOutputKind.FINAL_ONLY
427
- ),
428
- structured_outputs=structured_outputs,
429
- logit_bias=self.logit_bias,
430
- skip_clone=True, # Created fresh per request, safe to skip clone
431
- )
432
-
433
- def is_include_output_logprobs(self) -> bool:
434
- """Check if the request includes output logprobs."""
435
- if self.include is None:
436
- return False
437
- return (
438
- isinstance(self.include, list)
439
- and "message.output_text.logprobs" in self.include
440
- )
441
-
442
- @model_validator(mode="before")
443
- def validate_background(cls, data):
444
- if not data.get("background"):
445
- return data
446
- if not data.get("store", True):
447
- raise ValueError("background can only be used when `store` is true")
448
- return data
449
-
450
- @model_validator(mode="before")
451
- def validate_prompt(cls, data):
452
- if data.get("prompt") is not None:
453
- raise VLLMValidationError(
454
- "prompt template is not supported", parameter="prompt"
455
- )
456
- return data
457
-
458
- @model_validator(mode="before")
459
- def check_cache_salt_support(cls, data):
460
- if data.get("cache_salt") is not None and (
461
- not isinstance(data["cache_salt"], str) or not data["cache_salt"]
462
- ):
463
- raise ValueError(
464
- "Parameter 'cache_salt' must be a non-empty string if provided."
465
- )
466
- return data
467
-
468
- @model_validator(mode="before")
469
- def function_call_parsing(cls, data):
470
- """Parse function_call dictionaries into ResponseFunctionToolCall objects.
471
- This ensures Pydantic can properly resolve union types in the input field.
472
- Function calls provided as dicts are converted to ResponseFunctionToolCall
473
- objects before validation, while invalid structures are left for Pydantic
474
- to reject with appropriate error messages.
475
- """
476
-
477
- input_data = data.get("input")
478
-
479
- # Early return for None, strings, or bytes
480
- # (strings are iterable but shouldn't be processed)
481
- if input_data is None or isinstance(input_data, (str, bytes)):
482
- return data
483
-
484
- # Convert iterators (like ValidatorIterator) to list
485
- if not isinstance(input_data, list):
486
- try:
487
- input_data = list(input_data)
488
- except TypeError:
489
- # Not iterable, leave as-is for Pydantic to handle
490
- return data
491
-
492
- processed_input = []
493
- for item in input_data:
494
- if isinstance(item, dict) and item.get("type") == "function_call":
495
- try:
496
- processed_input.append(ResponseFunctionToolCall(**item))
497
- except ValidationError:
498
- # Let Pydantic handle validation for malformed function calls
499
- logger.debug(
500
- "Failed to parse function_call to ResponseFunctionToolCall, "
501
- "leaving for Pydantic validation"
502
- )
503
- processed_input.append(item)
504
- else:
505
- processed_input.append(item)
506
-
507
- data["input"] = processed_input
508
- return data
509
-
510
-
231
class CompletionRequest(OpenAIBaseModel):
232
# Ordered by official OpenAI API documentation
233
# https://platform.openai.com/docs/api-reference/completions/create
@@ -997,256 +717,6 @@ class TranscriptionStreamResponse(OpenAIBaseModel):
717
usage: UsageInfo | None = Field(default=None)
718
719
1000
-class InputTokensDetails(OpenAIBaseModel):
1001
- cached_tokens: int
1002
- input_tokens_per_turn: list[int] = Field(default_factory=list)
1003
- cached_tokens_per_turn: list[int] = Field(default_factory=list)
1004
-
1005
-
1006
-class OutputTokensDetails(OpenAIBaseModel):
1007
- reasoning_tokens: int = 0
1008
- tool_output_tokens: int = 0
1009
- output_tokens_per_turn: list[int] = Field(default_factory=list)
1010
- tool_output_tokens_per_turn: list[int] = Field(default_factory=list)
1011
-
1012
-
1013
-class ResponseUsage(OpenAIBaseModel):
1014
- input_tokens: int
1015
- input_tokens_details: InputTokensDetails
1016
- output_tokens: int
1017
- output_tokens_details: OutputTokensDetails
1018
- total_tokens: int
1019
-
1020
-
1021
-def serialize_message(msg):
1022
- """
1023
- Serializes a single message
1024
- """
1025
- if isinstance(msg, dict):
1026
- return msg
1027
- elif hasattr(msg, "to_dict"):
1028
- return msg.to_dict()
1029
- else:
1030
- # fallback to pyandic dump
1031
- return msg.model_dump_json()
1032
-
1033
-
1034
-def serialize_messages(msgs):
1035
- """
1036
- Serializes multiple messages
1037
- """
1038
- return [serialize_message(msg) for msg in msgs] if msgs else None
1039
-
1040
-
1041
-class ResponseRawMessageAndToken(OpenAIBaseModel):
1042
- """Class to show the raw message.
1043
- If message / tokens diverge, tokens is the source of truth"""
1044
-
1045
- message: str
1046
- tokens: list[int]
1047
- type: Literal["raw_message_tokens"] = "raw_message_tokens"
1048
-
1049
-
1050
-ResponseInputOutputMessage: TypeAlias = (
1051
- list[ChatCompletionMessageParam] | list[ResponseRawMessageAndToken]
1052
-)
1053
-
1054
-
1055
-class ResponsesResponse(OpenAIBaseModel):
1056
- id: str = Field(default_factory=lambda: f"resp_{random_uuid()}")
1057
- created_at: int = Field(default_factory=lambda: int(time.time()))
1058
- # error: Optional[ResponseError] = None
1059
- incomplete_details: IncompleteDetails | None = None
1060
- instructions: str | None = None
1061
- metadata: Metadata | None = None
1062
- model: str
1063
- object: Literal["response"] = "response"
1064
- output: list[ResponseOutputItem]
1065
- parallel_tool_calls: bool
1066
- temperature: float
1067
- tool_choice: ToolChoice
1068
- tools: list[Tool]
1069
- top_p: float
1070
- background: bool
1071
- max_output_tokens: int
1072
- max_tool_calls: int | None = None
1073
- previous_response_id: str | None = None
1074
- prompt: ResponsePrompt | None = None
1075
- reasoning: Reasoning | None = None
1076
- service_tier: Literal["auto", "default", "flex", "scale", "priority"]
1077
- status: ResponseStatus
1078
- text: ResponseTextConfig | None = None
1079
- top_logprobs: int | None = None
1080
- truncation: Literal["auto", "disabled"]
1081
- usage: ResponseUsage | None = None
1082
- user: str | None = None
1083
-
1084
- # --8<-- [start:responses-response-extra-params]
1085
- # These are populated when enable_response_messages is set to True
1086
- # NOTE: custom serialization is needed
1087
- # see serialize_input_messages and serialize_output_messages
1088
- input_messages: ResponseInputOutputMessage | None = Field(
1089
- default=None,
1090
- description=(
1091
- "If enable_response_messages, we can show raw token input to model."
1092
- ),
1093
- )
1094
- output_messages: ResponseInputOutputMessage | None = Field(
1095
- default=None,
1096
- description=(
1097
- "If enable_response_messages, we can show raw token output of model."
1098
- ),
1099
- )
1100
- # --8<-- [end:responses-response-extra-params]
1101
-
1102
- # NOTE: openAI harmony doesn't serialize TextContent properly,
1103
- # TODO: this fixes for TextContent, but need to verify for tools etc
1104
- # https://github.com/openai/harmony/issues/78
1105
- @field_serializer("output_messages", when_used="json")
1106
- def serialize_output_messages(self, msgs, _info):
1107
- return serialize_messages(msgs)
1108
-
1109
- # NOTE: openAI harmony doesn't serialize TextContent properly, this fixes it
1110
- # https://github.com/openai/harmony/issues/78
1111
- @field_serializer("input_messages", when_used="json")
1112
- def serialize_input_messages(self, msgs, _info):
1113
- return serialize_messages(msgs)
1114
-
1115
- @classmethod
1116
- def from_request(
1117
- cls,
1118
- request: ResponsesRequest,
1119
- sampling_params: SamplingParams,
1120
- model_name: str,
1121
- created_time: int,
1122
- output: list[ResponseOutputItem],
1123
- status: ResponseStatus,
1124
- usage: ResponseUsage | None = None,
1125
- input_messages: ResponseInputOutputMessage | None = None,
1126
- output_messages: ResponseInputOutputMessage | None = None,
1127
- ) -> "ResponsesResponse":
1128
- incomplete_details: IncompleteDetails | None = None
1129
- if status == "incomplete":
1130
- incomplete_details = IncompleteDetails(reason="max_output_tokens")
1131
- # TODO: implement the other reason for incomplete_details,
1132
- # which is content_filter
1133
- # incomplete_details = IncompleteDetails(reason='content_filter')
1134
- return cls(
1135
- id=request.request_id,
1136
- created_at=created_time,
1137
- incomplete_details=incomplete_details,
1138
- instructions=request.instructions,
1139
- metadata=request.metadata,
1140
- model=model_name,
1141
- output=output,
1142
- input_messages=input_messages,
1143
- output_messages=output_messages,
1144
- parallel_tool_calls=request.parallel_tool_calls,
1145
- temperature=sampling_params.temperature,
1146
- tool_choice=request.tool_choice,
1147
- tools=request.tools,
1148
- top_p=sampling_params.top_p,
1149
- background=request.background,
1150
- max_output_tokens=sampling_params.max_tokens,
1151
- max_tool_calls=request.max_tool_calls,
1152
- previous_response_id=request.previous_response_id,
1153
- prompt=request.prompt,
1154
- reasoning=request.reasoning,
1155
- service_tier=request.service_tier,
1156
- status=status,
1157
- text=request.text,
1158
- top_logprobs=sampling_params.logprobs,
1159
- truncation=request.truncation,
1160
- user=request.user,
1161
- usage=usage,
1162
- )
1163
-
1164
-
1165
-# TODO: this code can be removed once
1166
-# https://github.com/openai/openai-python/issues/2634 has been resolved
1167
-class ResponseReasoningPartDoneEvent(OpenAIBaseModel):
1168
- content_index: int
1169
- """The index of the content part that is done."""
1170
-
1171
- item_id: str
1172
- """The ID of the output item that the content part was added to."""
1173
-
1174
- output_index: int
1175
- """The index of the output item that the content part was added to."""
1176
-
1177
- part: ResponseReasoningTextContent
1178
- """The content part that is done."""
1179
-
1180
- sequence_number: int
1181
- """The sequence number of this event."""
1182
-
1183
- type: Literal["response.reasoning_part.done"]
1184
- """The type of the event. Always `response.reasoning_part.done`."""
1185
-
1186
-
1187
-# TODO: this code can be removed once
1188
-# https://github.com/openai/openai-python/issues/2634 has been resolved
1189
-class ResponseReasoningPartAddedEvent(OpenAIBaseModel):
1190
- content_index: int
1191
- """The index of the content part that is done."""
1192
-
1193
- item_id: str
1194
- """The ID of the output item that the content part was added to."""
1195
-
1196
- output_index: int
1197
- """The index of the output item that the content part was added to."""
1198
-
1199
- part: ResponseReasoningTextContent
1200
- """The content part that is done."""
1201
-
1202
- sequence_number: int
1203
- """The sequence number of this event."""
1204
-
1205
- type: Literal["response.reasoning_part.added"]
1206
- """The type of the event. Always `response.reasoning_part.added`."""
1207
-
1208
-
1209
-# vLLM Streaming Events
1210
-# Note: we override the response type with the vLLM ResponsesResponse type
1211
-class ResponseCompletedEvent(OpenAIResponseCompletedEvent):
1212
- response: ResponsesResponse # type: ignore[override]
1213
-
1214
-
1215
-class ResponseCreatedEvent(OpenAIResponseCreatedEvent):
1216
- response: ResponsesResponse # type: ignore[override]
1217
-
1218
-
1219
-class ResponseInProgressEvent(OpenAIResponseInProgressEvent):
1220
- response: ResponsesResponse # type: ignore[override]
1221
-
1222
-
1223
-StreamingResponsesResponse: TypeAlias = (
1224
- ResponseCreatedEvent
1225
- | ResponseInProgressEvent
1226
- | ResponseCompletedEvent
1227
- | ResponseOutputItemAddedEvent
1228
- | ResponseOutputItemDoneEvent
1229
- | ResponseContentPartAddedEvent
1230
- | ResponseContentPartDoneEvent
1231
- | ResponseReasoningTextDeltaEvent
1232
- | ResponseReasoningTextDoneEvent
1233
- | ResponseReasoningPartAddedEvent
1234
- | ResponseReasoningPartDoneEvent
1235
- | ResponseCodeInterpreterCallInProgressEvent
1236
- | ResponseCodeInterpreterCallCodeDeltaEvent
1237
- | ResponseWebSearchCallInProgressEvent
1238
- | ResponseWebSearchCallSearchingEvent
1239
- | ResponseWebSearchCallCompletedEvent
1240
- | ResponseCodeInterpreterCallCodeDoneEvent
1241
- | ResponseCodeInterpreterCallInterpretingEvent
1242
- | ResponseCodeInterpreterCallCompletedEvent
1243
- | ResponseMcpCallArgumentsDeltaEvent
1244
- | ResponseMcpCallArgumentsDoneEvent
1245
- | ResponseMcpCallInProgressEvent
1246
- | ResponseMcpCallCompletedEvent
1247
-)
1248
-
1249
-
720
## Protocols for Audio
721
AudioResponseFormat: TypeAlias = Literal["json", "text", "srt", "verbose_json", "vtt"]
722
vllm/entrypoints/openai/engine/serving.py
+4
-2
@@ -50,13 +50,15 @@ from vllm.entrypoints.openai.engine.protocol import (
50
ErrorResponse,
51
FunctionCall,
52
FunctionDefinition,
53
- ResponseInputOutputItem,
54
- ResponsesRequest,
53
TranscriptionRequest,
54
TranscriptionResponse,
55
TranslationRequest,
56
VLLMValidationError,
57
)
58
+from vllm.entrypoints.openai.responses.protocol import (
59
+ ResponseInputOutputItem,
60
+ ResponsesRequest,
61
+)
62
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
63
from vllm.entrypoints.pooling.classify.protocol import (
64
ClassificationChatRequest,
vllm/entrypoints/openai/parser/harmony_utils.py
+1
-1
@@ -44,7 +44,7 @@ from openai_harmony import Role as OpenAIHarmonyRole
44
45
from vllm import envs
46
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionToolsParam
47
-from vllm.entrypoints.openai.engine.protocol import (
47
+from vllm.entrypoints.openai.responses.protocol import (
48
ResponseInputOutputItem,
49
ResponsesRequest,
50
)
vllm/entrypoints/openai/parser/responses_parser.py
+1
-1
@@ -16,7 +16,7 @@ from openai.types.responses.response_reasoning_item import (
16
)
17
18
from vllm.entrypoints.constants import MCP_PREFIX
19
-from vllm.entrypoints.openai.engine.protocol import (
19
+from vllm.entrypoints.openai.responses.protocol import (
20
ResponseInputOutputItem,
21
ResponsesRequest,
22
)
vllm/entrypoints/openai/responses/__init__.py
new
+2
@@ -0,0 +1,2 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
vllm/entrypoints/openai/responses/api_router.py
new
+143
@@ -0,0 +1,143 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+from collections.abc import AsyncGenerator
6
+from http import HTTPStatus
7
+
8
+from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
9
+from fastapi.responses import JSONResponse, StreamingResponse
10
+
11
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
12
+from vllm.entrypoints.openai.responses.protocol import (
13
+ ResponsesRequest,
14
+ ResponsesResponse,
15
+ StreamingResponsesResponse,
16
+)
17
+from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
18
+from vllm.entrypoints.openai.utils import validate_json_request
19
+from vllm.entrypoints.utils import (
20
+ with_cancellation,
21
+)
22
+from vllm.logger import init_logger
23
+
24
+logger = init_logger(__name__)
25
+
26
+router = APIRouter()
27
+
28
+
29
+def responses(request: Request) -> OpenAIServingResponses | None:
30
+ return request.app.state.openai_serving_responses
31
+
32
+
33
+async def _convert_stream_to_sse_events(
34
+ generator: AsyncGenerator[StreamingResponsesResponse, None],
35
+) -> AsyncGenerator[str, None]:
36
+ """Convert the generator to a stream of events in SSE format"""
37
+ async for event in generator:
38
+ event_type = getattr(event, "type", "unknown")
39
+ # https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#event_stream_format
40
+ event_data = (
41
+ f"event: {event_type}\ndata: {event.model_dump_json(indent=None)}\n\n"
42
+ )
43
+ yield event_data
44
+
45
+
46
+@router.post(
47
+ "/v1/responses",
48
+ dependencies=[Depends(validate_json_request)],
49
+ responses={
50
+ HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
51
+ HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
52
+ HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
53
+ HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
54
+ },
55
+)
56
+@with_cancellation
57
+async def create_responses(request: ResponsesRequest, raw_request: Request):
58
+ handler = responses(raw_request)
59
+ if handler is None:
60
+ base_server = raw_request.app.state.openai_serving_tokenization
61
+ return base_server.create_error_response(
62
+ message="The model does not support Responses API"
63
+ )
64
+ try:
65
+ generator = await handler.create_responses(request, raw_request)
66
+ except Exception as e:
67
+ raise HTTPException(
68
+ status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
69
+ ) from e
70
+
71
+ if isinstance(generator, ErrorResponse):
72
+ return JSONResponse(
73
+ content=generator.model_dump(), status_code=generator.error.code
74
+ )
75
+ elif isinstance(generator, ResponsesResponse):
76
+ return JSONResponse(content=generator.model_dump())
77
+
78
+ return StreamingResponse(
79
+ content=_convert_stream_to_sse_events(generator), media_type="text/event-stream"
80
+ )
81
+
82
+
83
+@router.get("/v1/responses/{response_id}")
84
+async def retrieve_responses(
85
+ response_id: str,
86
+ raw_request: Request,
87
+ starting_after: int | None = None,
88
+ stream: bool | None = False,
89
+):
90
+ handler = responses(raw_request)
91
+ if handler is None:
92
+ base_server = raw_request.app.state.openai_serving_tokenization
93
+ return base_server.create_error_response(
94
+ message="The model does not support Responses API"
95
+ )
96
+
97
+ try:
98
+ response = await handler.retrieve_responses(
99
+ response_id,
100
+ starting_after=starting_after,
101
+ stream=stream,
102
+ )
103
+ except Exception as e:
104
+ raise HTTPException(
105
+ status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
106
+ ) from e
107
+
108
+ if isinstance(response, ErrorResponse):
109
+ return JSONResponse(
110
+ content=response.model_dump(), status_code=response.error.code
111
+ )
112
+ elif isinstance(response, ResponsesResponse):
113
+ return JSONResponse(content=response.model_dump())
114
+ return StreamingResponse(
115
+ content=_convert_stream_to_sse_events(response), media_type="text/event-stream"
116
+ )
117
+
118
+
119
+@router.post("/v1/responses/{response_id}/cancel")
120
+async def cancel_responses(response_id: str, raw_request: Request):
121
+ handler = responses(raw_request)
122
+ if handler is None:
123
+ base_server = raw_request.app.state.openai_serving_tokenization
124
+ return base_server.create_error_response(
125
+ message="The model does not support Responses API"
126
+ )
127
+
128
+ try:
129
+ response = await handler.cancel_responses(response_id)
130
+ except Exception as e:
131
+ raise HTTPException(
132
+ status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
133
+ ) from e
134
+
135
+ if isinstance(response, ErrorResponse):
136
+ return JSONResponse(
137
+ content=response.model_dump(), status_code=response.error.code
138
+ )
139
+ return JSONResponse(content=response.model_dump())
140
+
141
+
142
+def attach_router(app: FastAPI):
143
+ app.include_router(router)
vllm/entrypoints/openai/responses/protocol.py
new
+554
@@ -0,0 +1,554 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+# Adapted from
5
+# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
6
+import time
7
+from typing import Any, Literal, TypeAlias
8
+
9
+from openai.types.responses import (
10
+ ResponseCodeInterpreterCallCodeDeltaEvent,
11
+ ResponseCodeInterpreterCallCodeDoneEvent,
12
+ ResponseCodeInterpreterCallCompletedEvent,
13
+ ResponseCodeInterpreterCallInProgressEvent,
14
+ ResponseCodeInterpreterCallInterpretingEvent,
15
+ ResponseContentPartAddedEvent,
16
+ ResponseContentPartDoneEvent,
17
+ ResponseFunctionToolCall,
18
+ ResponseInputItemParam,
19
+ ResponseMcpCallArgumentsDeltaEvent,
20
+ ResponseMcpCallArgumentsDoneEvent,
21
+ ResponseMcpCallCompletedEvent,
22
+ ResponseMcpCallInProgressEvent,
23
+ ResponseOutputItem,
24
+ ResponseOutputItemAddedEvent,
25
+ ResponseOutputItemDoneEvent,
26
+ ResponsePrompt,
27
+ ResponseReasoningTextDeltaEvent,
28
+ ResponseReasoningTextDoneEvent,
29
+ ResponseStatus,
30
+ ResponseWebSearchCallCompletedEvent,
31
+ ResponseWebSearchCallInProgressEvent,
32
+ ResponseWebSearchCallSearchingEvent,
33
+)
34
+from openai.types.responses import (
35
+ ResponseCompletedEvent as OpenAIResponseCompletedEvent,
36
+)
37
+from openai.types.responses import ResponseCreatedEvent as OpenAIResponseCreatedEvent
38
+from openai.types.responses import (
39
+ ResponseInProgressEvent as OpenAIResponseInProgressEvent,
40
+)
41
+from openai.types.responses.tool import Tool
42
+from openai_harmony import Message as OpenAIHarmonyMessage
43
+
44
+# Backward compatibility for OpenAI client versions
45
+try: # For older openai versions (< 1.100.0)
46
+ from openai.types.responses import ResponseTextConfig
47
+except ImportError: # For newer openai versions (>= 1.100.0)
48
+ from openai.types.responses import ResponseFormatTextConfig as ResponseTextConfig
49
+
50
+from openai.types.responses.response import IncompleteDetails, ToolChoice
51
+from openai.types.responses.response_reasoning_item import (
52
+ Content as ResponseReasoningTextContent,
53
+)
54
+from openai.types.shared import Metadata, Reasoning
55
+from pydantic import (
56
+ Field,
57
+ ValidationError,
58
+ field_serializer,
59
+ model_validator,
60
+)
61
+
62
+from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
63
+from vllm.entrypoints.openai.engine.protocol import (
64
+ OpenAIBaseModel,
65
+)
66
+from vllm.exceptions import VLLMValidationError
67
+from vllm.logger import init_logger
68
+from vllm.sampling_params import (
69
+ RequestOutputKind,
70
+ SamplingParams,
71
+ StructuredOutputsParams,
72
+)
73
+from vllm.utils import random_uuid
74
+
75
+logger = init_logger(__name__)
76
+
77
+
78
+class InputTokensDetails(OpenAIBaseModel):
79
+ cached_tokens: int
80
+ input_tokens_per_turn: list[int] = Field(default_factory=list)
81
+ cached_tokens_per_turn: list[int] = Field(default_factory=list)
82
+
83
+
84
+class OutputTokensDetails(OpenAIBaseModel):
85
+ reasoning_tokens: int = 0
86
+ tool_output_tokens: int = 0
87
+ output_tokens_per_turn: list[int] = Field(default_factory=list)
88
+ tool_output_tokens_per_turn: list[int] = Field(default_factory=list)
89
+
90
+
91
+class ResponseUsage(OpenAIBaseModel):
92
+ input_tokens: int
93
+ input_tokens_details: InputTokensDetails
94
+ output_tokens: int
95
+ output_tokens_details: OutputTokensDetails
96
+ total_tokens: int
97
+
98
+
99
+def serialize_message(msg):
100
+ """
101
+ Serializes a single message
102
+ """
103
+ if isinstance(msg, dict):
104
+ return msg
105
+ elif hasattr(msg, "to_dict"):
106
+ return msg.to_dict()
107
+ else:
108
+ # fallback to pyandic dump
109
+ return msg.model_dump_json()
110
+
111
+
112
+def serialize_messages(msgs):
113
+ """
114
+ Serializes multiple messages
115
+ """
116
+ return [serialize_message(msg) for msg in msgs] if msgs else None
117
+
118
+
119
+class ResponseRawMessageAndToken(OpenAIBaseModel):
120
+ """Class to show the raw message.
121
+ If message / tokens diverge, tokens is the source of truth"""
122
+
123
+ message: str
124
+ tokens: list[int]
125
+ type: Literal["raw_message_tokens"] = "raw_message_tokens"
126
+
127
+
128
+ResponseInputOutputMessage: TypeAlias = (
129
+ list[ChatCompletionMessageParam] | list[ResponseRawMessageAndToken]
130
+)
131
+ResponseInputOutputItem: TypeAlias = ResponseInputItemParam | ResponseOutputItem
132
+
133
+
134
+class ResponsesRequest(OpenAIBaseModel):
135
+ # Ordered by official OpenAI API documentation
136
+ # https://platform.openai.com/docs/api-reference/responses/create
137
+ background: bool | None = False
138
+ include: (
139
+ list[
140
+ Literal[
141
+ "code_interpreter_call.outputs",
142
+ "computer_call_output.output.image_url",
143
+ "file_search_call.results",
144
+ "message.input_image.image_url",
145
+ "message.output_text.logprobs",
146
+ "reasoning.encrypted_content",
147
+ ],
148
+ ]
149
+ | None
150
+ ) = None
151
+ input: str | list[ResponseInputOutputItem]
152
+ instructions: str | None = None
153
+ max_output_tokens: int | None = None
154
+ max_tool_calls: int | None = None
155
+ metadata: Metadata | None = None
156
+ model: str | None = None
157
+ logit_bias: dict[str, float] | None = None
158
+ parallel_tool_calls: bool | None = True
159
+ previous_response_id: str | None = None
160
+ prompt: ResponsePrompt | None = None
161
+ reasoning: Reasoning | None = None
162
+ service_tier: Literal["auto", "default", "flex", "scale", "priority"] = "auto"
163
+ store: bool | None = True
164
+ stream: bool | None = False
165
+ temperature: float | None = None
166
+ text: ResponseTextConfig | None = None
167
+ tool_choice: ToolChoice = "auto"
168
+ tools: list[Tool] = Field(default_factory=list)
169
+ top_logprobs: int | None = 0
170
+ top_p: float | None = None
171
+ top_k: int | None = None
172
+ truncation: Literal["auto", "disabled"] | None = "disabled"
173
+ user: str | None = None
174
+
175
+ # --8<-- [start:responses-extra-params]
176
+ request_id: str = Field(
177
+ default_factory=lambda: f"resp_{random_uuid()}",
178
+ description=(
179
+ "The request_id related to this request. If the caller does "
180
+ "not set it, a random_uuid will be generated. This id is used "
181
+ "through out the inference process and return in response."
182
+ ),
183
+ )
184
+ mm_processor_kwargs: dict[str, Any] | None = Field(
185
+ default=None,
186
+ description=("Additional kwargs to pass to the HF processor."),
187
+ )
188
+ priority: int = Field(
189
+ default=0,
190
+ description=(
191
+ "The priority of the request (lower means earlier handling; "
192
+ "default: 0). Any priority other than 0 will raise an error "
193
+ "if the served model does not use priority scheduling."
194
+ ),
195
+ )
196
+ cache_salt: str | None = Field(
197
+ default=None,
198
+ description=(
199
+ "If specified, the prefix cache will be salted with the provided "
200
+ "string to prevent an attacker to guess prompts in multi-user "
201
+ "environments. The salt should be random, protected from "
202
+ "access by 3rd parties, and long enough to be "
203
+ "unpredictable (e.g., 43 characters base64-encoded, corresponding "
204
+ "to 256 bit)."
205
+ ),
206
+ )
207
+
208
+ enable_response_messages: bool = Field(
209
+ default=False,
210
+ description=(
211
+ "Dictates whether or not to return messages as part of the "
212
+ "response object. Currently only supported for"
213
+ "non-background and gpt-oss only. "
214
+ ),
215
+ )
216
+ # similar to input_messages / output_messages in ResponsesResponse
217
+ # we take in previous_input_messages (ie in harmony format)
218
+ # this cannot be used in conjunction with previous_response_id
219
+ # TODO: consider supporting non harmony messages as well
220
+ previous_input_messages: list[OpenAIHarmonyMessage | dict] | None = None
221
+ # --8<-- [end:responses-extra-params]
222
+
223
+ _DEFAULT_SAMPLING_PARAMS = {
224
+ "temperature": 1.0,
225
+ "top_p": 1.0,
226
+ "top_k": 0,
227
+ }
228
+
229
+ def to_sampling_params(
230
+ self,
231
+ default_max_tokens: int,
232
+ default_sampling_params: dict | None = None,
233
+ ) -> SamplingParams:
234
+ if self.max_output_tokens is None:
235
+ max_tokens = default_max_tokens
236
+ else:
237
+ max_tokens = min(self.max_output_tokens, default_max_tokens)
238
+
239
+ default_sampling_params = default_sampling_params or {}
240
+ if (temperature := self.temperature) is None:
241
+ temperature = default_sampling_params.get(
242
+ "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
243
+ )
244
+ if (top_p := self.top_p) is None:
245
+ top_p = default_sampling_params.get(
246
+ "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
247
+ )
248
+ if (top_k := self.top_k) is None:
249
+ top_k = default_sampling_params.get(
250
+ "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
251
+ )
252
+ stop_token_ids = default_sampling_params.get("stop_token_ids")
253
+
254
+ # Structured output
255
+ structured_outputs = None
256
+ if self.text is not None and self.text.format is not None:
257
+ response_format = self.text.format
258
+ if (
259
+ response_format.type == "json_schema"
260
+ and response_format.schema_ is not None
261
+ ):
262
+ structured_outputs = StructuredOutputsParams(
263
+ json=response_format.schema_
264
+ )
265
+ elif response_format.type == "json_object":
266
+ raise NotImplementedError("json_object is not supported")
267
+
268
+ # TODO: add more parameters
269
+ return SamplingParams.from_optional(
270
+ temperature=temperature,
271
+ top_p=top_p,
272
+ top_k=top_k,
273
+ max_tokens=max_tokens,
274
+ logprobs=self.top_logprobs if self.is_include_output_logprobs() else None,
275
+ stop_token_ids=stop_token_ids,
276
+ output_kind=(
277
+ RequestOutputKind.DELTA if self.stream else RequestOutputKind.FINAL_ONLY
278
+ ),
279
+ structured_outputs=structured_outputs,
280
+ logit_bias=self.logit_bias,
281
+ skip_clone=True, # Created fresh per request, safe to skip clone
282
+ )
283
+
284
+ def is_include_output_logprobs(self) -> bool:
285
+ """Check if the request includes output logprobs."""
286
+ if self.include is None:
287
+ return False
288
+ return (
289
+ isinstance(self.include, list)
290
+ and "message.output_text.logprobs" in self.include
291
+ )
292
+
293
+ @model_validator(mode="before")
294
+ def validate_background(cls, data):
295
+ if not data.get("background"):
296
+ return data
297
+ if not data.get("store", True):
298
+ raise ValueError("background can only be used when `store` is true")
299
+ return data
300
+
301
+ @model_validator(mode="before")
302
+ def validate_prompt(cls, data):
303
+ if data.get("prompt") is not None:
304
+ raise VLLMValidationError(
305
+ "prompt template is not supported", parameter="prompt"
306
+ )
307
+ return data
308
+
309
+ @model_validator(mode="before")
310
+ def check_cache_salt_support(cls, data):
311
+ if data.get("cache_salt") is not None and (
312
+ not isinstance(data["cache_salt"], str) or not data["cache_salt"]
313
+ ):
314
+ raise ValueError(
315
+ "Parameter 'cache_salt' must be a non-empty string if provided."
316
+ )
317
+ return data
318
+
319
+ @model_validator(mode="before")
320
+ def function_call_parsing(cls, data):
321
+ """Parse function_call dictionaries into ResponseFunctionToolCall objects.
322
+ This ensures Pydantic can properly resolve union types in the input field.
323
+ Function calls provided as dicts are converted to ResponseFunctionToolCall
324
+ objects before validation, while invalid structures are left for Pydantic
325
+ to reject with appropriate error messages.
326
+ """
327
+
328
+ input_data = data.get("input")
329
+
330
+ # Early return for None, strings, or bytes
331
+ # (strings are iterable but shouldn't be processed)
332
+ if input_data is None or isinstance(input_data, (str, bytes)):
333
+ return data
334
+
335
+ # Convert iterators (like ValidatorIterator) to list
336
+ if not isinstance(input_data, list):
337
+ try:
338
+ input_data = list(input_data)
339
+ except TypeError:
340
+ # Not iterable, leave as-is for Pydantic to handle
341
+ return data
342
+
343
+ processed_input = []
344
+ for item in input_data:
345
+ if isinstance(item, dict) and item.get("type") == "function_call":
346
+ try:
347
+ processed_input.append(ResponseFunctionToolCall(**item))
348
+ except ValidationError:
349
+ # Let Pydantic handle validation for malformed function calls
350
+ logger.debug(
351
+ "Failed to parse function_call to ResponseFunctionToolCall, "
352
+ "leaving for Pydantic validation"
353
+ )
354
+ processed_input.append(item)
355
+ else:
356
+ processed_input.append(item)
357
+
358
+ data["input"] = processed_input
359
+ return data
360
+
361
+
362
+class ResponsesResponse(OpenAIBaseModel):
363
+ id: str = Field(default_factory=lambda: f"resp_{random_uuid()}")
364
+ created_at: int = Field(default_factory=lambda: int(time.time()))
365
+ # error: Optional[ResponseError] = None
366
+ incomplete_details: IncompleteDetails | None = None
367
+ instructions: str | None = None
368
+ metadata: Metadata | None = None
369
+ model: str
370
+ object: Literal["response"] = "response"
371
+ output: list[ResponseOutputItem]
372
+ parallel_tool_calls: bool
373
+ temperature: float
374
+ tool_choice: ToolChoice
375
+ tools: list[Tool]
376
+ top_p: float
377
+ background: bool
378
+ max_output_tokens: int
379
+ max_tool_calls: int | None = None
380
+ previous_response_id: str | None = None
381
+ prompt: ResponsePrompt | None = None
382
+ reasoning: Reasoning | None = None
383
+ service_tier: Literal["auto", "default", "flex", "scale", "priority"]
384
+ status: ResponseStatus
385
+ text: ResponseTextConfig | None = None
386
+ top_logprobs: int | None = None
387
+ truncation: Literal["auto", "disabled"]
388
+ usage: ResponseUsage | None = None
389
+ user: str | None = None
390
+
391
+ # --8<-- [start:responses-response-extra-params]
392
+ # These are populated when enable_response_messages is set to True
393
+ # NOTE: custom serialization is needed
394
+ # see serialize_input_messages and serialize_output_messages
395
+ input_messages: ResponseInputOutputMessage | None = Field(
396
+ default=None,
397
+ description=(
398
+ "If enable_response_messages, we can show raw token input to model."
399
+ ),
400
+ )
401
+ output_messages: ResponseInputOutputMessage | None = Field(
402
+ default=None,
403
+ description=(
404
+ "If enable_response_messages, we can show raw token output of model."
405
+ ),
406
+ )
407
+ # --8<-- [end:responses-response-extra-params]
408
+
409
+ # NOTE: openAI harmony doesn't serialize TextContent properly,
410
+ # TODO: this fixes for TextContent, but need to verify for tools etc
411
+ # https://github.com/openai/harmony/issues/78
412
+ @field_serializer("output_messages", when_used="json")
413
+ def serialize_output_messages(self, msgs, _info):
414
+ return serialize_messages(msgs)
415
+
416
+ # NOTE: openAI harmony doesn't serialize TextContent properly, this fixes it
417
+ # https://github.com/openai/harmony/issues/78
418
+ @field_serializer("input_messages", when_used="json")
419
+ def serialize_input_messages(self, msgs, _info):
420
+ return serialize_messages(msgs)
421
+
422
+ @classmethod
423
+ def from_request(
424
+ cls,
425
+ request: ResponsesRequest,
426
+ sampling_params: SamplingParams,
427
+ model_name: str,
428
+ created_time: int,
429
+ output: list[ResponseOutputItem],
430
+ status: ResponseStatus,
431
+ usage: ResponseUsage | None = None,
432
+ input_messages: ResponseInputOutputMessage | None = None,
433
+ output_messages: ResponseInputOutputMessage | None = None,
434
+ ) -> "ResponsesResponse":
435
+ incomplete_details: IncompleteDetails | None = None
436
+ if status == "incomplete":
437
+ incomplete_details = IncompleteDetails(reason="max_output_tokens")
438
+ # TODO: implement the other reason for incomplete_details,
439
+ # which is content_filter
440
+ # incomplete_details = IncompleteDetails(reason='content_filter')
441
+ return cls(
442
+ id=request.request_id,
443
+ created_at=created_time,
444
+ incomplete_details=incomplete_details,
445
+ instructions=request.instructions,
446
+ metadata=request.metadata,
447
+ model=model_name,
448
+ output=output,
449
+ input_messages=input_messages,
450
+ output_messages=output_messages,
451
+ parallel_tool_calls=request.parallel_tool_calls,
452
+ temperature=sampling_params.temperature,
453
+ tool_choice=request.tool_choice,
454
+ tools=request.tools,
455
+ top_p=sampling_params.top_p,
456
+ background=request.background,
457
+ max_output_tokens=sampling_params.max_tokens,
458
+ max_tool_calls=request.max_tool_calls,
459
+ previous_response_id=request.previous_response_id,
460
+ prompt=request.prompt,
461
+ reasoning=request.reasoning,
462
+ service_tier=request.service_tier,
463
+ status=status,
464
+ text=request.text,
465
+ top_logprobs=sampling_params.logprobs,
466
+ truncation=request.truncation,
467
+ user=request.user,
468
+ usage=usage,
469
+ )
470
+
471
+
472
+# TODO: this code can be removed once
473
+# https://github.com/openai/openai-python/issues/2634 has been resolved
474
+class ResponseReasoningPartDoneEvent(OpenAIBaseModel):
475
+ content_index: int
476
+ """The index of the content part that is done."""
477
+
478
+ item_id: str
479
+ """The ID of the output item that the content part was added to."""
480
+
481
+ output_index: int
482
+ """The index of the output item that the content part was added to."""
483
+
484
+ part: ResponseReasoningTextContent
485
+ """The content part that is done."""
486
+
487
+ sequence_number: int
488
+ """The sequence number of this event."""
489
+
490
+ type: Literal["response.reasoning_part.done"]
491
+ """The type of the event. Always `response.reasoning_part.done`."""
492
+
493
+
494
+# TODO: this code can be removed once
495
+# https://github.com/openai/openai-python/issues/2634 has been resolved
496
+class ResponseReasoningPartAddedEvent(OpenAIBaseModel):
497
+ content_index: int
498
+ """The index of the content part that is done."""
499
+
500
+ item_id: str
501
+ """The ID of the output item that the content part was added to."""
502
+
503
+ output_index: int
504
+ """The index of the output item that the content part was added to."""
505
+
506
+ part: ResponseReasoningTextContent
507
+ """The content part that is done."""
508
+
509
+ sequence_number: int
510
+ """The sequence number of this event."""
511
+
512
+ type: Literal["response.reasoning_part.added"]
513
+ """The type of the event. Always `response.reasoning_part.added`."""
514
+
515
+
516
+# vLLM Streaming Events
517
+# Note: we override the response type with the vLLM ResponsesResponse type
518
+class ResponseCompletedEvent(OpenAIResponseCompletedEvent):
519
+ response: ResponsesResponse # type: ignore[override]
520
+
521
+
522
+class ResponseCreatedEvent(OpenAIResponseCreatedEvent):
523
+ response: ResponsesResponse # type: ignore[override]
524
+
525
+
526
+class ResponseInProgressEvent(OpenAIResponseInProgressEvent):
527
+ response: ResponsesResponse # type: ignore[override]
528
+
529
+
530
+StreamingResponsesResponse: TypeAlias = (
531
+ ResponseCreatedEvent
532
+ | ResponseInProgressEvent
533
+ | ResponseCompletedEvent
534
+ | ResponseOutputItemAddedEvent
535
+ | ResponseOutputItemDoneEvent
536
+ | ResponseContentPartAddedEvent
537
+ | ResponseContentPartDoneEvent
538
+ | ResponseReasoningTextDeltaEvent
539
+ | ResponseReasoningTextDoneEvent
540
+ | ResponseReasoningPartAddedEvent
541
+ | ResponseReasoningPartDoneEvent
542
+ | ResponseCodeInterpreterCallInProgressEvent
543
+ | ResponseCodeInterpreterCallCodeDeltaEvent
544
+ | ResponseWebSearchCallInProgressEvent
545
+ | ResponseWebSearchCallSearchingEvent
546
+ | ResponseWebSearchCallCompletedEvent
547
+ | ResponseCodeInterpreterCallCodeDoneEvent
548
+ | ResponseCodeInterpreterCallInterpretingEvent
549
+ | ResponseCodeInterpreterCallCompletedEvent
550
+ | ResponseMcpCallArgumentsDeltaEvent
551
+ | ResponseMcpCallArgumentsDoneEvent
552
+ | ResponseMcpCallInProgressEvent
553
+ | ResponseMcpCallCompletedEvent
554
+)
vllm/entrypoints/openai/responses/serving.py
renamed
+14
-13
@@ -75,19 +75,7 @@ from vllm.entrypoints.logger import RequestLogger
75
from vllm.entrypoints.openai.engine.protocol import (
76
DeltaMessage,
77
ErrorResponse,
78
- InputTokensDetails,
79
- OutputTokensDetails,
78
RequestResponseMetadata,
81
- ResponseCompletedEvent,
82
- ResponseCreatedEvent,
83
- ResponseInProgressEvent,
84
- ResponseInputOutputMessage,
85
- ResponseReasoningPartAddedEvent,
86
- ResponseReasoningPartDoneEvent,
87
- ResponsesRequest,
88
- ResponsesResponse,
89
- ResponseUsage,
90
- StreamingResponsesResponse,
79
VLLMValidationError,
80
)
81
from vllm.entrypoints.openai.engine.serving import (
@@ -106,6 +94,20 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
94
parse_response_input,
95
render_for_completion,
96
)
97
+from vllm.entrypoints.openai.responses.protocol import (
98
+ InputTokensDetails,
99
+ OutputTokensDetails,
100
+ ResponseCompletedEvent,
101
+ ResponseCreatedEvent,
102
+ ResponseInProgressEvent,
103
+ ResponseInputOutputMessage,
104
+ ResponseReasoningPartAddedEvent,
105
+ ResponseReasoningPartDoneEvent,
106
+ ResponsesRequest,
107
+ ResponsesResponse,
108
+ ResponseUsage,
109
+ StreamingResponsesResponse,
110
+)
111
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
112
from vllm.entrypoints.responses_utils import (
113
construct_input_messages,
@@ -590,7 +592,6 @@ class OpenAIServingResponses(OpenAIServing):
592
prev_msg=self.msg_store.get(prev_response.id) if prev_response else None,
593
prev_response_output=prev_response.output if prev_response else None,
594
)
593
-
595
# Check if we should continue the final message (partial completion)
596
# This enables Anthropic-style partial message completion where the
597
# user provides an incomplete assistant message to continue from.
vllm/entrypoints/responses_utils.py
+2
-4
@@ -22,10 +22,8 @@ from openai.types.responses.tool import Tool
22
23
from vllm import envs
24
from vllm.entrypoints.constants import MCP_PREFIX
25
-from vllm.entrypoints.openai.engine.protocol import (
26
- ChatCompletionMessageParam,
27
- ResponseInputOutputItem,
28
-)
25
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionMessageParam
26
+from vllm.entrypoints.openai.responses.protocol import ResponseInputOutputItem
27
28
29
def should_continue_final_message(
vllm/reasoning/abs_reasoning_parsers.py
+2
@@ -19,6 +19,8 @@ if TYPE_CHECKING:
19
)
20
from vllm.entrypoints.openai.engine.protocol import (
21
DeltaMessage,
22
+ )
23
+ from vllm.entrypoints.openai.responses.protocol import (
24
ResponsesRequest,
25
)
26
from vllm.tokenizers import TokenizerLike
vllm/reasoning/basic_parsers.py
+1
-1
@@ -13,7 +13,7 @@ if TYPE_CHECKING:
13
from vllm.entrypoints.openai.chat_completion.protocol import (
14
ChatCompletionRequest,
15
)
16
- from vllm.entrypoints.openai.engine.protocol import (
16
+ from vllm.entrypoints.openai.responses.protocol import (
17
ResponsesRequest,
18
)
19
else:
vllm/reasoning/minimax_m2_reasoning_parser.py
+2
@@ -8,6 +8,8 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
8
)
9
from vllm.entrypoints.openai.engine.protocol import (
10
DeltaMessage,
11
+)
12
+from vllm.entrypoints.openai.responses.protocol import (
13
ResponsesRequest,
14
)
15
from vllm.logger import init_logger
vllm/reasoning/mistral_reasoning_parser.py
+1
-1
@@ -6,7 +6,7 @@ from functools import cached_property
6
from vllm.entrypoints.openai.chat_completion.protocol import (
7
ChatCompletionRequest,
8
)
9
-from vllm.entrypoints.openai.engine.protocol import (
9
+from vllm.entrypoints.openai.responses.protocol import (
10
ResponsesRequest,
11
)
12
from vllm.logger import init_logger
vllm/reasoning/olmo3_reasoning_parser.py
+2
@@ -15,6 +15,8 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
15
)
16
from vllm.entrypoints.openai.engine.protocol import (
17
DeltaMessage,
18
+)
19
+from vllm.entrypoints.openai.responses.protocol import (
20
ResponsesRequest,
21
)
22
from vllm.logger import init_logger
vllm/reasoning/qwen3_reasoning_parser.py
+3
-1
@@ -4,7 +4,9 @@
4
from vllm.entrypoints.openai.chat_completion.protocol import (
5
ChatCompletionRequest,
6
)
7
-from vllm.entrypoints.openai.engine.protocol import ResponsesRequest
7
+from vllm.entrypoints.openai.responses.protocol import (
8
+ ResponsesRequest,
9
+)
10
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
11
12
vllm/tool_parsers/abstract_tool_parser.py
+2
@@ -14,6 +14,8 @@ from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionReque
14
from vllm.entrypoints.openai.engine.protocol import (
15
DeltaMessage,
16
ExtractedToolCallInformation,
17
+)
18
+from vllm.entrypoints.openai.responses.protocol import (
19
ResponsesRequest,
20
ResponseTextConfig,
21
)