| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | # Adapted from |
| 5 | # https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py |
| 6 | import time |
| 7 | from typing import Annotated, Any, ClassVar, Literal |
| 8 | |
| 9 | from openai.types.chat.chat_completion_audio import ( |
| 10 | ChatCompletionAudio as OpenAIChatCompletionAudio, |
| 11 | ) |
| 12 | from openai.types.chat.chat_completion_message import Annotation as OpenAIAnnotation |
| 13 | from pydantic import ( |
| 14 | Field, |
| 15 | PrivateAttr, |
| 16 | SerializeAsAny, |
| 17 | model_serializer, |
| 18 | model_validator, |
| 19 | ) |
| 20 | |
| 21 | from vllm.config import ModelConfig |
| 22 | from vllm.entrypoints.chat_utils import ( |
| 23 | ChatCompletionMessageParam, |
| 24 | ChatTemplateContentFormatOption, |
| 25 | has_non_text_content, |
| 26 | ) |
| 27 | from vllm.entrypoints.generate.base.protocol import ( |
| 28 | AnyResponseFormat, |
| 29 | DeltaMessage, |
| 30 | FunctionCall, |
| 31 | FunctionDefinition, |
| 32 | PerRequestMetrics, |
| 33 | StopParam, |
| 34 | StreamOptions, |
| 35 | ToolCall, |
| 36 | structured_outputs_from_response_format, |
| 37 | validate_cache_salt, |
| 38 | validate_structural_tag_response_format, |
| 39 | validate_structured_outputs_structural_tag, |
| 40 | ) |
| 41 | from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo |
| 42 | from vllm.exceptions import VLLMValidationError |
| 43 | from vllm.logger import init_logger |
| 44 | from vllm.logprobs import Logprob |
| 45 | from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs |
| 46 | from vllm.sampling_params import ( |
| 47 | BeamSearchParams, |
| 48 | RepetitionDetectionParams, |
| 49 | RequestOutputKind, |
| 50 | SamplingParams, |
| 51 | StructuredOutputsParams, |
| 52 | ThinkingTokenBudget, |
| 53 | ) |
| 54 | from vllm.utils import random_uuid |
| 55 | |
| 56 | logger = init_logger(__name__) |
| 57 | |
| 58 | |
| 59 | _INT64_MIN = -(2**63) |
| 60 | _INT64_MAX = 2**63 - 1 |
| 61 | |
| 62 | |
| 63 | class ChatMessage(OpenAIBaseModel): |
| 64 | role: str |
| 65 | content: str | None = None |
| 66 | refusal: str | None = None |
| 67 | annotations: OpenAIAnnotation | None = None |
| 68 | audio: OpenAIChatCompletionAudio | None = None |
| 69 | function_call: FunctionCall | None = None |
| 70 | tool_calls: list[ToolCall] = Field(default_factory=list) |
| 71 | |
| 72 | # vLLM-specific fields that are not in OpenAI spec |
| 73 | reasoning: str | None = None |
| 74 | |
| 75 | @model_serializer(mode="wrap") |
| 76 | def _serialize(self, handler): |
| 77 | data = handler(self) |
| 78 | if len(data.get("tool_calls", [])) == 0: |
| 79 | data.pop("tool_calls", None) |
| 80 | return data |
| 81 | |
| 82 | |
| 83 | class ChatCompletionLogProb(OpenAIBaseModel): |
| 84 | token: str |
| 85 | logprob: float = -9999.0 |
| 86 | bytes: list[int] | None = None |
| 87 | |
| 88 | |
| 89 | class ChatCompletionLogProbsContent(ChatCompletionLogProb): |
| 90 | # Workaround: redefine fields name cache so that it's not |
| 91 | # shared with the super class. |
| 92 | field_names: ClassVar[set[str] | None] = None |
| 93 | top_logprobs: list[ChatCompletionLogProb] = Field(default_factory=list) |
| 94 | |
| 95 | |
| 96 | class ChatCompletionLogProbs(OpenAIBaseModel): |
| 97 | content: list[ChatCompletionLogProbsContent] | None = None |
| 98 | |
| 99 | |
| 100 | class ChatCompletionResponseChoice(OpenAIBaseModel): |
| 101 | index: int |
| 102 | # ``SerializeAsAny`` lets pydantic honor subclasses of ``ChatMessage`` |
| 103 | # (e.g. ``vllm.entrypoints.cohere.cohere_chat_message.CohereChatMessage``) |
| 104 | # so that added fields like ``citations`` survive JSON serialization |
| 105 | # instead of being stripped down to the base schema. Plain |
| 106 | # ``ChatMessage`` instances serialize identically to before. |
| 107 | message: SerializeAsAny[ChatMessage] |
| 108 | logprobs: ChatCompletionLogProbs | None = None |
| 109 | # per OpenAI spec this is the default |
| 110 | finish_reason: str | None = "stop" |
| 111 | # not part of the OpenAI spec but included in vLLM for legacy reasons |
| 112 | stop_reason: int | str | None = None |
| 113 | # not part of the OpenAI spec but is useful for tracing the tokens |
| 114 | # in agent scenarios |
| 115 | token_ids: list[int] | None = None |
| 116 | # Per-token expert routing decisions, base64-encoded ``.npy`` bytes |
| 117 | # (numpy serialization). Shape after decode: |
| 118 | # (num_tokens - 1, num_layers, num_experts_per_tok) dtype uint8/uint16/int32 |
| 119 | # ``num_tokens - 1`` because the last sampled token has not been |
| 120 | # forwarded yet and therefore has no routing data. |
| 121 | # Decode: |
| 122 | # np.load(io.BytesIO(base64.b64decode(s))) |
| 123 | # ``None`` if (a) the request was aborted before any forward pass, |
| 124 | # or (b) ``enable_return_routed_experts`` is off server-side. |
| 125 | routed_experts: str | None = None |
| 126 | |
| 127 | |
| 128 | class ChatCompletionResponse(OpenAIBaseModel): |
| 129 | id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}") |
| 130 | object: Literal["chat.completion"] = "chat.completion" |
| 131 | created: int = Field(default_factory=lambda: int(time.time())) |
| 132 | model: str |
| 133 | choices: list[ChatCompletionResponseChoice] |
| 134 | service_tier: Literal["auto", "default", "flex", "scale", "priority"] | None = None |
| 135 | system_fingerprint: str | None = None |
| 136 | usage: UsageInfo |
| 137 | |
| 138 | # vLLM-specific fields that are not in OpenAI spec |
| 139 | prompt_logprobs: list[dict[int, Logprob] | None] | None = None |
| 140 | prompt_token_ids: list[int] | None = None |
| 141 | # Rendered prompt text from chat templating (only set when |
| 142 | # ``return_prompt_text=True`` on the request). |
| 143 | prompt_text: str | None = None |
| 144 | kv_transfer_params: dict[str, Any] | None = Field( |
| 145 | default=None, description="KVTransfer parameters." |
| 146 | ) |
| 147 | ec_transfer_params: dict[str, Any] | None = Field( |
| 148 | default=None, description="ECTransfer parameters." |
| 149 | ) |
| 150 | metrics: PerRequestMetrics | None = None |
| 151 | |
| 152 | |
| 153 | class ChatCompletionResponseStreamChoice(OpenAIBaseModel): |
| 154 | index: int |
| 155 | # ``SerializeAsAny`` lets pydantic honor subclasses of ``DeltaMessage`` |
| 156 | # (e.g. ``vllm.entrypoints.cohere.cohere_chat_message.CohereDeltaMessage``) |
| 157 | # so streaming ``citations`` survive JSON serialization. Plain |
| 158 | # ``DeltaMessage`` instances serialize identically to before. |
| 159 | delta: SerializeAsAny[DeltaMessage] |
| 160 | logprobs: ChatCompletionLogProbs | None = None |
| 161 | finish_reason: str | None = None |
| 162 | stop_reason: int | str | None = None |
| 163 | # not part of the OpenAI spec but for tracing the tokens |
| 164 | token_ids: list[int] | None = None |
| 165 | |
| 166 | |
| 167 | class ChatCompletionStreamResponse(OpenAIBaseModel): |
| 168 | id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}") |
| 169 | object: Literal["chat.completion.chunk"] = "chat.completion.chunk" |
| 170 | created: int = Field(default_factory=lambda: int(time.time())) |
| 171 | model: str |
| 172 | choices: list[ChatCompletionResponseStreamChoice] |
| 173 | usage: UsageInfo | None = Field(default=None) |
| 174 | # Set only on the final chunk of a stream to mirror non-streaming responses |
| 175 | # without the per-chunk serialization overhead. |
| 176 | system_fingerprint: str | None = None |
| 177 | # not part of the OpenAI spec but for tracing the tokens |
| 178 | prompt_token_ids: list[int] | None = None |
| 179 | # Rendered prompt text from chat templating (only set when |
| 180 | # ``return_prompt_text=True`` on the request); only sent on the first chunk. |
| 181 | prompt_text: str | None = None |
| 182 | metrics: PerRequestMetrics | None = None |
| 183 | |
| 184 | |
| 185 | class ChatCompletionToolsParam(OpenAIBaseModel): |
| 186 | type: Literal["function"] = "function" |
| 187 | function: FunctionDefinition |
| 188 | defer_loading: bool | None = None |
| 189 | |
| 190 | @model_validator(mode="after") |
| 191 | def _propagate_defer_loading(self) -> "ChatCompletionToolsParam": |
| 192 | if self.defer_loading is not None and self.function.defer_loading is None: |
| 193 | self.function.defer_loading = self.defer_loading |
| 194 | return self |
| 195 | |
| 196 | @model_serializer(mode="wrap") |
| 197 | def _serialize(self, handler): |
| 198 | data = handler(self) |
| 199 | data = {k: v for k, v in data.items() if k in type(self).model_fields} |
| 200 | if self.defer_loading is None: |
| 201 | data.pop("defer_loading", None) |
| 202 | return data |
| 203 | |
| 204 | |
| 205 | class ChatCompletionNamedFunction(OpenAIBaseModel): |
| 206 | name: str |
| 207 | |
| 208 | |
| 209 | class ChatCompletionNamedToolChoiceParam(OpenAIBaseModel): |
| 210 | function: ChatCompletionNamedFunction |
| 211 | type: Literal["function"] = "function" |
| 212 | |
| 213 | |
| 214 | class ChatCompletionRequest(OpenAIBaseModel): |
| 215 | # Ordered by official OpenAI API documentation |
| 216 | # https://platform.openai.com/docs/api-reference/chat/create |
| 217 | messages: list[ChatCompletionMessageParam] |
| 218 | model: str | None = None |
| 219 | frequency_penalty: float | None = None |
| 220 | logit_bias: dict[str, float] | None = None |
| 221 | logprobs: bool | None = False |
| 222 | top_logprobs: int | None = 0 |
| 223 | max_tokens: int | None = Field( |
| 224 | default=None, |
| 225 | deprecated="max_tokens is deprecated in favor of " |
| 226 | "the max_completion_tokens field", |
| 227 | ) |
| 228 | max_completion_tokens: int | None = None |
| 229 | n: int | None = 1 |
| 230 | presence_penalty: float | None = None |
| 231 | response_format: AnyResponseFormat | None = None |
| 232 | seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX) |
| 233 | stop: StopParam = [] |
| 234 | stream: bool | None = False |
| 235 | stream_options: StreamOptions | None = None |
| 236 | temperature: float | None = None |
| 237 | top_p: float | None = None |
| 238 | tools: list[ChatCompletionToolsParam] | None = None |
| 239 | tool_choice: ( |
| 240 | Literal["none"] |
| 241 | | Literal["auto"] |
| 242 | | Literal["required"] |
| 243 | | ChatCompletionNamedToolChoiceParam |
| 244 | | None |
| 245 | ) = "none" |
| 246 | reasoning_effort: ( |
| 247 | Literal["none", "minimal", "low", "medium", "high", "xhigh", "max"] | None |
| 248 | ) = Field( |
| 249 | default=None, |
| 250 | description=( |
| 251 | "Constrains effort on reasoning for reasoning models. " |
| 252 | "Currently supported values are none, minimal, low, medium, " |
| 253 | "high, xhigh, and max. Reducing reasoning effort can result in " |
| 254 | "faster responses and fewer tokens used on reasoning in a response. " |
| 255 | "Note that 'max' is specific to the DeepSeek V4 series and is not " |
| 256 | "part of the standard OpenAI API specification." |
| 257 | ), |
| 258 | ) |
| 259 | thinking_token_budget: ThinkingTokenBudget = None |
| 260 | include_reasoning: bool = True |
| 261 | parallel_tool_calls: bool | None = True |
| 262 | |
| 263 | # NOTE this will be ignored by vLLM |
| 264 | user: str | None = None |
| 265 | |
| 266 | # --8<-- [start:chat-completion-sampling-params] |
| 267 | use_beam_search: bool = False |
| 268 | top_k: int | None = None |
| 269 | min_p: float | None = None |
| 270 | repetition_penalty: float | None = None |
| 271 | watermarking: bool = True |
| 272 | length_penalty: float = 1.0 |
| 273 | stop_token_ids: list[int] | None = [] |
| 274 | include_stop_str_in_output: bool = False |
| 275 | ignore_eos: bool = False |
| 276 | min_tokens: int = 0 |
| 277 | skip_special_tokens: bool = True |
| 278 | spaces_between_special_tokens: bool = True |
| 279 | truncate_prompt_tokens: Annotated[int, Field(ge=-1, le=_INT64_MAX)] | None = None |
| 280 | truncation_side: Literal["left", "right"] | None = Field( |
| 281 | default=None, |
| 282 | description=( |
| 283 | "Which side to truncate from when truncate_prompt_tokens is active. " |
| 284 | "'right' keeps the first N tokens. " |
| 285 | "'left' keeps the last N tokens." |
| 286 | ), |
| 287 | ) |
| 288 | prompt_logprobs: int | None = None |
| 289 | logprob_token_ids: list[int] | None = Field( |
| 290 | default=None, |
| 291 | description=( |
| 292 | "Specific vocab token IDs to return logprobs for at each generated " |
| 293 | "position, in addition to the sampled token. More efficient than " |
| 294 | "`top_logprobs=-1` when only a small fixed label set is needed " |
| 295 | "(e.g. multilabel scoring " |
| 296 | "where each label corresponds to a known vocab id). When set, " |
| 297 | "this explicit token selection takes precedence over the natural " |
| 298 | "top-k selected by `top_logprobs`. Requires `logprobs=True`." |
| 299 | ), |
| 300 | ) |
| 301 | allowed_token_ids: list[int] | None = None |
| 302 | bad_words: list[str] = Field(default_factory=list) |
| 303 | # --8<-- [end:chat-completion-sampling-params] |
| 304 | |
| 305 | # --8<-- [start:chat-completion-extra-params] |
| 306 | echo: bool = Field( |
| 307 | default=False, |
| 308 | description=( |
| 309 | "If true, the new message will be prepended with the last message " |
| 310 | "if they belong to the same role." |
| 311 | ), |
| 312 | ) |
| 313 | add_generation_prompt: bool = Field( |
| 314 | default=True, |
| 315 | description=( |
| 316 | "If true, the generation prompt will be added to the chat template. " |
| 317 | "This is a parameter used by chat template in tokenizer config of the " |
| 318 | "model." |
| 319 | ), |
| 320 | ) |
| 321 | continue_final_message: bool = Field( |
| 322 | default=False, |
| 323 | description=( |
| 324 | "If this is set, the chat will be formatted so that the final " |
| 325 | "message in the chat is open-ended, without any EOS tokens. The " |
| 326 | "model will continue this message rather than starting a new one. " |
| 327 | 'This allows you to "prefill" part of the model\'s response for it. ' |
| 328 | "Cannot be used at the same time as `add_generation_prompt`." |
| 329 | ), |
| 330 | ) |
| 331 | add_special_tokens: bool = Field( |
| 332 | default=False, |
| 333 | description=( |
| 334 | "If true, special tokens (e.g. BOS) will be added to the prompt " |
| 335 | "on top of what is added by the chat template. " |
| 336 | "For most models, the chat template takes care of adding the " |
| 337 | "special tokens so this should be set to false (as is the " |
| 338 | "default)." |
| 339 | ), |
| 340 | ) |
| 341 | documents: list[dict[str, str]] | None = Field( |
| 342 | default=None, |
| 343 | description=( |
| 344 | "A list of dicts representing documents that will be accessible to " |
| 345 | "the model if it is performing RAG (retrieval-augmented generation)." |
| 346 | " If the template does not support RAG, this argument will have no " |
| 347 | "effect. We recommend that each document should be a dict containing " |
| 348 | '"title" and "text" keys.' |
| 349 | ), |
| 350 | ) |
| 351 | chat_template: str | None = Field( |
| 352 | default=None, |
| 353 | description=( |
| 354 | "A Jinja template to use for this conversion. " |
| 355 | "As of transformers v4.44, default chat template is no longer " |
| 356 | "allowed, so you must provide a chat template if the tokenizer " |
| 357 | "does not define one." |
| 358 | ), |
| 359 | ) |
| 360 | chat_template_kwargs: dict[str, Any] | None = Field( |
| 361 | default=None, |
| 362 | description=( |
| 363 | "Additional keyword args to pass to the template renderer. " |
| 364 | "Will be accessible by the chat template." |
| 365 | ), |
| 366 | ) |
| 367 | media_io_kwargs: dict[str, dict[str, Any]] | None = Field( |
| 368 | default=None, |
| 369 | description=( |
| 370 | "Additional kwargs to pass to the media IO connectors, " |
| 371 | "keyed by modality. Merged with engine-level media_io_kwargs." |
| 372 | ), |
| 373 | ) |
| 374 | mm_processor_kwargs: dict[str, Any] | None = Field( |
| 375 | default=None, |
| 376 | description=("Additional kwargs to pass to the HF processor."), |
| 377 | ) |
| 378 | structured_outputs: StructuredOutputsParams | None = Field( |
| 379 | default=None, |
| 380 | description="Additional kwargs for structured outputs", |
| 381 | ) |
| 382 | priority: int = Field( |
| 383 | default=0, |
| 384 | ge=_INT64_MIN, |
| 385 | le=_INT64_MAX, |
| 386 | description=( |
| 387 | "The priority of the request (lower means earlier handling; " |
| 388 | "default: 0). Any priority other than 0 will raise an error " |
| 389 | "if the served model does not use priority scheduling." |
| 390 | ), |
| 391 | ) |
| 392 | request_id: str = Field( |
| 393 | default_factory=random_uuid, |
| 394 | description=( |
| 395 | "The request_id related to this request. If the caller does " |
| 396 | "not set it, a random_uuid will be generated. This id is used " |
| 397 | "through out the inference process and return in response." |
| 398 | ), |
| 399 | ) |
| 400 | session_id: str | None = Field( |
| 401 | default=None, |
| 402 | description=( |
| 403 | "Stable session identity shared by related requests. Unlike " |
| 404 | "request_id, this value is expected to remain stable across " |
| 405 | "multiple requests in the same conversation or agent session." |
| 406 | ), |
| 407 | ) |
| 408 | |
| 409 | return_tokens_as_token_ids: bool | None = Field( |
| 410 | default=None, |
| 411 | description=( |
| 412 | "If specified with 'logprobs', tokens are represented " |
| 413 | " as strings of the form 'token_id:{token_id}' so that tokens " |
| 414 | "that are not JSON-encodable can be identified." |
| 415 | ), |
| 416 | ) |
| 417 | return_token_ids: bool | None = Field( |
| 418 | default=None, |
| 419 | description=( |
| 420 | "If specified, the result will include token IDs alongside the " |
| 421 | "generated text. In streaming mode, prompt_token_ids is included " |
| 422 | "only in the first chunk, and token_ids contains the delta tokens " |
| 423 | "for each chunk. This is useful for debugging or when you " |
| 424 | "need to map generated text back to input tokens." |
| 425 | ), |
| 426 | ) |
| 427 | routed_experts_prompt_start: int = Field( |
| 428 | default=0, |
| 429 | ge=0, |
| 430 | description="Skip the first N prompt tokens from returned routed-expert data.", |
| 431 | ) |
| 432 | return_token_offsets: bool | None = Field( |
| 433 | default=False, |
| 434 | description=( |
| 435 | "If true, return char-level (start, end) offsets for each " |
| 436 | "token relative to the tokenized source string in the " |
| 437 | "`token_offsets` field of the rendered response. Only " |
| 438 | "supported on the `/v1/completions/render` and " |
| 439 | "`/v1/chat/completions/render` endpoints; ignored on regular " |
| 440 | "generation endpoints. Honored only for Fast (Rust-backed) " |
| 441 | "tokenizers; otherwise `token_offsets` is null. For chat " |
| 442 | "requests, offsets are relative to the templated prompt " |
| 443 | "string (after applying the chat template). Multimodal " |
| 444 | "inputs and pre-tokenized inputs always yield null." |
| 445 | ), |
| 446 | ) |
| 447 | return_prompt_text: bool | None = Field( |
| 448 | default=None, |
| 449 | description=( |
| 450 | "If true, the response will include ``prompt_text`` containing the " |
| 451 | "prompt string produced by chat templating. In streaming mode it " |
| 452 | "is sent only on the first chunk. This is useful for inspecting " |
| 453 | "exactly what was fed into the model." |
| 454 | ), |
| 455 | ) |
| 456 | |
| 457 | cache_salt: str | None = Field( |
| 458 | default=None, |
| 459 | min_length=1, |
| 460 | max_length=1024, |
| 461 | description=( |
| 462 | "If specified, the prefix cache will be salted with the provided " |
| 463 | "string to prevent an attacker to guess prompts in multi-user " |
| 464 | "environments. The salt should be random, protected from " |
| 465 | "access by 3rd parties, and long enough to be " |
| 466 | "unpredictable (e.g., 43 characters base64-encoded, corresponding " |
| 467 | "to 256 bit)." |
| 468 | ), |
| 469 | ) |
| 470 | |
| 471 | kv_transfer_params: dict[str, Any] | None = Field( |
| 472 | default=None, |
| 473 | description="KVTransfer parameters used for disaggregated serving.", |
| 474 | ) |
| 475 | |
| 476 | ec_transfer_params: dict[str, Any] | None = Field( |
| 477 | default=None, |
| 478 | description=( |
| 479 | "ECTransfer parameters used for encoder-cache disaggregated serving." |
| 480 | ), |
| 481 | ) |
| 482 | |
| 483 | vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = Field( |
| 484 | default=None, |
| 485 | description=( |
| 486 | "Additional request parameters with (list of) string or " |
| 487 | "numeric values, used by custom extensions." |
| 488 | ), |
| 489 | ) |
| 490 | |
| 491 | repetition_detection: RepetitionDetectionParams | None = Field( |
| 492 | default=None, |
| 493 | description="Parameters for detecting repetitive N-gram patterns " |
| 494 | "in output tokens. If such repetition is detected, generation will " |
| 495 | "be ended early. LLMs can sometimes generate repetitive, unhelpful " |
| 496 | "token patterns, stopping only when they hit the maximum output length " |
| 497 | "(e.g. 'abcdabcdabcd...' or '\\emoji \\emoji \\emoji ...'). This feature " |
| 498 | "can detect such behavior and terminate early, saving time and tokens.", |
| 499 | ) |
| 500 | |
| 501 | stream_interval: Annotated[int, Field(ge=1)] | None = Field( |
| 502 | default=None, |
| 503 | description=( |
| 504 | "Number of tokens to batch into each streamed chunk. Raises the " |
| 505 | "server's `--stream-interval` for this request. Values below the " |
| 506 | "server setting are clamped up to it. The first and last chunks " |
| 507 | "are always sent immediately. Ignored for non-streaming requests." |
| 508 | ), |
| 509 | ) |
| 510 | |
| 511 | # --8<-- [end:chat-completion-extra-params] |
| 512 | |
| 513 | @model_validator(mode="before") |
| 514 | @classmethod |
| 515 | def check_cache_salt_support(cls, data: Any) -> Any: |
| 516 | if not isinstance(data, dict): |
| 517 | return data |
| 518 | validate_cache_salt(data.get("cache_salt")) |
| 519 | return data |
| 520 | |
| 521 | @model_validator(mode="before") |
| 522 | @classmethod |
| 523 | def _normalize_messages_before(cls, data: Any) -> Any: |
| 524 | """Pre-process message dicts before Pydantic field validation. |
| 525 | |
| 526 | Performs two normalizations in a single pass: |
| 527 | - Converts tool_calls generators/iterators to lists so one-shot |
| 528 | generators are not consumed during union type matching. |
| 529 | - Renames the deprecated ``reasoning_content`` field to |
| 530 | ``reasoning`` so downstream code only needs to check one field. |
| 531 | """ |
| 532 | if not isinstance(data, dict): |
| 533 | return data |
| 534 | messages = data.get("messages") |
| 535 | if not isinstance(messages, list): |
| 536 | return data |
| 537 | for msg in messages: |
| 538 | if not isinstance(msg, dict): |
| 539 | continue |
| 540 | tool_calls = msg.get("tool_calls") |
| 541 | if tool_calls is not None and not isinstance(tool_calls, list): |
| 542 | msg["tool_calls"] = list(tool_calls) |
| 543 | reasoning_content = msg.pop("reasoning_content", None) |
| 544 | if reasoning_content is not None and msg.get("reasoning") is None: |
| 545 | msg["reasoning"] = reasoning_content |
| 546 | return data |
| 547 | |
| 548 | @model_validator(mode="after") |
| 549 | def _materialize_tool_calls_after(self) -> "ChatCompletionRequest": |
| 550 | """Convert Pydantic ValidatorIterator wrappers back to lists. |
| 551 | |
| 552 | Even after the "before" validator converts iterables to lists, |
| 553 | Pydantic re-wraps them in a ValidatorIterator when validating |
| 554 | against ChatCompletionAssistantMessageParam's Iterable[...] type. |
| 555 | This "after" pass materialises those wrappers so downstream code |
| 556 | (tokenizers, model_dump_json) always sees plain lists. |
| 557 | """ |
| 558 | for msg in self.messages: |
| 559 | if not isinstance(msg, dict): |
| 560 | continue |
| 561 | tool_calls = msg.get("tool_calls") |
| 562 | if tool_calls is not None and not isinstance(tool_calls, list): |
| 563 | msg["tool_calls"] = list(tool_calls) |
| 564 | return self |
| 565 | |
| 566 | _grammar_from_parser: bool = PrivateAttr(default=False) |
| 567 | """CAUTION: Should only be set by the parser-engine adapter's adjust_request.""" |
| 568 | |
| 569 | def build_chat_params( |
| 570 | self, |
| 571 | default_template: str | None, |
| 572 | default_template_content_format: ChatTemplateContentFormatOption, |
| 573 | ) -> ChatParams: |
| 574 | extra_kwargs: dict[str, Any] = dict( |
| 575 | add_generation_prompt=self.add_generation_prompt, |
| 576 | continue_final_message=self.continue_final_message, |
| 577 | documents=self.documents, |
| 578 | reasoning_effort=self.reasoning_effort, |
| 579 | ) |
| 580 | |
| 581 | # When reasoning is requested, activate thinking for models whose |
| 582 | # chat templates require explicit opt-in (e.g., Gemma4 defaults |
| 583 | # enable_thinking to false). For templates that don't declare the |
| 584 | # variable, resolve_chat_template_kwargs filters it out harmlessly. |
| 585 | user_kwargs = self.chat_template_kwargs or {} |
| 586 | if self.reasoning_effort is not None and "enable_thinking" not in user_kwargs: |
| 587 | extra_kwargs["enable_thinking"] = self.reasoning_effort != "none" |
| 588 | |
| 589 | return ChatParams( |
| 590 | chat_template=self.chat_template or default_template, |
| 591 | chat_template_content_format=default_template_content_format, |
| 592 | chat_template_kwargs=merge_kwargs( |
| 593 | self.chat_template_kwargs, |
| 594 | extra_kwargs, |
| 595 | ), |
| 596 | media_io_kwargs=self.media_io_kwargs, |
| 597 | # No-tools requests default to tool_choice="none" at the API |
| 598 | # layer. Collapse that default before rendering, so K3 emits a |
| 599 | # model-visible tool-choice instruction only for requests with a |
| 600 | # tools block. |
| 601 | tool_choice=self.tool_choice if self.tools else None, |
| 602 | response_format=self.response_format, |
| 603 | ) |
| 604 | |
| 605 | def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams: |
| 606 | if self.max_completion_tokens is not None: |
| 607 | max_output_tokens: int | None = self.max_completion_tokens |
| 608 | max_output_tokens_param = "max_completion_tokens" |
| 609 | else: |
| 610 | max_output_tokens = self.max_tokens |
| 611 | max_output_tokens_param = "max_tokens" |
| 612 | |
| 613 | return TokenizeParams( |
| 614 | max_total_tokens=model_config.max_model_len, |
| 615 | max_output_tokens=max_output_tokens or 0, |
| 616 | truncate_prompt_tokens=self.truncate_prompt_tokens, |
| 617 | truncation_side=self.truncation_side, |
| 618 | add_special_tokens=self.add_special_tokens, |
| 619 | needs_detokenization=bool(self.echo and not self.return_token_ids), |
| 620 | max_total_tokens_param="max_model_len", |
| 621 | max_output_tokens_param=max_output_tokens_param, |
| 622 | return_token_offsets=bool(self.return_token_offsets), |
| 623 | ) |
| 624 | |
| 625 | # Default sampling parameters for chat completion requests |
| 626 | _DEFAULT_SAMPLING_PARAMS: dict = { |
| 627 | "repetition_penalty": 1.0, |
| 628 | "presence_penalty": 0.0, |
| 629 | "frequency_penalty": 0.0, |
| 630 | "temperature": 1.0, |
| 631 | "top_p": 1.0, |
| 632 | "top_k": 0, |
| 633 | "min_p": 0.0, |
| 634 | } |
| 635 | |
| 636 | def to_beam_search_params( |
| 637 | self, max_tokens: int, default_sampling_params: dict |
| 638 | ) -> BeamSearchParams: |
| 639 | n = self.n if self.n is not None else 1 |
| 640 | if (temperature := self.temperature) is None: |
| 641 | temperature = default_sampling_params.get( |
| 642 | "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"] |
| 643 | ) |
| 644 | |
| 645 | return BeamSearchParams( |
| 646 | beam_width=n, |
| 647 | max_tokens=max_tokens, |
| 648 | ignore_eos=self.ignore_eos, |
| 649 | temperature=temperature, |
| 650 | length_penalty=self.length_penalty, |
| 651 | include_stop_str_in_output=self.include_stop_str_in_output, |
| 652 | skip_special_tokens=self.skip_special_tokens, |
| 653 | ) |
| 654 | |
| 655 | def extract_structured_outputs(self) -> StructuredOutputsParams | None: |
| 656 | """Normalize request constraints into ``StructuredOutputsParams``.""" |
| 657 | return structured_outputs_from_response_format( |
| 658 | self.structured_outputs, |
| 659 | self.response_format, |
| 660 | ) |
| 661 | |
| 662 | def to_sampling_params( |
| 663 | self, |
| 664 | max_tokens: int, |
| 665 | default_sampling_params: dict, |
| 666 | ) -> SamplingParams: |
| 667 | # Priority: user -> server default -> OpenAI default |
| 668 | sampling_params = { |
| 669 | name: ( |
| 670 | value |
| 671 | if (value := getattr(self, name)) is not None |
| 672 | else default_sampling_params.get(name, default) |
| 673 | ) |
| 674 | for name, default in self._DEFAULT_SAMPLING_PARAMS.items() |
| 675 | } |
| 676 | |
| 677 | # Merge server-default stop_token_ids (e.g., model-specific tokens |
| 678 | # like </call> for gpt-oss) with any request-specified ones |
| 679 | stop_token_ids = self.stop_token_ids |
| 680 | default_stop_ids = default_sampling_params.get("stop_token_ids") |
| 681 | if default_stop_ids: |
| 682 | if not stop_token_ids: |
| 683 | stop_token_ids = list(default_stop_ids) |
| 684 | else: |
| 685 | stop_token_ids = list( |
| 686 | dict.fromkeys([*stop_token_ids, *default_stop_ids]) |
| 687 | ) |
| 688 | |
| 689 | prompt_logprobs = self.prompt_logprobs |
| 690 | if prompt_logprobs is None and self.echo: |
| 691 | prompt_logprobs = self.top_logprobs |
| 692 | |
| 693 | extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {} |
| 694 | if self.kv_transfer_params: |
| 695 | # Pass in kv_transfer_params via extra_args |
| 696 | extra_args["kv_transfer_params"] = self.kv_transfer_params |
| 697 | if self.ec_transfer_params: |
| 698 | # Pass in ec_transfer_params via extra_args |
| 699 | extra_args["ec_transfer_params"] = self.ec_transfer_params |
| 700 | return SamplingParams.from_optional( |
| 701 | n=self.n, |
| 702 | **sampling_params, |
| 703 | watermarking=self.watermarking, |
| 704 | seed=self.seed, |
| 705 | stop=self.stop, |
| 706 | stop_token_ids=stop_token_ids, |
| 707 | logprobs=( |
| 708 | self.top_logprobs |
| 709 | if self.logprobs and not self.logprob_token_ids |
| 710 | else None |
| 711 | ), |
| 712 | prompt_logprobs=prompt_logprobs, |
| 713 | logprob_token_ids=self.logprob_token_ids or None, |
| 714 | ignore_eos=self.ignore_eos, |
| 715 | max_tokens=max_tokens, |
| 716 | min_tokens=self.min_tokens, |
| 717 | skip_special_tokens=self.skip_special_tokens, |
| 718 | spaces_between_special_tokens=self.spaces_between_special_tokens, |
| 719 | include_stop_str_in_output=self.include_stop_str_in_output, |
| 720 | output_kind=( |
| 721 | RequestOutputKind.DELTA if self.stream else RequestOutputKind.FINAL_ONLY |
| 722 | ), |
| 723 | stream_interval=self.stream_interval, |
| 724 | structured_outputs=self.extract_structured_outputs(), |
| 725 | logit_bias=self.logit_bias, |
| 726 | bad_words=self.bad_words, |
| 727 | thinking_token_budget=self.thinking_token_budget, |
| 728 | allowed_token_ids=self.allowed_token_ids, |
| 729 | extra_args=extra_args or None, |
| 730 | skip_clone=True, # Created fresh per request, safe to skip clone |
| 731 | repetition_detection=self.repetition_detection, |
| 732 | routed_experts_prompt_start=self.routed_experts_prompt_start, |
| 733 | ) |
| 734 | |
| 735 | @model_validator(mode="before") |
| 736 | @classmethod |
| 737 | def validate_response_format(cls, data): |
| 738 | if not isinstance(data, dict): |
| 739 | return data |
| 740 | response_format = data.get("response_format") |
| 741 | if response_format is None: |
| 742 | return data |
| 743 | |
| 744 | rf_type = ( |
| 745 | response_format.get("type") |
| 746 | if isinstance(response_format, dict) |
| 747 | else getattr(response_format, "type", None) |
| 748 | ) |
| 749 | |
| 750 | if rf_type == "json_schema": |
| 751 | json_schema = ( |
| 752 | response_format.get("json_schema") |
| 753 | if isinstance(response_format, dict) |
| 754 | else getattr(response_format, "json_schema", None) |
| 755 | ) |
| 756 | if json_schema is None: |
| 757 | raise VLLMValidationError( |
| 758 | "When response_format type is 'json_schema', the " |
| 759 | "'json_schema' field must be provided.", |
| 760 | parameter="response_format", |
| 761 | ) |
| 762 | |
| 763 | if rf_type == "structural_tag": |
| 764 | validate_structural_tag_response_format(response_format) |
| 765 | |
| 766 | return data |
| 767 | |
| 768 | @model_validator(mode="before") |
| 769 | @classmethod |
| 770 | def validate_stream_options(cls, data): |
| 771 | if not isinstance(data, dict): |
| 772 | return data |
| 773 | if data.get("stream_options") and not data.get("stream"): |
| 774 | raise VLLMValidationError( |
| 775 | "Stream options can only be defined when `stream=True`.", |
| 776 | parameter="stream_options", |
| 777 | ) |
| 778 | |
| 779 | return data |
| 780 | |
| 781 | @model_validator(mode="before") |
| 782 | @classmethod |
| 783 | def check_logprobs(cls, data): |
| 784 | if not isinstance(data, dict): |
| 785 | return data |
| 786 | if data.get("logprob_token_ids") and data.get("use_beam_search"): |
| 787 | raise VLLMValidationError( |
| 788 | "`logprob_token_ids` is not supported with beam search.", |
| 789 | parameter="logprob_token_ids", |
| 790 | ) |
| 791 | |
| 792 | if data.get("logprob_token_ids") and not data.get("logprobs"): |
| 793 | raise VLLMValidationError( |
| 794 | "when using `logprob_token_ids`, `logprobs` must be set to true.", |
| 795 | parameter="logprob_token_ids", |
| 796 | ) |
| 797 | |
| 798 | # These fields are integers, but `mode="before"` runs on the raw |
| 799 | # request data, so a non-numeric value (e.g. a JSON string) would |
| 800 | # reach the comparisons below and raise TypeError -> HTTP 500. Reject |
| 801 | # it here so the client gets a clean 400 instead. |
| 802 | for field_name in ("prompt_logprobs", "top_logprobs"): |
| 803 | field_value = data.get(field_name) |
| 804 | if field_value is not None and not isinstance(field_value, (int, float)): |
| 805 | raise VLLMValidationError( |
| 806 | f"`{field_name}` must be an integer.", |
| 807 | parameter=field_name, |
| 808 | value=field_value, |
| 809 | ) |
| 810 | if (prompt_logprobs := data.get("prompt_logprobs")) is not None: |
| 811 | if data.get("stream") and (prompt_logprobs > 0 or prompt_logprobs == -1): |
| 812 | raise VLLMValidationError( |
| 813 | "`prompt_logprobs` are not available when `stream=True`.", |
| 814 | parameter="prompt_logprobs", |
| 815 | ) |
| 816 | |
| 817 | if prompt_logprobs < 0 and prompt_logprobs != -1: |
| 818 | raise VLLMValidationError( |
| 819 | "`prompt_logprobs` must be a positive value or -1.", |
| 820 | parameter="prompt_logprobs", |
| 821 | value=prompt_logprobs, |
| 822 | ) |
| 823 | if (top_logprobs := data.get("top_logprobs")) is not None: |
| 824 | if top_logprobs < 0 and top_logprobs != -1: |
| 825 | raise VLLMValidationError( |
| 826 | "`top_logprobs` must be a positive value or -1.", |
| 827 | parameter="top_logprobs", |
| 828 | value=top_logprobs, |
| 829 | ) |
| 830 | |
| 831 | if (top_logprobs == -1 or top_logprobs > 0) and not data.get("logprobs"): |
| 832 | raise VLLMValidationError( |
| 833 | "when using `top_logprobs`, `logprobs` must be set to true.", |
| 834 | parameter="top_logprobs", |
| 835 | ) |
| 836 | |
| 837 | return data |
| 838 | |
| 839 | @model_validator(mode="before") |
| 840 | @classmethod |
| 841 | def check_structured_outputs_count(cls, data): |
| 842 | if isinstance(data, ValueError): |
| 843 | raise data |
| 844 | if not isinstance(data, dict): |
| 845 | return data |
| 846 | |
| 847 | if data.get("structured_outputs", None) is None: |
| 848 | return data |
| 849 | |
| 850 | structured_outputs_kwargs = data["structured_outputs"] |
| 851 | # structured_outputs may arrive as a dict (from JSON/raw kwargs) or |
| 852 | # as a StructuredOutputsParams dataclass instance. |
| 853 | is_dataclass = isinstance(structured_outputs_kwargs, StructuredOutputsParams) |
| 854 | count = sum( |
| 855 | ( |
| 856 | getattr(structured_outputs_kwargs, k, None) |
| 857 | if is_dataclass |
| 858 | else structured_outputs_kwargs.get(k) |
| 859 | ) |
| 860 | is not None |
| 861 | for k in ("json", "regex", "choice") |
| 862 | ) |
| 863 | # you can only use one kind of constraints for structured outputs |
| 864 | if count > 1: |
| 865 | raise VLLMValidationError( |
| 866 | "You can only use one kind of constraints for structured " |
| 867 | "outputs ('json', 'regex' or 'choice').", |
| 868 | ) |
| 869 | # you can only either use structured outputs or tools, not both |
| 870 | if count > 0 and data.get("tool_choice", "none") not in ( |
| 871 | "none", |
| 872 | "auto", |
| 873 | "required", |
| 874 | ): |
| 875 | raise VLLMValidationError( |
| 876 | "You can only either use constraints for structured outputs " |
| 877 | "or tools, not both.", |
| 878 | ) |
| 879 | validate_structured_outputs_structural_tag(structured_outputs_kwargs) |
| 880 | return data |
| 881 | |
| 882 | @model_validator(mode="before") |
| 883 | @classmethod |
| 884 | def check_tool_usage(cls, data): |
| 885 | if isinstance(data, ValueError): |
| 886 | raise data |
| 887 | if not isinstance(data, dict): |
| 888 | return data |
| 889 | |
| 890 | # Reject empty tools array, matching OpenAI API behavior |
| 891 | if data.get("tools") == []: |
| 892 | raise VLLMValidationError( |
| 893 | "`tools` must not be an empty array. " |
| 894 | "Either provide at least one tool or omit the field entirely.", |
| 895 | parameter="tools", |
| 896 | ) |
| 897 | |
| 898 | # if "tool_choice" is not specified but tools are provided, |
| 899 | # default to "auto" tool_choice |
| 900 | if "tool_choice" not in data and data.get("tools"): |
| 901 | data["tool_choice"] = "auto" |
| 902 | |
| 903 | # if "tool_choice" is "none" -- no validation is needed for tools |
| 904 | if "tool_choice" in data and data["tool_choice"] == "none": |
| 905 | return data |
| 906 | |
| 907 | # if "tool_choice" is specified -- validation |
| 908 | if "tool_choice" in data and data["tool_choice"] is not None: |
| 909 | # ensure that if "tool choice" is specified, tools are present |
| 910 | if "tools" not in data or data["tools"] is None: |
| 911 | raise VLLMValidationError( |
| 912 | "When using `tool_choice`, `tools` must be set.", |
| 913 | parameter="tool_choice", |
| 914 | ) |
| 915 | |
| 916 | # make sure that tool choice is either a named tool |
| 917 | # OR that it's set to "auto" or "required" |
| 918 | if data["tool_choice"] not in ["auto", "required"] and not isinstance( |
| 919 | data["tool_choice"], dict |
| 920 | ): |
| 921 | raise VLLMValidationError( |
| 922 | f"Invalid value for `tool_choice`: {data['tool_choice']}! " |
| 923 | 'Only named tools, "none", "auto" or "required" ' |
| 924 | "are supported.", |
| 925 | parameter="tool_choice", |
| 926 | ) |
| 927 | |
| 928 | # ensure that if "tool_choice" is specified as an object, |
| 929 | # it matches a valid tool |
| 930 | correct_usage_message = ( |
| 931 | 'Correct usage: `{"type": "function",' |
| 932 | ' "function": {"name": "my_function"}}`' |
| 933 | ) |
| 934 | if isinstance(data["tool_choice"], dict): |
| 935 | valid_tool = False |
| 936 | function = data["tool_choice"].get("function") |
| 937 | if not isinstance(function, dict): |
| 938 | raise VLLMValidationError( |
| 939 | f"Invalid value for `function`: `{function}` in " |
| 940 | f"`tool_choice`! {correct_usage_message}", |
| 941 | parameter="tool_choice.function", |
| 942 | ) |
| 943 | if "name" not in function: |
| 944 | raise VLLMValidationError( |
| 945 | f"Expected field `name` in `function` in " |
| 946 | f"`tool_choice`! {correct_usage_message}", |
| 947 | parameter="tool_choice.function.name", |
| 948 | ) |
| 949 | function_name = function["name"] |
| 950 | if not isinstance(function_name, str) or len(function_name) == 0: |
| 951 | raise VLLMValidationError( |
| 952 | f"Invalid `name` in `function`: `{function_name}`" |
| 953 | f" in `tool_choice`! {correct_usage_message}", |
| 954 | parameter="tool_choice.function.name", |
| 955 | ) |
| 956 | for tool in data["tools"]: |
| 957 | if tool["function"]["name"] == function_name: |
| 958 | valid_tool = True |
| 959 | break |
| 960 | if not valid_tool: |
| 961 | raise VLLMValidationError( |
| 962 | "The tool specified in `tool_choice` does not match any" |
| 963 | " of the specified `tools`", |
| 964 | parameter="tool_choice", |
| 965 | ) |
| 966 | return data |
| 967 | |
| 968 | @model_validator(mode="before") |
| 969 | @classmethod |
| 970 | def check_generation_prompt(cls, data): |
| 971 | if not isinstance(data, dict): |
| 972 | return data |
| 973 | if data.get("continue_final_message") and data.get("add_generation_prompt"): |
| 974 | raise VLLMValidationError( |
| 975 | "Cannot set both `continue_final_message` and " |
| 976 | "`add_generation_prompt` to True.", |
| 977 | ) |
| 978 | return data |
| 979 | |
| 980 | @model_validator(mode="before") |
| 981 | @classmethod |
| 982 | def drop_prompt_token_ids_with_media(cls, data): |
| 983 | # The forwarded ids would drop media in ``messages``, so ignore them. Runs |
| 984 | # before validation, which can turn content into a one-shot iterator. |
| 985 | if not isinstance(data, dict): |
| 986 | return data |
| 987 | kv_transfer_params = data.get("kv_transfer_params") |
| 988 | if ( |
| 989 | isinstance(kv_transfer_params, dict) |
| 990 | and kv_transfer_params.get("prompt_token_ids") is not None |
| 991 | and has_non_text_content(data.get("messages")) |
| 992 | ): |
| 993 | logger.debug( |
| 994 | "Ignoring kv_transfer_params['prompt_token_ids']: " |
| 995 | "messages have non-text content and are rendered instead." |
| 996 | ) |
| 997 | kv_transfer_params.pop("prompt_token_ids") |
| 998 | return data |
| 999 | |
| 1000 | @model_validator(mode="before") |
| 1001 | @classmethod |
| 1002 | def check_system_message_content_type(cls, data): |
| 1003 | """Warn if system messages contain non-text content. |
| 1004 | |
| 1005 | According to OpenAI API spec, system messages can only be of type |
| 1006 | 'text'. We log a warning instead of rejecting to avoid breaking |
| 1007 | users who intentionally send multimodal system messages. |
| 1008 | See: https://platform.openai.com/docs/api-reference/chat/create#chat_create-messages-system_message |
| 1009 | """ |
| 1010 | if not isinstance(data, dict): |
| 1011 | return data |
| 1012 | messages = data.get("messages", []) |
| 1013 | for msg in messages: |
| 1014 | # Check if this is a system message |
| 1015 | if isinstance(msg, dict) and msg.get("role") == "system": |
| 1016 | content = msg.get("content") |
| 1017 | |
| 1018 | # If content is a list (multimodal format) |
| 1019 | if isinstance(content, list): |
| 1020 | for part in content: |
| 1021 | if isinstance(part, dict): |
| 1022 | part_type = part.get("type") |
| 1023 | # Infer type when 'type' field is not explicit |
| 1024 | if part_type is None: |
| 1025 | if "image_url" in part or "image_pil" in part: |
| 1026 | part_type = "image_url" |
| 1027 | elif "image_embeds" in part: |
| 1028 | part_type = "image_embeds" |
| 1029 | elif "audio_url" in part: |
| 1030 | part_type = "audio_url" |
| 1031 | elif "input_audio" in part: |
| 1032 | part_type = "input_audio" |
| 1033 | elif "audio_embeds" in part: |
| 1034 | part_type = "audio_embeds" |
| 1035 | elif "video_url" in part: |
| 1036 | part_type = "video_url" |
| 1037 | |
| 1038 | # Warn about non-text content in system messages |
| 1039 | if part_type and part_type != "text": |
| 1040 | logger.warning_once( |
| 1041 | "System messages should only contain text " |
| 1042 | "content according to the OpenAI API spec. " |
| 1043 | "Found content type: '%s'.", |
| 1044 | part_type, |
| 1045 | ) |
| 1046 | |
| 1047 | return data |
| 1048 | |
| 1049 | |
| 1050 | class BatchChatCompletionRequest(OpenAIBaseModel): |
| 1051 | """Request model for the /v1/chat/completions/batch endpoint. |
| 1052 | |
| 1053 | Accepts the same fields as ChatCompletionRequest except that ``messages`` |
| 1054 | is a list of conversations (each conversation is a |
| 1055 | ``list[ChatCompletionMessageParam]``). Each conversation is processed |
| 1056 | independently and the response contains one choice per conversation, |
| 1057 | indexed 0, 1, ..., N-1. |
| 1058 | |
| 1059 | Current limitations compared to the single-conversation endpoint: |
| 1060 | - Streaming is not supported (``stream`` must be False or omitted). |
| 1061 | - Tool use is not supported (``tools`` must be omitted). |
| 1062 | - Beam search is not supported (``use_beam_search`` must be False or omitted). |
| 1063 | - The ``n`` parameter must be 1 (or omitted). |
| 1064 | """ |
| 1065 | |
| 1066 | messages: list[Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]] = ( |
| 1067 | Field(..., min_length=1) |
| 1068 | ) |
| 1069 | model: str | None = None |
| 1070 | |
| 1071 | # Shared sampling / generation fields — mirror ChatCompletionRequest. |
| 1072 | frequency_penalty: float | None = 0.0 |
| 1073 | logit_bias: dict[str, float] | None = None |
| 1074 | logprobs: bool | None = False |
| 1075 | top_logprobs: int | None = 0 |
| 1076 | logprob_token_ids: list[int] | None = Field( |
| 1077 | default=None, |
| 1078 | description=( |
| 1079 | "Specific vocab token IDs to return logprobs for at each generated " |
| 1080 | "position, in addition to the sampled token. Requires " |
| 1081 | "`logprobs=True`." |
| 1082 | ), |
| 1083 | ) |
| 1084 | max_tokens: int | None = None |
| 1085 | max_completion_tokens: int | None = None |
| 1086 | n: int | None = 1 |
| 1087 | presence_penalty: float | None = 0.0 |
| 1088 | response_format: Any | None = None |
| 1089 | seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX) |
| 1090 | stop: StopParam = Field(default_factory=list) |
| 1091 | temperature: float | None = None |
| 1092 | top_p: float | None = None |
| 1093 | user: str | None = None |
| 1094 | tool_choice: Literal["none"] | None = "none" |
| 1095 | include_reasoning: bool = True |
| 1096 | |
| 1097 | # vLLM extensions |
| 1098 | best_of: int | None = None |
| 1099 | use_beam_search: bool = False |
| 1100 | top_k: int | None = None |
| 1101 | min_p: float | None = None |
| 1102 | repetition_penalty: float | None = None |
| 1103 | length_penalty: float | None = 1.0 |
| 1104 | early_stopping: bool = False |
| 1105 | structured_outputs: StructuredOutputsParams | None = None |
| 1106 | request_id: str | None = None |
| 1107 | add_generation_prompt: bool = True |
| 1108 | continue_final_message: bool = False |
| 1109 | chat_template: str | None = None |
| 1110 | chat_template_kwargs: dict[str, Any] | None = None |
| 1111 | media_io_kwargs: dict[str, dict[str, Any]] | None = None |
| 1112 | mm_processor_kwargs: dict[str, Any] | None = None |
| 1113 | priority: int = Field(default=0, ge=_INT64_MIN, le=_INT64_MAX) |
| 1114 | cache_salt: str | None = None |
| 1115 | include_stop_str_in_output: bool = False |
| 1116 | guided_decoding_backend: str | None = None |
| 1117 | echo: bool = False |
| 1118 | # None falls back to the server-level --return-tokens-as-token-ids default, |
| 1119 | # matching ChatCompletionRequest.return_tokens_as_token_ids. |
| 1120 | return_tokens_as_token_ids: bool | None = None |
| 1121 | return_token_ids: bool = False |
| 1122 | |
| 1123 | @model_validator(mode="before") |
| 1124 | @classmethod |
| 1125 | def check_batch_mode(cls, data: Any) -> Any: |
| 1126 | if isinstance(data, BatchChatCompletionRequest): |
| 1127 | data = data.model_dump(exclude_unset=True) |
| 1128 | if not isinstance(data, dict): |
| 1129 | return data |
| 1130 | if data.get("use_beam_search"): |
| 1131 | raise VLLMValidationError( |
| 1132 | "Batch chat completions do not support beam search. " |
| 1133 | "Please set `use_beam_search` to False.", |
| 1134 | parameter="use_beam_search", |
| 1135 | ) |
| 1136 | if data.get("logprob_token_ids") and not data.get("logprobs"): |
| 1137 | raise VLLMValidationError( |
| 1138 | "when using `logprob_token_ids`, `logprobs` must be set to true.", |
| 1139 | parameter="logprob_token_ids", |
| 1140 | ) |
| 1141 | kv_transfer_params = data.get("kv_transfer_params") |
| 1142 | if ( |
| 1143 | isinstance(kv_transfer_params, dict) |
| 1144 | and kv_transfer_params.get("prompt_token_ids") is not None |
| 1145 | ): |
| 1146 | raise VLLMValidationError( |
| 1147 | "Batch chat completions do not support " |
| 1148 | "`kv_transfer_params['prompt_token_ids']`: one pre-tokenized " |
| 1149 | "prompt cannot serve several conversations.", |
| 1150 | parameter="kv_transfer_params.prompt_token_ids", |
| 1151 | ) |
| 1152 | response_format = data.get("response_format") |
| 1153 | if response_format is not None: |
| 1154 | rf_type = ( |
| 1155 | response_format.get("type") |
| 1156 | if isinstance(response_format, dict) |
| 1157 | else getattr(response_format, "type", None) |
| 1158 | ) |
| 1159 | if rf_type == "structural_tag": |
| 1160 | validate_structural_tag_response_format(response_format) |
| 1161 | if (structured_outputs := data.get("structured_outputs")) is not None: |
| 1162 | validate_structured_outputs_structural_tag(structured_outputs) |
| 1163 | n = data.get("n", 1) |
| 1164 | if n is not None and n != 1: |
| 1165 | raise VLLMValidationError( |
| 1166 | "Batch chat completions do not support `n > 1`. Please set `n` to 1.", |
| 1167 | parameter="n", |
| 1168 | value=n, |
| 1169 | ) |
| 1170 | return data |
| 1171 | |
| 1172 | def to_chat_completion_request( |
| 1173 | self, messages: list[ChatCompletionMessageParam] |
| 1174 | ) -> ChatCompletionRequest: |
| 1175 | """Build a single-conversation ChatCompletionRequest from one conversation.""" |
| 1176 | data = self.model_dump(exclude={"messages"}, exclude_none=True) |
| 1177 | data["messages"] = messages |
| 1178 | return ChatCompletionRequest.model_validate(data) |