| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import asyncio |
| 5 | import json |
| 6 | import types |
| 7 | from abc import ABC, abstractmethod |
| 8 | from collections import Counter, defaultdict |
| 9 | from collections.abc import Awaitable, Callable, Iterable |
| 10 | from dataclasses import dataclass |
| 11 | from functools import cached_property, lru_cache, partial |
| 12 | from itertools import accumulate |
| 13 | from pathlib import Path |
| 14 | from typing import ( |
| 15 | TYPE_CHECKING, |
| 16 | Any, |
| 17 | Final, |
| 18 | Generic, |
| 19 | Literal, |
| 20 | TypeAlias, |
| 21 | TypeVar, |
| 22 | Union, |
| 23 | cast, |
| 24 | get_args, |
| 25 | get_origin, |
| 26 | ) |
| 27 | |
| 28 | from openai.types.chat import ( |
| 29 | ChatCompletionAssistantMessageParam, |
| 30 | ChatCompletionContentPartImageParam, |
| 31 | ChatCompletionContentPartInputAudioParam, |
| 32 | ChatCompletionContentPartRefusalParam, |
| 33 | ChatCompletionContentPartTextParam, |
| 34 | ChatCompletionFunctionToolParam, |
| 35 | ChatCompletionMessageToolCallParam, |
| 36 | ChatCompletionToolMessageParam, |
| 37 | ) |
| 38 | from openai.types.chat import ( |
| 39 | ChatCompletionContentPartParam as OpenAIChatCompletionContentPartParam, |
| 40 | ) |
| 41 | from openai.types.chat import ( |
| 42 | ChatCompletionMessageParam as OpenAIChatCompletionMessageParam, |
| 43 | ) |
| 44 | from openai.types.chat.chat_completion_content_part_input_audio_param import InputAudio |
| 45 | from openai.types.responses import ResponseInputImageParam |
| 46 | from openai_harmony import Message as OpenAIHarmonyMessage |
| 47 | from PIL import Image |
| 48 | from pydantic import BaseModel, ConfigDict, TypeAdapter |
| 49 | |
| 50 | # pydantic needs the TypedDict from typing_extensions |
| 51 | from typing_extensions import Required, TypedDict, override |
| 52 | |
| 53 | from vllm import envs |
| 54 | from vllm.config import ModelConfig |
| 55 | from vllm.exceptions import VLLMValidationError |
| 56 | from vllm.inputs import MultiModalDataDict, MultiModalUUIDDict |
| 57 | from vllm.logger import init_logger |
| 58 | from vllm.model_executor.models import SupportsMultiModal |
| 59 | from vllm.multimodal import MULTIMODAL_REGISTRY |
| 60 | from vllm.multimodal.inputs import ( |
| 61 | MultiModalBatchedField, |
| 62 | MultiModalFlatField, |
| 63 | MultiModalSharedField, |
| 64 | VisionChunk, |
| 65 | VisionChunkImage, |
| 66 | VisionChunkVideo, |
| 67 | ) |
| 68 | from vllm.multimodal.media import MEDIA_CONNECTOR_REGISTRY, MediaConnector |
| 69 | from vllm.multimodal.processing import BaseMultiModalProcessor |
| 70 | from vllm.renderers.embed_utils import ( |
| 71 | safe_load_prompt_embeds, |
| 72 | safe_load_prompt_embeds_async, |
| 73 | ) |
| 74 | from vllm.transformers_utils.processor import get_video_processor_cls_name |
| 75 | from vllm.utils import random_uuid |
| 76 | from vllm.utils.collection_utils import is_list_of, is_list_of_numbers |
| 77 | from vllm.utils.import_utils import LazyLoader |
| 78 | |
| 79 | if TYPE_CHECKING: |
| 80 | import torch |
| 81 | import transformers |
| 82 | else: |
| 83 | transformers = LazyLoader("transformers", globals(), "transformers") |
| 84 | torch = LazyLoader("torch", globals(), "torch") |
| 85 | |
| 86 | logger = init_logger(__name__) |
| 87 | |
| 88 | |
| 89 | class ChatTemplateResolutionError(ValueError): |
| 90 | """Raised when chat template resolution fails. |
| 91 | |
| 92 | This is a subclass of ValueError for backward compatibility with |
| 93 | existing exception handlers. |
| 94 | """ |
| 95 | |
| 96 | |
| 97 | MODALITY_PLACEHOLDERS_MAP = { |
| 98 | "image": "<##IMAGE##>", |
| 99 | "audio": "<##AUDIO##>", |
| 100 | "video": "<##VIDEO##>", |
| 101 | "prompt_embeds": "<##PROMPT_EMBEDS##>", |
| 102 | } |
| 103 | |
| 104 | |
| 105 | PROMPT_EMBEDS_PLACEHOLDER_TOKEN: Final[str] = "<prompt_embeds>" |
| 106 | """The special token used as a placeholder for each embedding |
| 107 | position during chat template rendering. |
| 108 | |
| 109 | Registered as an additional special token when `--enable-prompt-embeds` is set. |
| 110 | See `_ensure_prompt_embeds_placeholder_token` in `vllm/renderers/hf.py`. |
| 111 | """ |
| 112 | |
| 113 | |
| 114 | _REQUIRE_MM_PROCESSOR_ERROR: Final[str] = ( |
| 115 | "Resolving modality {modality!r} requires a multimodal processor " |
| 116 | "but none is available." |
| 117 | ) |
| 118 | |
| 119 | _ENABLE_PROMPT_EMBEDS_ERROR: Final[str] = ( |
| 120 | "You must set `--enable-prompt-embeds` to input `prompt_embeds`" |
| 121 | ) |
| 122 | |
| 123 | _PROMPT_EMBEDS_MISSING_DATA_ERROR: Final[str] = ( |
| 124 | "prompt_embeds content part requires a non-empty `data` field " |
| 125 | "with base64-encoded tensor bytes." |
| 126 | ) |
| 127 | |
| 128 | _RESERVED_PLACEHOLDER_IN_TEXT_ERROR: Final[str] = ( |
| 129 | "Text content may not contain the reserved placeholder {token!r}. " |
| 130 | "This placeholder is used internally to mark `prompt_embeds` splice " |
| 131 | "positions in the tokenized prompt." |
| 132 | ) |
| 133 | |
| 134 | |
| 135 | class AudioURL(TypedDict, total=False): |
| 136 | url: Required[str] |
| 137 | """ |
| 138 | Either a URL of the audio or a data URL with base64 encoded audio data. |
| 139 | """ |
| 140 | |
| 141 | |
| 142 | class ChatCompletionContentPartAudioParam(TypedDict, total=False): |
| 143 | audio_url: Required[AudioURL] |
| 144 | |
| 145 | type: Required[Literal["audio_url"]] |
| 146 | """The type of the content part.""" |
| 147 | |
| 148 | |
| 149 | MultiModalEmbedsPayload: TypeAlias = str | dict[str, str | list[int | float]] |
| 150 | |
| 151 | |
| 152 | class ChatCompletionContentPartImageEmbedsParam(TypedDict, total=False): |
| 153 | image_embeds: MultiModalEmbedsPayload | None |
| 154 | """ |
| 155 | The image embeddings. It can be either: |
| 156 | - A single base64 string. |
| 157 | - A dictionary of base64 tensors or numeric JSON metadata arrays. |
| 158 | """ |
| 159 | type: Required[Literal["image_embeds"]] |
| 160 | """The type of the content part.""" |
| 161 | uuid: str | None |
| 162 | """ |
| 163 | User-provided UUID of a media. User must guarantee that it is properly |
| 164 | generated and unique for different medias. |
| 165 | """ |
| 166 | |
| 167 | |
| 168 | class ChatCompletionContentPartAudioEmbedsParam(TypedDict, total=False): |
| 169 | audio_embeds: MultiModalEmbedsPayload | None |
| 170 | """ |
| 171 | The audio embeddings. It can be either: |
| 172 | - A single base64 string representing a serialized torch tensor. |
| 173 | - A dictionary of base64 tensors or numeric JSON metadata arrays. |
| 174 | """ |
| 175 | type: Required[Literal["audio_embeds"]] |
| 176 | """The type of the content part.""" |
| 177 | uuid: str | None |
| 178 | """ |
| 179 | User-provided UUID of a media. User must guarantee that it is properly |
| 180 | generated and unique for different medias. |
| 181 | """ |
| 182 | |
| 183 | |
| 184 | class ChatCompletionContentPartVideoEmbedsParam(TypedDict, total=False): |
| 185 | video_embeds: MultiModalEmbedsPayload | None |
| 186 | """ |
| 187 | The video embeddings. It can be either: |
| 188 | - A single base64 string representing a serialized torch tensor. |
| 189 | - A dictionary of base64 tensors or numeric JSON metadata arrays. |
| 190 | """ |
| 191 | type: Required[Literal["video_embeds"]] |
| 192 | """The type of the content part.""" |
| 193 | uuid: str | None |
| 194 | """ |
| 195 | User-provided UUID of a media. User must guarantee that it is properly |
| 196 | generated and unique for different medias. |
| 197 | """ |
| 198 | |
| 199 | |
| 200 | class ChatCompletionContentPartPromptEmbedsParam(TypedDict, total=False): |
| 201 | data: Required[str] |
| 202 | """ |
| 203 | Base64-encoded bytes of a serialized `torch.Tensor` of shape |
| 204 | `(num_tokens, hidden_size)`. The tensor's `dtype` and `hidden_size` must |
| 205 | match the model's input embedding layer. |
| 206 | """ |
| 207 | type: Required[Literal["prompt_embeds"]] |
| 208 | """The type of the content part.""" |
| 209 | |
| 210 | |
| 211 | class VideoURL(TypedDict, total=False): |
| 212 | url: Required[str] |
| 213 | """ |
| 214 | Either a URL of the video or a data URL with base64 encoded video data. |
| 215 | """ |
| 216 | |
| 217 | |
| 218 | class ChatCompletionContentPartVideoParam(TypedDict, total=False): |
| 219 | video_url: Required[VideoURL] |
| 220 | |
| 221 | type: Required[Literal["video_url"]] |
| 222 | """The type of the content part.""" |
| 223 | |
| 224 | |
| 225 | class PILImage(BaseModel): |
| 226 | """A PIL.Image.Image object.""" |
| 227 | |
| 228 | image_pil: Image.Image |
| 229 | model_config = ConfigDict(arbitrary_types_allowed=True) |
| 230 | |
| 231 | |
| 232 | class CustomChatCompletionContentPILImageParam(TypedDict, total=False): |
| 233 | """A simpler version of the param that only accepts a PIL image. |
| 234 | |
| 235 | Example: |
| 236 | { |
| 237 | "image_pil": ImageAsset('cherry_blossom').pil_image |
| 238 | } |
| 239 | |
| 240 | """ |
| 241 | |
| 242 | image_pil: PILImage | None |
| 243 | uuid: str | None |
| 244 | """ |
| 245 | User-provided UUID of a media. User must guarantee that it is properly |
| 246 | generated and unique for different medias. |
| 247 | """ |
| 248 | |
| 249 | |
| 250 | class CustomChatCompletionContentSimpleImageParam(TypedDict, total=False): |
| 251 | """A simpler version of the param that only accepts a plain image_url. |
| 252 | This is supported by OpenAI API, although it is not documented. |
| 253 | |
| 254 | Example: |
| 255 | { |
| 256 | "image_url": "https://example.com/image.jpg" |
| 257 | } |
| 258 | |
| 259 | """ |
| 260 | |
| 261 | image_url: str | None |
| 262 | uuid: str | None |
| 263 | """ |
| 264 | User-provided UUID of a media. User must guarantee that it is properly |
| 265 | generated and unique for different medias. |
| 266 | """ |
| 267 | |
| 268 | |
| 269 | class CustomChatCompletionContentSimpleAudioParam(TypedDict, total=False): |
| 270 | """A simpler version of the param that only accepts a plain audio_url. |
| 271 | |
| 272 | Example: |
| 273 | { |
| 274 | "audio_url": "https://example.com/audio.mp3" |
| 275 | } |
| 276 | |
| 277 | """ |
| 278 | |
| 279 | audio_url: str | None |
| 280 | |
| 281 | |
| 282 | class CustomChatCompletionContentSimpleVideoParam(TypedDict, total=False): |
| 283 | """A simpler version of the param that only accepts a plain audio_url. |
| 284 | |
| 285 | Example: |
| 286 | { |
| 287 | "video_url": "https://example.com/video.mp4" |
| 288 | } |
| 289 | |
| 290 | """ |
| 291 | |
| 292 | video_url: str | None |
| 293 | uuid: str | None |
| 294 | """ |
| 295 | User-provided UUID of a media. User must guarantee that it is properly |
| 296 | generated and unique for different medias. |
| 297 | """ |
| 298 | |
| 299 | |
| 300 | class CustomThinkCompletionContentParam(TypedDict, total=False): |
| 301 | """A Think Completion Content Param that accepts a plain text and a boolean. |
| 302 | |
| 303 | Example: |
| 304 | { |
| 305 | "thinking": "I am thinking about the answer", |
| 306 | "closed": True, |
| 307 | "type": "thinking" |
| 308 | } |
| 309 | |
| 310 | """ |
| 311 | |
| 312 | thinking: Required[str] |
| 313 | """The thinking content.""" |
| 314 | |
| 315 | closed: bool |
| 316 | """Whether the thinking is closed.""" |
| 317 | |
| 318 | type: Required[Literal["thinking"]] |
| 319 | """The thinking type.""" |
| 320 | |
| 321 | |
| 322 | class CustomChatCompletionContentToolReferenceParam(TypedDict, total=False): |
| 323 | """A tool reference content param that only accepts a plain tool name. |
| 324 | |
| 325 | Example: |
| 326 | { |
| 327 | "name": "get_weather", |
| 328 | "type": "tool_reference" |
| 329 | } |
| 330 | |
| 331 | """ |
| 332 | |
| 333 | name: str |
| 334 | """The name of the tool being referenced.""" |
| 335 | |
| 336 | type: Literal["tool_reference"] |
| 337 | """The content type.""" |
| 338 | |
| 339 | |
| 340 | ChatCompletionContentPartParam: TypeAlias = ( |
| 341 | OpenAIChatCompletionContentPartParam |
| 342 | | ChatCompletionContentPartAudioParam |
| 343 | | ChatCompletionContentPartInputAudioParam |
| 344 | | ChatCompletionContentPartVideoParam |
| 345 | | ChatCompletionContentPartRefusalParam |
| 346 | | CustomChatCompletionContentPILImageParam |
| 347 | | CustomChatCompletionContentSimpleImageParam |
| 348 | | ChatCompletionContentPartImageEmbedsParam |
| 349 | | ChatCompletionContentPartAudioEmbedsParam |
| 350 | | ChatCompletionContentPartVideoEmbedsParam |
| 351 | | ChatCompletionContentPartPromptEmbedsParam |
| 352 | | CustomChatCompletionContentSimpleAudioParam |
| 353 | | CustomChatCompletionContentSimpleVideoParam |
| 354 | | CustomChatCompletionContentToolReferenceParam |
| 355 | | str |
| 356 | | CustomThinkCompletionContentParam |
| 357 | ) |
| 358 | |
| 359 | |
| 360 | class CustomChatCompletionMessageParam(TypedDict, total=False): |
| 361 | """Enables custom roles in the Chat Completion API.""" |
| 362 | |
| 363 | role: Required[str] |
| 364 | """The role of the message's author.""" |
| 365 | |
| 366 | content: str | list[ChatCompletionContentPartParam] |
| 367 | """The contents of the message.""" |
| 368 | |
| 369 | name: str |
| 370 | """An optional name for the participant. |
| 371 | |
| 372 | Provides the model information to differentiate between participants of the |
| 373 | same role. |
| 374 | """ |
| 375 | |
| 376 | tool_call_id: str | None |
| 377 | """Tool call that this message is responding to.""" |
| 378 | |
| 379 | tool_calls: list[ChatCompletionMessageToolCallParam] | None |
| 380 | """The tool calls generated by the model, such as function calls.""" |
| 381 | |
| 382 | reasoning: str | None |
| 383 | """The reasoning content for interleaved thinking.""" |
| 384 | |
| 385 | tools: list[ChatCompletionFunctionToolParam] | None |
| 386 | """The tools for developer role.""" |
| 387 | |
| 388 | task: str | None |
| 389 | """Model-specific task marker. Currently passed through for DeepSeek V4.""" |
| 390 | |
| 391 | |
| 392 | ChatCompletionMessageParam: TypeAlias = ( |
| 393 | OpenAIChatCompletionMessageParam |
| 394 | | CustomChatCompletionMessageParam |
| 395 | | OpenAIHarmonyMessage |
| 396 | ) |
| 397 | |
| 398 | |
| 399 | # TODO: Make fields ReadOnly once mypy supports it |
| 400 | class ConversationMessage(TypedDict, total=False): |
| 401 | role: Required[str] |
| 402 | """The role of the message's author.""" |
| 403 | |
| 404 | content: str | None | list[dict[str, str]] |
| 405 | """The contents of the message""" |
| 406 | |
| 407 | tool_call_id: str | None |
| 408 | """Tool call that this message is responding to.""" |
| 409 | |
| 410 | name: str | None |
| 411 | """The name of the function to call""" |
| 412 | |
| 413 | tool_calls: list[ChatCompletionMessageToolCallParam] | None |
| 414 | """The tool calls generated by the model, such as function calls.""" |
| 415 | |
| 416 | reasoning: str | None |
| 417 | """The reasoning content for interleaved thinking.""" |
| 418 | |
| 419 | reasoning_content: str | None |
| 420 | """Deprecated: The reasoning content for interleaved thinking.""" |
| 421 | |
| 422 | tools: list[ChatCompletionFunctionToolParam] | None |
| 423 | """The tools for developer role.""" |
| 424 | |
| 425 | task: str | None |
| 426 | """Model-specific task marker. Currently passed through for DeepSeek V4.""" |
| 427 | |
| 428 | |
| 429 | # Passed in by user |
| 430 | ChatTemplateContentFormatOption = Literal["auto", "string", "openai"] |
| 431 | |
| 432 | # After resolving "auto" |
| 433 | ChatTemplateContentFormat = Literal["string", "openai"] |
| 434 | |
| 435 | |
| 436 | ModalityStr = Literal[ |
| 437 | "image", |
| 438 | "audio", |
| 439 | "video", |
| 440 | "image_embeds", |
| 441 | "audio_embeds", |
| 442 | "video_embeds", |
| 443 | "vision_chunk", |
| 444 | "prompt_embeds", |
| 445 | ] |
| 446 | _T = TypeVar("_T") |
| 447 | _AsyncMultiModalItem: TypeAlias = Callable[[], Awaitable[tuple[object, str | None]]] |
| 448 | |
| 449 | |
| 450 | # Backward compatibility for single item input |
| 451 | class _BatchedSingleItemField(MultiModalSharedField): |
| 452 | pass |
| 453 | |
| 454 | |
| 455 | def _detect_field( |
| 456 | tensors: list[torch.Tensor], |
| 457 | mm_processor: BaseMultiModalProcessor, |
| 458 | ): |
| 459 | first_item = tensors[0] |
| 460 | hidden_size = mm_processor.info.ctx.model_config.get_inputs_embeds_size() |
| 461 | |
| 462 | if ( |
| 463 | len(tensors) == 1 |
| 464 | and first_item.ndim == 3 |
| 465 | and first_item.shape[0] == 1 |
| 466 | and first_item.shape[-1] == hidden_size |
| 467 | ): |
| 468 | logger.warning( |
| 469 | "Batched multi-modal embedding inputs are deprecated for Chat API. " |
| 470 | "Please pass a separate content part for each multi-modal item." |
| 471 | ) |
| 472 | return _BatchedSingleItemField(batch_size=1) |
| 473 | |
| 474 | first_shape = first_item.shape |
| 475 | if all(t.shape == first_shape for t in tensors): |
| 476 | return MultiModalBatchedField() |
| 477 | |
| 478 | size_per_item = [len(tensor) for tensor in tensors] |
| 479 | slice_idxs = [0, *accumulate(size_per_item)] |
| 480 | slices = [ |
| 481 | (slice(slice_idxs[i], slice_idxs[i + 1]),) for i in range(len(size_per_item)) |
| 482 | ] |
| 483 | return MultiModalFlatField(slices=slices) |
| 484 | |
| 485 | |
| 486 | def _merge_embeds( |
| 487 | data_items: list[dict[str, "torch.Tensor"]], |
| 488 | mm_processor: BaseMultiModalProcessor, |
| 489 | ): |
| 490 | if not data_items: |
| 491 | return {} |
| 492 | |
| 493 | first_keys = set(data_items[0].keys()) |
| 494 | if any(set(item.keys()) != first_keys for item in data_items[1:]): |
| 495 | raise VLLMValidationError( |
| 496 | "All dictionaries in the list of embeddings must have the same keys." |
| 497 | ) |
| 498 | |
| 499 | fields = { |
| 500 | key: _detect_field([item[key] for item in data_items], mm_processor) |
| 501 | for key in first_keys |
| 502 | } |
| 503 | data_merged = { |
| 504 | key: field._reduce_data([item[key] for item in data_items], pin_memory=False) |
| 505 | for key, field in fields.items() |
| 506 | } |
| 507 | |
| 508 | try: |
| 509 | # TODO: Support per-request mm_processor_kwargs |
| 510 | parsed_configs = mm_processor._get_mm_fields_config( |
| 511 | transformers.BatchFeature(data_merged), |
| 512 | {}, |
| 513 | ) |
| 514 | parsed_fields = {key: parsed_configs[key].field for key in first_keys} |
| 515 | keys_to_update = [ |
| 516 | key |
| 517 | for key in first_keys |
| 518 | if ( |
| 519 | fields[key] != parsed_fields[key] |
| 520 | and not isinstance(fields[key], _BatchedSingleItemField) |
| 521 | ) |
| 522 | ] |
| 523 | |
| 524 | for key in keys_to_update: |
| 525 | data_merged[key] = parsed_fields[key]._reduce_data( |
| 526 | [item[key] for item in data_items], pin_memory=False |
| 527 | ) |
| 528 | except Exception: |
| 529 | logger.exception( |
| 530 | "Error when parsing merged embeddings. " |
| 531 | "Falling back to auto-detected fields." |
| 532 | ) |
| 533 | |
| 534 | return data_merged |
| 535 | |
| 536 | |
| 537 | async def _load_embeds_dict( |
| 538 | data: dict[str, str | list[int | float]], |
| 539 | fetch: Callable[[str], Awaitable["torch.Tensor"]], |
| 540 | ) -> dict[str, Any]: |
| 541 | encoded = {key: value for key, value in data.items() if isinstance(value, str)} |
| 542 | tensors = await asyncio.gather(*(fetch(value) for value in encoded.values())) |
| 543 | return {**data, **dict(zip(encoded, tensors))} |
| 544 | |
| 545 | |
| 546 | def _parse_metadata_array(key: str, value: list, metadata_fields: set[str]): |
| 547 | if key not in metadata_fields: |
| 548 | raise VLLMValidationError(f"JSON arrays are only supported for metadata: {key}") |
| 549 | if not is_list_of_numbers(value): |
| 550 | raise VLLMValidationError(f"Metadata {key} must be a finite numeric array.") |
| 551 | dtype = torch.float64 if any(isinstance(v, float) for v in value) else torch.int64 |
| 552 | try: |
| 553 | return torch.tensor(value, dtype=dtype) |
| 554 | except (ValueError, TypeError, OverflowError, RuntimeError) as error: |
| 555 | raise VLLMValidationError(f"Invalid metadata array: {key}") from error |
| 556 | |
| 557 | |
| 558 | def _get_embeds_data( |
| 559 | modality: str, |
| 560 | data_items: list[Any], |
| 561 | mm_processor: BaseMultiModalProcessor, |
| 562 | ): |
| 563 | if len(data_items) == 0: |
| 564 | return data_items |
| 565 | |
| 566 | if all(item is None for item in data_items): |
| 567 | return data_items |
| 568 | |
| 569 | if is_list_of(data_items, torch.Tensor): |
| 570 | embeds_key = f"{modality}_embeds" |
| 571 | dict_items = [{embeds_key: item} for item in data_items] |
| 572 | return _merge_embeds(dict_items, mm_processor)[embeds_key] |
| 573 | |
| 574 | if is_list_of(data_items, dict): |
| 575 | metadata_fields = mm_processor.info.data_parser.placeholder_metadata_fields( |
| 576 | modality |
| 577 | ) |
| 578 | data_items = [ |
| 579 | { |
| 580 | key: _parse_metadata_array(key, value, metadata_fields) |
| 581 | if isinstance(value, list) |
| 582 | else value |
| 583 | for key, value in item.items() |
| 584 | } |
| 585 | for item in data_items |
| 586 | ] |
| 587 | return _merge_embeds(data_items, mm_processor) |
| 588 | |
| 589 | raise NotImplementedError(type(data_items)) |
| 590 | |
| 591 | |
| 592 | class BaseMultiModalItemTracker(ABC, Generic[_T]): |
| 593 | """Tracks multi-modal items in a given request and ensures that the number |
| 594 | of multi-modal items in a given request does not exceed the configured |
| 595 | maximum per prompt. |
| 596 | """ |
| 597 | |
| 598 | def __init__( |
| 599 | self, |
| 600 | model_config: ModelConfig, |
| 601 | media_io_kwargs: dict[str, dict[str, Any]] | None = None, |
| 602 | ): |
| 603 | super().__init__() |
| 604 | |
| 605 | self._model_config = model_config |
| 606 | self._media_io_kwargs = media_io_kwargs |
| 607 | |
| 608 | self._items_by_modality = defaultdict[str, list[_T]](list) |
| 609 | # Track original modality for each vision_chunk item (image or video) |
| 610 | self._modality_order = defaultdict[str, list[str]](list) |
| 611 | |
| 612 | @cached_property |
| 613 | def use_unified_vision_chunk_modality(self) -> bool: |
| 614 | """Check if model uses unified vision_chunk modality for images/videos.""" |
| 615 | return getattr(self._model_config.hf_config, "use_unified_vision_chunk", False) |
| 616 | |
| 617 | @property |
| 618 | def model_config(self) -> ModelConfig: |
| 619 | return self._model_config |
| 620 | |
| 621 | @cached_property |
| 622 | def model_cls(self) -> type[SupportsMultiModal]: |
| 623 | from vllm.model_executor.model_loader import get_model_cls |
| 624 | |
| 625 | model_cls = get_model_cls(self.model_config) |
| 626 | return cast(type[SupportsMultiModal], model_cls) |
| 627 | |
| 628 | @property |
| 629 | def media_io_kwargs(self) -> dict[str, dict[str, Any]] | None: |
| 630 | return self._media_io_kwargs or ( |
| 631 | self._model_config.multimodal_config.media_io_kwargs |
| 632 | if self._model_config.multimodal_config |
| 633 | else None |
| 634 | ) |
| 635 | |
| 636 | @property |
| 637 | def allowed_local_media_path(self): |
| 638 | return self._model_config.allowed_local_media_path |
| 639 | |
| 640 | @property |
| 641 | def allowed_media_domains(self): |
| 642 | return self._model_config.allowed_media_domains |
| 643 | |
| 644 | @property |
| 645 | def mm_registry(self): |
| 646 | return MULTIMODAL_REGISTRY |
| 647 | |
| 648 | @cached_property |
| 649 | def mm_processor(self): |
| 650 | return self.mm_registry.create_processor(self.model_config) |
| 651 | |
| 652 | @property |
| 653 | def video_processor_name(self) -> str | None: |
| 654 | return get_video_processor_cls_name(self.model_config) |
| 655 | |
| 656 | def add(self, modality: ModalityStr, item: _T) -> str | None: |
| 657 | """Add a multi-modal item to the current prompt and returns the |
| 658 | placeholder string to use, if any. |
| 659 | |
| 660 | An optional uuid can be added which serves as a unique identifier of the |
| 661 | media. |
| 662 | |
| 663 | Note: |
| 664 | `prompt_embeds` bypass MM-processor validation because they are |
| 665 | pre-computed embeddings that do not go through any HF processor, encoder, |
| 666 | or model-specific placeholder logic. The corresponding placeholder string is |
| 667 | managed by the parser via `_add_placeholder`, so we return None here. |
| 668 | |
| 669 | """ |
| 670 | add_info = self._validate_add(modality) |
| 671 | if add_info is None: |
| 672 | self._items_by_modality["prompt_embeds"].append(item) |
| 673 | return None |
| 674 | |
| 675 | input_modality, original_modality, use_vision_chunk, num_items = add_info |
| 676 | |
| 677 | # Track original modality for vision_chunk items |
| 678 | if use_vision_chunk: |
| 679 | self._items_by_modality[input_modality].append(item) # type: ignore |
| 680 | self._modality_order["vision_chunk"].append(original_modality) |
| 681 | else: |
| 682 | self._items_by_modality[original_modality].append(item) |
| 683 | |
| 684 | return self.model_cls.get_placeholder_str(modality, num_items) |
| 685 | |
| 686 | def _validate_add(self, modality: ModalityStr) -> tuple[str, str, bool, int] | None: |
| 687 | """Validate that one more item of the modality can be tracked.""" |
| 688 | if modality == "prompt_embeds": |
| 689 | return None |
| 690 | |
| 691 | input_modality = modality.replace("_embeds", "") |
| 692 | original_modality = modality |
| 693 | use_vision_chunk = ( |
| 694 | self.use_unified_vision_chunk_modality |
| 695 | and original_modality in ["video", "image"] |
| 696 | ) |
| 697 | |
| 698 | # If use_unified_vision_chunk_modality is enabled, |
| 699 | # map image/video to vision_chunk |
| 700 | if use_vision_chunk: |
| 701 | # To avoid validation fail |
| 702 | # because models with use_unified_vision_chunk_modality=True |
| 703 | # will only accept vision_chunk modality. |
| 704 | input_modality = "vision_chunk" |
| 705 | num_items = len(self._items_by_modality[input_modality]) + 1 |
| 706 | else: |
| 707 | num_items = len(self._items_by_modality[original_modality]) + 1 |
| 708 | |
| 709 | mm_config = self.model_config.multimodal_config |
| 710 | if ( |
| 711 | mm_config is not None |
| 712 | and mm_config.enable_mm_embeds |
| 713 | and mm_config.get_limit_per_prompt(input_modality) == 0 |
| 714 | and original_modality.endswith("_embeds") |
| 715 | ): |
| 716 | # Skip validation: embeddings bypass limit when enable_mm_embeds=True |
| 717 | pass |
| 718 | else: |
| 719 | self.mm_processor.info.validate_num_items(input_modality, num_items) |
| 720 | |
| 721 | return input_modality, original_modality, use_vision_chunk, num_items |
| 722 | |
| 723 | @abstractmethod |
| 724 | def create_parser( |
| 725 | self, mm_processor_kwargs: dict[str, Any] | None = None |
| 726 | ) -> "BaseMultiModalContentParser": |
| 727 | raise NotImplementedError |
| 728 | |
| 729 | |
| 730 | def _resolve_vision_chunk_items( |
| 731 | vision_chunk_items: list[tuple[object, str | None]], |
| 732 | mm_processor: BaseMultiModalProcessor, |
| 733 | vision_chunks_modality_order: list[str], |
| 734 | ): |
| 735 | # Process vision_chunk items - extract from (data, modality) tuples |
| 736 | # and convert to VisionChunk types with proper UUID handling |
| 737 | vision_chunks_uuids = [uuid for data, uuid in vision_chunk_items] |
| 738 | |
| 739 | assert len(vision_chunk_items) == len(vision_chunks_modality_order), ( |
| 740 | f"vision_chunk items ({len(vision_chunk_items)}) and " |
| 741 | f"modality_order ({len(vision_chunks_modality_order)}) must have same length" |
| 742 | ) |
| 743 | |
| 744 | processed_chunks: list[VisionChunk] = [] |
| 745 | video_idx = 0 |
| 746 | for inner_modality, (data, uuid) in zip( |
| 747 | vision_chunks_modality_order, vision_chunk_items |
| 748 | ): |
| 749 | if inner_modality == "image": |
| 750 | # Cast data to proper type for image |
| 751 | # Use .media (PIL.Image) directly to avoid redundant |
| 752 | # bytes→PIL conversion in media_processor |
| 753 | if hasattr(data, "media"): |
| 754 | image_data = data.media # type: ignore[union-attr] |
| 755 | processed_chunks.append( |
| 756 | VisionChunkImage(type="image", image=image_data, uuid=uuid) |
| 757 | ) |
| 758 | else: |
| 759 | processed_chunks.append(data) # type: ignore[arg-type] |
| 760 | elif inner_modality == "video": |
| 761 | # For video, we may need to split into chunks |
| 762 | # if processor supports it |
| 763 | # For now, just wrap as a video chunk placeholder |
| 764 | if hasattr(mm_processor, "split_video_chunks") and data is not None: |
| 765 | try: |
| 766 | video_uuid = uuid or random_uuid() |
| 767 | # video await result is (video_data, video_meta) tuple |
| 768 | if isinstance(data, tuple) and len(data) >= 1: |
| 769 | video_data = data[0] |
| 770 | else: |
| 771 | video_data = data |
| 772 | video_chunks = mm_processor.split_video_chunks(video_data) |
| 773 | for i, vc in enumerate(video_chunks): |
| 774 | processed_chunks.append( |
| 775 | VisionChunkVideo( |
| 776 | type="video_chunk", |
| 777 | video_chunk=vc["video_chunk"], |
| 778 | uuid=f"{video_uuid}-{i}", |
| 779 | video_idx=video_idx, |
| 780 | prompt=vc["prompt"], |
| 781 | ) |
| 782 | ) |
| 783 | video_idx += 1 |
| 784 | except Exception as e: |
| 785 | logger.warning("Failed to split video chunks: %s", e) |
| 786 | processed_chunks.append(data) # type: ignore[arg-type] |
| 787 | else: |
| 788 | processed_chunks.append(data) # type: ignore[arg-type] |
| 789 | return processed_chunks, vision_chunks_uuids |
| 790 | |
| 791 | |
| 792 | def _resolve_items( |
| 793 | items_by_modality: dict[str, list[tuple[object, str | None]]], |
| 794 | mm_processor: BaseMultiModalProcessor | None, |
| 795 | modality_order: dict[str, list[str]], |
| 796 | ) -> tuple[MultiModalDataDict, MultiModalUUIDDict]: |
| 797 | """Materialize the tracker's per-modality items into `mm_data` / `mm_uuids`. |
| 798 | |
| 799 | Note: |
| 800 | `mm_processor` is `None` for text-only models (no registered HF |
| 801 | processor) whose only modality is `prompt_embeds`. Every other |
| 802 | modality requires a processor, enforced by the guard below. |
| 803 | |
| 804 | """ |
| 805 | if "image" in items_by_modality and "image_embeds" in items_by_modality: |
| 806 | raise VLLMValidationError( |
| 807 | "Mixing raw image and embedding inputs is not allowed", |
| 808 | parameter="image_embeds", |
| 809 | ) |
| 810 | if "audio" in items_by_modality and "audio_embeds" in items_by_modality: |
| 811 | raise VLLMValidationError( |
| 812 | "Mixing raw audio and embedding inputs is not allowed", |
| 813 | parameter="audio_embeds", |
| 814 | ) |
| 815 | if "video" in items_by_modality and "video_embeds" in items_by_modality: |
| 816 | raise VLLMValidationError( |
| 817 | "Mixing raw video and embedding inputs is not allowed", |
| 818 | parameter="video_embeds", |
| 819 | ) |
| 820 | # `prompt_embeds` bypasses HF MM processors. Every other modality requires one. |
| 821 | processor_modalities = items_by_modality.keys() - {"prompt_embeds"} |
| 822 | if processor_modalities and mm_processor is None: |
| 823 | raise RuntimeError( |
| 824 | _REQUIRE_MM_PROCESSOR_ERROR.format(modality=processor_modalities) |
| 825 | ) |
| 826 | |
| 827 | mm_data = {} |
| 828 | mm_uuids = {} |
| 829 | if "image_embeds" in items_by_modality: |
| 830 | assert mm_processor is not None |
| 831 | mm_data["image"] = _get_embeds_data( |
| 832 | "image", |
| 833 | [data for data, uuid in items_by_modality["image_embeds"]], |
| 834 | mm_processor, |
| 835 | ) |
| 836 | mm_uuids["image"] = [uuid for data, uuid in items_by_modality["image_embeds"]] |
| 837 | if "image" in items_by_modality: |
| 838 | mm_data["image"] = [data for data, uuid in items_by_modality["image"]] |
| 839 | mm_uuids["image"] = [uuid for data, uuid in items_by_modality["image"]] |
| 840 | if "audio_embeds" in items_by_modality: |
| 841 | assert mm_processor is not None |
| 842 | mm_data["audio"] = _get_embeds_data( |
| 843 | "audio", |
| 844 | [data for data, uuid in items_by_modality["audio_embeds"]], |
| 845 | mm_processor, |
| 846 | ) |
| 847 | mm_uuids["audio"] = [uuid for data, uuid in items_by_modality["audio_embeds"]] |
| 848 | if "audio" in items_by_modality: |
| 849 | mm_data["audio"] = [data for data, uuid in items_by_modality["audio"]] |
| 850 | mm_uuids["audio"] = [uuid for data, uuid in items_by_modality["audio"]] |
| 851 | if "video" in items_by_modality: |
| 852 | mm_data["video"] = [data for data, uuid in items_by_modality["video"]] |
| 853 | mm_uuids["video"] = [uuid for data, uuid in items_by_modality["video"]] |
| 854 | if "video_embeds" in items_by_modality: |
| 855 | assert mm_processor is not None |
| 856 | mm_data["video"] = _get_embeds_data( |
| 857 | "video", |
| 858 | [data for data, uuid in items_by_modality["video_embeds"]], |
| 859 | mm_processor, |
| 860 | ) |
| 861 | mm_uuids["video"] = [uuid for data, uuid in items_by_modality["video_embeds"]] |
| 862 | if "vision_chunk" in items_by_modality: |
| 863 | assert mm_processor is not None |
| 864 | # Process vision_chunk items - extract from (data, modality) tuples |
| 865 | # and convert to VisionChunk types with proper UUID handling |
| 866 | processed_chunks, vision_chunk_uuids = _resolve_vision_chunk_items( |
| 867 | items_by_modality["vision_chunk"], |
| 868 | mm_processor, |
| 869 | modality_order.get("vision_chunk", []), |
| 870 | ) |
| 871 | mm_data["vision_chunk"] = processed_chunks |
| 872 | mm_uuids["vision_chunk"] = vision_chunk_uuids |
| 873 | if "prompt_embeds" in items_by_modality: |
| 874 | mm_data["prompt_embeds"] = [ |
| 875 | data for data, _uuid in items_by_modality["prompt_embeds"] |
| 876 | ] |
| 877 | |
| 878 | return mm_data, mm_uuids |
| 879 | |
| 880 | |
| 881 | class MultiModalItemTracker(BaseMultiModalItemTracker[tuple[object, str | None]]): |
| 882 | def resolve_items( |
| 883 | self, |
| 884 | ) -> tuple[MultiModalDataDict | None, MultiModalUUIDDict | None]: |
| 885 | if not self._items_by_modality: |
| 886 | return None, None |
| 887 | |
| 888 | # Text-only models (`is_multimodal_model=False`) with inputs of |
| 889 | # modality `prompt_embeds` have no MM processor since `prompt_embeds` are |
| 890 | # pre-computed and require no processing, so we pass `None`. |
| 891 | mm_processor = ( |
| 892 | self.mm_processor if self._model_config.is_multimodal_model else None |
| 893 | ) |
| 894 | return _resolve_items( |
| 895 | dict(self._items_by_modality), |
| 896 | mm_processor, |
| 897 | self._modality_order, |
| 898 | ) |
| 899 | |
| 900 | def create_parser( |
| 901 | self, mm_processor_kwargs: dict[str, Any] | None = None |
| 902 | ) -> "BaseMultiModalContentParser": |
| 903 | return MultiModalContentParser(self, mm_processor_kwargs=mm_processor_kwargs) |
| 904 | |
| 905 | |
| 906 | class AsyncMultiModalItemTracker(BaseMultiModalItemTracker[_AsyncMultiModalItem]): |
| 907 | async def resolve_items( |
| 908 | self, |
| 909 | ) -> tuple[MultiModalDataDict | None, MultiModalUUIDDict | None]: |
| 910 | if not self._items_by_modality: |
| 911 | return None, None |
| 912 | |
| 913 | # Fetch all modalities together. Each tracked item is already an |
| 914 | # independent awaitable, and the async connector offloads blocking |
| 915 | # decode work, so waiting for one modality before starting the next |
| 916 | # needlessly adds their latency. |
| 917 | # Keep the original group and item order when rebuilding the result. |
| 918 | item_groups = list(self._items_by_modality.items()) |
| 919 | items = [item for _, group in item_groups for item in group] |
| 920 | results = await asyncio.gather( |
| 921 | *(item() for item in items), return_exceptions=True |
| 922 | ) |
| 923 | for result in results: |
| 924 | if isinstance(result, BaseException): |
| 925 | # Gathering with return_exceptions=True lets every task finish |
| 926 | # (or itself fail) before we raise, instead of abandoning |
| 927 | # still-in-flight fetches (real network/thread-pool work) the |
| 928 | # moment the first one fails. |
| 929 | raise result |
| 930 | |
| 931 | resolved_items_by_modality: dict[str, list[Any]] = {} |
| 932 | result_idx = 0 |
| 933 | for modality, group in item_groups: |
| 934 | next_result_idx = result_idx + len(group) |
| 935 | resolved_items_by_modality[modality] = results[result_idx:next_result_idx] |
| 936 | result_idx = next_result_idx |
| 937 | |
| 938 | mm_processor = ( |
| 939 | self.mm_processor if self._model_config.is_multimodal_model else None |
| 940 | ) |
| 941 | return _resolve_items( |
| 942 | resolved_items_by_modality, |
| 943 | mm_processor, |
| 944 | self._modality_order, |
| 945 | ) |
| 946 | |
| 947 | def create_parser( |
| 948 | self, mm_processor_kwargs: dict[str, Any] | None = None |
| 949 | ) -> "BaseMultiModalContentParser": |
| 950 | return AsyncMultiModalContentParser( |
| 951 | self, mm_processor_kwargs=mm_processor_kwargs |
| 952 | ) |
| 953 | |
| 954 | |
| 955 | class BaseMultiModalContentParser(ABC): |
| 956 | def __init__(self) -> None: |
| 957 | super().__init__() |
| 958 | |
| 959 | # stores model placeholders list with corresponding |
| 960 | # general MM placeholder: |
| 961 | # { |
| 962 | # "<##IMAGE##>": ["<image>", "<image>", "<image>"], |
| 963 | # "<##AUDIO##>": ["<audio>", "<audio>"], |
| 964 | # "<##PROMPT_EMBEDS##>": ["<prompt_embeds>", "<prompt_embeds>"] |
| 965 | # } |
| 966 | self._placeholder_storage: dict[str, list] = defaultdict(list) |
| 967 | |
| 968 | @property |
| 969 | @abstractmethod |
| 970 | def model_config(self) -> ModelConfig: |
| 971 | raise NotImplementedError |
| 972 | |
| 973 | def _add_placeholder(self, modality: ModalityStr, placeholder: str | None): |
| 974 | mod_placeholder = MODALITY_PLACEHOLDERS_MAP[modality] |
| 975 | if placeholder: |
| 976 | self._placeholder_storage[mod_placeholder].append(placeholder) |
| 977 | |
| 978 | def mm_placeholder_storage(self) -> dict[str, list]: |
| 979 | return dict(self._placeholder_storage) |
| 980 | |
| 981 | @abstractmethod |
| 982 | def parse_image(self, image_url: str | None, uuid: str | None = None) -> None: |
| 983 | raise NotImplementedError |
| 984 | |
| 985 | @abstractmethod |
| 986 | def parse_image_embeds( |
| 987 | self, |
| 988 | image_embeds: MultiModalEmbedsPayload | None, |
| 989 | uuid: str | None = None, |
| 990 | ) -> None: |
| 991 | raise NotImplementedError |
| 992 | |
| 993 | @abstractmethod |
| 994 | def parse_image_pil( |
| 995 | self, image_pil: Image.Image | None, uuid: str | None = None |
| 996 | ) -> None: |
| 997 | raise NotImplementedError |
| 998 | |
| 999 | @abstractmethod |
| 1000 | def parse_audio(self, audio_url: str | None, uuid: str | None = None) -> None: |
| 1001 | raise NotImplementedError |
| 1002 | |
| 1003 | @abstractmethod |
| 1004 | def parse_input_audio( |
| 1005 | self, input_audio: InputAudio | None, uuid: str | None = None |
| 1006 | ) -> None: |
| 1007 | raise NotImplementedError |
| 1008 | |
| 1009 | @abstractmethod |
| 1010 | def parse_audio_embeds( |
| 1011 | self, |
| 1012 | audio_embeds: MultiModalEmbedsPayload | None, |
| 1013 | uuid: str | None = None, |
| 1014 | ) -> None: |
| 1015 | raise NotImplementedError |
| 1016 | |
| 1017 | @abstractmethod |
| 1018 | def parse_prompt_embeds(self, data: str) -> None: |
| 1019 | raise NotImplementedError |
| 1020 | |
| 1021 | @abstractmethod |
| 1022 | def parse_video(self, video_url: str | None, uuid: str | None = None) -> None: |
| 1023 | raise NotImplementedError |
| 1024 | |
| 1025 | @abstractmethod |
| 1026 | def parse_video_embeds( |
| 1027 | self, |
| 1028 | video_embeds: MultiModalEmbedsPayload | None, |
| 1029 | uuid: str | None = None, |
| 1030 | ) -> None: |
| 1031 | raise NotImplementedError |
| 1032 | |
| 1033 | |
| 1034 | class MultiModalContentParser(BaseMultiModalContentParser): |
| 1035 | def __init__( |
| 1036 | self, |
| 1037 | tracker: MultiModalItemTracker, |
| 1038 | mm_processor_kwargs: dict[str, Any] | None = None, |
| 1039 | ) -> None: |
| 1040 | super().__init__() |
| 1041 | |
| 1042 | self._tracker = tracker |
| 1043 | self._mm_processor_kwargs = mm_processor_kwargs |
| 1044 | |
| 1045 | @cached_property |
| 1046 | def _connector(self) -> MediaConnector: |
| 1047 | # Connector setup may probe VLLM_MEDIA_CACHE. Defer it until a request |
| 1048 | # actually contains media so text-only parsing never blocks on that I/O. |
| 1049 | return MEDIA_CONNECTOR_REGISTRY.load( |
| 1050 | envs.VLLM_MEDIA_CONNECTOR, |
| 1051 | media_io_kwargs=self._tracker.media_io_kwargs, |
| 1052 | allowed_local_media_path=self._tracker.allowed_local_media_path, |
| 1053 | allowed_media_domains=self._tracker.allowed_media_domains, |
| 1054 | ) |
| 1055 | |
| 1056 | @property |
| 1057 | def model_config(self) -> ModelConfig: |
| 1058 | return self._tracker.model_config |
| 1059 | |
| 1060 | @override |
| 1061 | def parse_prompt_embeds(self, data: str) -> None: |
| 1062 | """Decode a base64 prompt embeds tensor and store it in the tracker. |
| 1063 | |
| 1064 | Emits a single `PROMPT_EMBEDS_PLACEHOLDER_TOKEN` sentinel per |
| 1065 | content part. The renderer later expands each sentinel to a span of |
| 1066 | `tensor.shape[0]` placeholder tokens after tokenization. |
| 1067 | """ |
| 1068 | if not self.model_config.enable_prompt_embeds: |
| 1069 | raise VLLMValidationError( |
| 1070 | _ENABLE_PROMPT_EMBEDS_ERROR, parameter="prompt_embeds" |
| 1071 | ) |
| 1072 | |
| 1073 | tensor = safe_load_prompt_embeds(self.model_config, data.encode()) |
| 1074 | self._tracker.add("prompt_embeds", (tensor, None)) |
| 1075 | self._add_placeholder("prompt_embeds", PROMPT_EMBEDS_PLACEHOLDER_TOKEN) |
| 1076 | |
| 1077 | def parse_image(self, image_url: str | None, uuid: str | None = None) -> None: |
| 1078 | image = self._connector.fetch_image(image_url) if image_url else None |
| 1079 | |
| 1080 | placeholder = self._tracker.add("image", (image, uuid)) |
| 1081 | self._add_placeholder("image", placeholder) |
| 1082 | |
| 1083 | def parse_image_embeds( |
| 1084 | self, |
| 1085 | image_embeds: MultiModalEmbedsPayload | None, |
| 1086 | uuid: str | None = None, |
| 1087 | ) -> None: |
| 1088 | mm_config = self.model_config.get_multimodal_config() |
| 1089 | if not mm_config.enable_mm_embeds: |
| 1090 | raise VLLMValidationError( |
| 1091 | "You must set `--enable-mm-embeds` to input `image_embeds`", |
| 1092 | parameter="image_embeds", |
| 1093 | ) |
| 1094 | |
| 1095 | if isinstance(image_embeds, dict): |
| 1096 | embeds = { |
| 1097 | k: self._connector.fetch_image_embedding(v) if isinstance(v, str) else v |
| 1098 | for k, v in image_embeds.items() |
| 1099 | } |
| 1100 | placeholder = self._tracker.add("image_embeds", (embeds, uuid)) |
| 1101 | |
| 1102 | if isinstance(image_embeds, str): |
| 1103 | embedding = self._connector.fetch_image_embedding(image_embeds) |
| 1104 | placeholder = self._tracker.add("image_embeds", (embedding, uuid)) |
| 1105 | |
| 1106 | if image_embeds is None: |
| 1107 | placeholder = self._tracker.add("image_embeds", (None, uuid)) |
| 1108 | |
| 1109 | self._add_placeholder("image", placeholder) |
| 1110 | |
| 1111 | def parse_audio_embeds( |
| 1112 | self, |
| 1113 | audio_embeds: MultiModalEmbedsPayload | None, |
| 1114 | uuid: str | None = None, |
| 1115 | ) -> None: |
| 1116 | mm_config = self.model_config.get_multimodal_config() |
| 1117 | if not mm_config.enable_mm_embeds: |
| 1118 | raise VLLMValidationError( |
| 1119 | "You must set `--enable-mm-embeds` to input `audio_embeds`", |
| 1120 | parameter="audio_embeds", |
| 1121 | ) |
| 1122 | |
| 1123 | if isinstance(audio_embeds, dict): |
| 1124 | embeds = { |
| 1125 | k: self._connector.fetch_audio_embedding(v) if isinstance(v, str) else v |
| 1126 | for k, v in audio_embeds.items() |
| 1127 | } |
| 1128 | placeholder = self._tracker.add("audio_embeds", (embeds, uuid)) |
| 1129 | elif isinstance(audio_embeds, str): |
| 1130 | embedding = self._connector.fetch_audio_embedding(audio_embeds) |
| 1131 | placeholder = self._tracker.add("audio_embeds", (embedding, uuid)) |
| 1132 | else: |
| 1133 | placeholder = self._tracker.add("audio_embeds", (None, uuid)) |
| 1134 | |
| 1135 | self._add_placeholder("audio", placeholder) |
| 1136 | |
| 1137 | def parse_image_pil( |
| 1138 | self, image_pil: Image.Image | None, uuid: str | None = None |
| 1139 | ) -> None: |
| 1140 | placeholder = self._tracker.add("image", (image_pil, uuid)) |
| 1141 | self._add_placeholder("image", placeholder) |
| 1142 | |
| 1143 | def parse_audio(self, audio_url: str | None, uuid: str | None = None) -> None: |
| 1144 | audio = self._connector.fetch_audio(audio_url) if audio_url else None |
| 1145 | |
| 1146 | placeholder = self._tracker.add("audio", (audio, uuid)) |
| 1147 | self._add_placeholder("audio", placeholder) |
| 1148 | |
| 1149 | def parse_input_audio( |
| 1150 | self, input_audio: InputAudio | None, uuid: str | None = None |
| 1151 | ) -> None: |
| 1152 | if input_audio: |
| 1153 | audio_data = input_audio.get("data", "") |
| 1154 | audio_format = input_audio.get("format", "") |
| 1155 | if audio_data: |
| 1156 | audio_url = f"data:audio/{audio_format};base64,{audio_data}" |
| 1157 | else: |
| 1158 | # If a UUID is provided, audio data may be empty. |
| 1159 | audio_url = None |
| 1160 | else: |
| 1161 | audio_url = None |
| 1162 | |
| 1163 | return self.parse_audio(audio_url, uuid) |
| 1164 | |
| 1165 | def parse_video(self, video_url: str | None, uuid: str | None = None) -> None: |
| 1166 | video = ( |
| 1167 | self._connector.fetch_video( |
| 1168 | video_url=video_url, |
| 1169 | video_processor=self._tracker.video_processor_name, |
| 1170 | ) |
| 1171 | if video_url |
| 1172 | else None |
| 1173 | ) |
| 1174 | |
| 1175 | placeholder = self._tracker.add("video", (video, uuid)) |
| 1176 | self._add_placeholder("video", placeholder) |
| 1177 | |
| 1178 | # Extract audio from video if use_audio_in_video is True |
| 1179 | if ( |
| 1180 | video_url |
| 1181 | and self._mm_processor_kwargs |
| 1182 | and self._mm_processor_kwargs.get("use_audio_in_video", False) |
| 1183 | ): |
| 1184 | audio = self._connector.fetch_audio(video_url) if video_url else None |
| 1185 | audio_placeholder = self._tracker.add("audio", (audio, uuid)) |
| 1186 | self._add_placeholder("audio", audio_placeholder) |
| 1187 | |
| 1188 | def parse_video_embeds( |
| 1189 | self, |
| 1190 | video_embeds: MultiModalEmbedsPayload | None, |
| 1191 | uuid: str | None = None, |
| 1192 | ) -> None: |
| 1193 | mm_config = self.model_config.get_multimodal_config() |
| 1194 | if not mm_config.enable_mm_embeds: |
| 1195 | raise VLLMValidationError( |
| 1196 | "You must set `--enable-mm-embeds` to input `video_embeds`", |
| 1197 | parameter="video_embeds", |
| 1198 | ) |
| 1199 | |
| 1200 | if isinstance(video_embeds, dict): |
| 1201 | embeds = { |
| 1202 | k: self._connector.fetch_video_embedding(v) if isinstance(v, str) else v |
| 1203 | for k, v in video_embeds.items() |
| 1204 | } |
| 1205 | placeholder = self._tracker.add("video_embeds", (embeds, uuid)) |
| 1206 | elif isinstance(video_embeds, str): |
| 1207 | embedding = self._connector.fetch_video_embedding(video_embeds) |
| 1208 | placeholder = self._tracker.add("video_embeds", (embedding, uuid)) |
| 1209 | else: |
| 1210 | placeholder = self._tracker.add("video_embeds", (None, uuid)) |
| 1211 | |
| 1212 | self._add_placeholder("video", placeholder) |
| 1213 | |
| 1214 | |
| 1215 | class AsyncMultiModalContentParser(BaseMultiModalContentParser): |
| 1216 | def __init__( |
| 1217 | self, |
| 1218 | tracker: AsyncMultiModalItemTracker, |
| 1219 | mm_processor_kwargs: dict[str, Any] | None = None, |
| 1220 | ) -> None: |
| 1221 | super().__init__() |
| 1222 | |
| 1223 | self._tracker = tracker |
| 1224 | self._mm_processor_kwargs: dict[str, Any] | None = mm_processor_kwargs |
| 1225 | |
| 1226 | @cached_property |
| 1227 | def _connector(self) -> MediaConnector: |
| 1228 | # Connector setup may probe VLLM_MEDIA_CACHE. Defer it until a request |
| 1229 | # actually contains media so text-only parsing never blocks on that I/O. |
| 1230 | return MEDIA_CONNECTOR_REGISTRY.load( |
| 1231 | envs.VLLM_MEDIA_CONNECTOR, |
| 1232 | media_io_kwargs=self._tracker.media_io_kwargs, |
| 1233 | allowed_local_media_path=self._tracker.allowed_local_media_path, |
| 1234 | allowed_media_domains=self._tracker.allowed_media_domains, |
| 1235 | ) |
| 1236 | |
| 1237 | @property |
| 1238 | def model_config(self) -> ModelConfig: |
| 1239 | return self._tracker.model_config |
| 1240 | |
| 1241 | async def _item_with_uuid_async(self, item: object, uuid: str | None): |
| 1242 | return item, uuid |
| 1243 | |
| 1244 | @override |
| 1245 | def parse_prompt_embeds(self, data: str) -> None: |
| 1246 | """Schedule async prompt embeds decode and store the coroutine in the tracker. |
| 1247 | |
| 1248 | Like the sync variant, emits a single sentinel `PROMPT_EMBEDS_PLACEHOLDER_TOKEN` |
| 1249 | per content part. Unlike the sync variant, the tensor decode is deferred to a |
| 1250 | thread-pool executor via `safe_load_prompt_embeds_async`. |
| 1251 | """ |
| 1252 | if not self.model_config.enable_prompt_embeds: |
| 1253 | raise VLLMValidationError( |
| 1254 | _ENABLE_PROMPT_EMBEDS_ERROR, parameter="prompt_embeds" |
| 1255 | ) |
| 1256 | |
| 1257 | self._tracker.add( |
| 1258 | "prompt_embeds", partial(self._load_prompt_embeds_async, data.encode()) |
| 1259 | ) |
| 1260 | self._add_placeholder("prompt_embeds", PROMPT_EMBEDS_PLACEHOLDER_TOKEN) |
| 1261 | |
| 1262 | async def _load_prompt_embeds_async( |
| 1263 | self, data_bytes: bytes |
| 1264 | ) -> tuple[torch.Tensor, None]: |
| 1265 | # Second tuple slot fills the tracker's generic `(item, uuid | None)` |
| 1266 | # contract. prompt_embeds has no UUID concept, so it's always `None`. |
| 1267 | tensor = await safe_load_prompt_embeds_async(self.model_config, data_bytes) |
| 1268 | return tensor, None |
| 1269 | |
| 1270 | async def _image_with_uuid_async(self, image_url: str | None, uuid: str | None): |
| 1271 | image = ( |
| 1272 | await self._connector.fetch_image_async(image_url) if image_url else None |
| 1273 | ) |
| 1274 | return image, uuid |
| 1275 | |
| 1276 | def parse_image(self, image_url: str | None, uuid: str | None = None) -> None: |
| 1277 | placeholder = self._tracker.add( |
| 1278 | "image", partial(self._image_with_uuid_async, image_url, uuid) |
| 1279 | ) |
| 1280 | self._add_placeholder("image", placeholder) |
| 1281 | |
| 1282 | def parse_image_embeds( |
| 1283 | self, |
| 1284 | image_embeds: MultiModalEmbedsPayload | None, |
| 1285 | uuid: str | None = None, |
| 1286 | ) -> None: |
| 1287 | mm_config = self.model_config.get_multimodal_config() |
| 1288 | if not mm_config.enable_mm_embeds: |
| 1289 | raise VLLMValidationError( |
| 1290 | "You must set `--enable-mm-embeds` to input `image_embeds`", |
| 1291 | parameter="image_embeds", |
| 1292 | ) |
| 1293 | |
| 1294 | placeholder = self._tracker.add( |
| 1295 | "image_embeds", |
| 1296 | partial(self._image_embeds_with_uuid_async, image_embeds, uuid), |
| 1297 | ) |
| 1298 | self._add_placeholder("image", placeholder) |
| 1299 | |
| 1300 | async def _image_embeds_with_uuid_async( |
| 1301 | self, |
| 1302 | image_embeds: MultiModalEmbedsPayload | None, |
| 1303 | uuid: str | None, |
| 1304 | ): |
| 1305 | if isinstance(image_embeds, dict): |
| 1306 | embeds = await _load_embeds_dict( |
| 1307 | image_embeds, self._connector.fetch_image_embedding_async |
| 1308 | ) |
| 1309 | elif isinstance(image_embeds, str): |
| 1310 | embeds = await self._connector.fetch_image_embedding_async(image_embeds) |
| 1311 | else: |
| 1312 | embeds = None |
| 1313 | return embeds, uuid |
| 1314 | |
| 1315 | def parse_audio_embeds( |
| 1316 | self, |
| 1317 | audio_embeds: MultiModalEmbedsPayload | None, |
| 1318 | uuid: str | None = None, |
| 1319 | ) -> None: |
| 1320 | mm_config = self.model_config.get_multimodal_config() |
| 1321 | if not mm_config.enable_mm_embeds: |
| 1322 | raise VLLMValidationError( |
| 1323 | "You must set `--enable-mm-embeds` to input `audio_embeds`", |
| 1324 | parameter="audio_embeds", |
| 1325 | ) |
| 1326 | |
| 1327 | placeholder = self._tracker.add( |
| 1328 | "audio_embeds", |
| 1329 | partial(self._audio_embeds_with_uuid_async, audio_embeds, uuid), |
| 1330 | ) |
| 1331 | self._add_placeholder("audio", placeholder) |
| 1332 | |
| 1333 | async def _audio_embeds_with_uuid_async( |
| 1334 | self, |
| 1335 | audio_embeds: MultiModalEmbedsPayload | None, |
| 1336 | uuid: str | None, |
| 1337 | ): |
| 1338 | if isinstance(audio_embeds, dict): |
| 1339 | embeds = await _load_embeds_dict( |
| 1340 | audio_embeds, self._connector.fetch_audio_embedding_async |
| 1341 | ) |
| 1342 | elif isinstance(audio_embeds, str): |
| 1343 | embeds = await self._connector.fetch_audio_embedding_async(audio_embeds) |
| 1344 | else: |
| 1345 | embeds = None |
| 1346 | return embeds, uuid |
| 1347 | |
| 1348 | def parse_image_pil( |
| 1349 | self, |
| 1350 | image_pil: Image.Image | None, |
| 1351 | uuid: str | None = None, |
| 1352 | ) -> None: |
| 1353 | placeholder = self._tracker.add( |
| 1354 | "image", partial(self._item_with_uuid_async, image_pil, uuid) |
| 1355 | ) |
| 1356 | self._add_placeholder("image", placeholder) |
| 1357 | |
| 1358 | async def _audio_with_uuid_async(self, audio_url: str | None, uuid: str | None): |
| 1359 | audio = ( |
| 1360 | await self._connector.fetch_audio_async(audio_url) if audio_url else None |
| 1361 | ) |
| 1362 | return audio, uuid |
| 1363 | |
| 1364 | def parse_audio(self, audio_url: str | None, uuid: str | None = None) -> None: |
| 1365 | placeholder = self._tracker.add( |
| 1366 | "audio", partial(self._audio_with_uuid_async, audio_url, uuid) |
| 1367 | ) |
| 1368 | self._add_placeholder("audio", placeholder) |
| 1369 | |
| 1370 | def parse_input_audio( |
| 1371 | self, input_audio: InputAudio | None, uuid: str | None = None |
| 1372 | ) -> None: |
| 1373 | if input_audio: |
| 1374 | audio_data = input_audio.get("data", "") |
| 1375 | audio_format = input_audio.get("format", "") |
| 1376 | if audio_data: |
| 1377 | audio_url = f"data:audio/{audio_format};base64,{audio_data}" |
| 1378 | else: |
| 1379 | # If a UUID is provided, audio data may be empty. |
| 1380 | audio_url = None |
| 1381 | else: |
| 1382 | audio_url = None |
| 1383 | |
| 1384 | return self.parse_audio(audio_url, uuid) |
| 1385 | |
| 1386 | async def _video_with_uuid_async(self, video_url: str | None, uuid: str | None): |
| 1387 | video = ( |
| 1388 | await self._connector.fetch_video_async( |
| 1389 | video_url, |
| 1390 | video_processor=self._tracker.video_processor_name, |
| 1391 | ) |
| 1392 | if video_url |
| 1393 | else None |
| 1394 | ) |
| 1395 | return video, uuid |
| 1396 | |
| 1397 | def parse_video(self, video_url: str | None, uuid: str | None = None) -> None: |
| 1398 | placeholder = self._tracker.add( |
| 1399 | "video", partial(self._video_with_uuid_async, video_url, uuid) |
| 1400 | ) |
| 1401 | self._add_placeholder("video", placeholder) |
| 1402 | |
| 1403 | # Extract audio from video if use_audio_in_video is True |
| 1404 | if ( |
| 1405 | video_url |
| 1406 | and self._mm_processor_kwargs |
| 1407 | and self._mm_processor_kwargs.get("use_audio_in_video", False) |
| 1408 | ): |
| 1409 | audio_placeholder = self._tracker.add( |
| 1410 | "audio", partial(self._audio_with_uuid_async, video_url, uuid) |
| 1411 | ) |
| 1412 | self._add_placeholder("audio", audio_placeholder) |
| 1413 | |
| 1414 | def parse_video_embeds( |
| 1415 | self, |
| 1416 | video_embeds: MultiModalEmbedsPayload | None, |
| 1417 | uuid: str | None = None, |
| 1418 | ) -> None: |
| 1419 | mm_config = self.model_config.get_multimodal_config() |
| 1420 | if not mm_config.enable_mm_embeds: |
| 1421 | raise VLLMValidationError( |
| 1422 | "You must set `--enable-mm-embeds` to input `video_embeds`", |
| 1423 | parameter="video_embeds", |
| 1424 | ) |
| 1425 | |
| 1426 | placeholder = self._tracker.add( |
| 1427 | "video_embeds", |
| 1428 | partial(self._video_embeds_with_uuid_async, video_embeds, uuid), |
| 1429 | ) |
| 1430 | self._add_placeholder("video", placeholder) |
| 1431 | |
| 1432 | async def _video_embeds_with_uuid_async( |
| 1433 | self, |
| 1434 | video_embeds: MultiModalEmbedsPayload | None, |
| 1435 | uuid: str | None, |
| 1436 | ): |
| 1437 | if isinstance(video_embeds, dict): |
| 1438 | embeds = await _load_embeds_dict( |
| 1439 | video_embeds, self._connector.fetch_video_embedding_async |
| 1440 | ) |
| 1441 | elif isinstance(video_embeds, str): |
| 1442 | embeds = await self._connector.fetch_video_embedding_async(video_embeds) |
| 1443 | else: |
| 1444 | embeds = None |
| 1445 | return embeds, uuid |
| 1446 | |
| 1447 | |
| 1448 | @dataclass |
| 1449 | class ChatTemplateConfig: |
| 1450 | chat_template: str | None = None |
| 1451 | chat_template_content_format: ChatTemplateContentFormatOption = "auto" |
| 1452 | trust_request_chat_template: bool = False |
| 1453 | trust_request_mm_kwargs: bool = False |
| 1454 | |
| 1455 | |
| 1456 | def validate_chat_template(chat_template: Path | str | None): |
| 1457 | """Raises if the provided chat template appears invalid.""" |
| 1458 | if chat_template is None: |
| 1459 | return |
| 1460 | |
| 1461 | elif isinstance(chat_template, Path) and not chat_template.exists(): |
| 1462 | raise FileNotFoundError("the supplied chat template path doesn't exist") |
| 1463 | |
| 1464 | elif isinstance(chat_template, str): |
| 1465 | JINJA_CHARS = "{}\n" |
| 1466 | if ( |
| 1467 | not any(c in chat_template for c in JINJA_CHARS) |
| 1468 | and not Path(chat_template).exists() |
| 1469 | ): |
| 1470 | # Try to find the template in the built-in templates directory |
| 1471 | from vllm.transformers_utils.chat_templates.registry import ( |
| 1472 | CHAT_TEMPLATES_DIR, |
| 1473 | ) |
| 1474 | |
| 1475 | builtin_template_path = CHAT_TEMPLATES_DIR / chat_template |
| 1476 | if not builtin_template_path.exists(): |
| 1477 | raise VLLMValidationError( |
| 1478 | f"The supplied chat template string ({chat_template}) " |
| 1479 | f"appears path-like, but doesn't exist! " |
| 1480 | f"Tried: {chat_template} and {builtin_template_path}", |
| 1481 | parameter="chat_template", |
| 1482 | ) |
| 1483 | |
| 1484 | else: |
| 1485 | raise VLLMValidationError( |
| 1486 | f"{type(chat_template)} is not a valid chat template type", |
| 1487 | parameter="chat_template", |
| 1488 | ) |
| 1489 | |
| 1490 | |
| 1491 | def _load_chat_template( |
| 1492 | chat_template: Path | str | None, |
| 1493 | *, |
| 1494 | is_literal: bool = False, |
| 1495 | ) -> str | None: |
| 1496 | if chat_template is None: |
| 1497 | return None |
| 1498 | |
| 1499 | if is_literal: |
| 1500 | if isinstance(chat_template, Path): |
| 1501 | raise TypeError( |
| 1502 | "chat_template is expected to be read directly from its value" |
| 1503 | ) |
| 1504 | |
| 1505 | return chat_template |
| 1506 | |
| 1507 | try: |
| 1508 | with open(chat_template) as f: |
| 1509 | return f.read() |
| 1510 | except OSError as e: |
| 1511 | if isinstance(chat_template, Path): |
| 1512 | raise |
| 1513 | |
| 1514 | JINJA_CHARS = "{}\n" |
| 1515 | if not any(c in chat_template for c in JINJA_CHARS): |
| 1516 | # Try to load from the built-in templates directory |
| 1517 | from vllm.transformers_utils.chat_templates.registry import ( |
| 1518 | CHAT_TEMPLATES_DIR, |
| 1519 | ) |
| 1520 | |
| 1521 | builtin_template_path = CHAT_TEMPLATES_DIR / chat_template |
| 1522 | try: |
| 1523 | with open(builtin_template_path) as f: |
| 1524 | return f.read() |
| 1525 | except OSError: |
| 1526 | msg = ( |
| 1527 | f"The supplied chat template ({chat_template}) " |
| 1528 | f"looks like a file path, but it failed to be opened. " |
| 1529 | f"Tried: {chat_template} and {builtin_template_path}. " |
| 1530 | f"Reason: {e}" |
| 1531 | ) |
| 1532 | raise VLLMValidationError(msg, parameter="chat_template") from e |
| 1533 | |
| 1534 | # If opening a file fails, set chat template to be args to |
| 1535 | # ensure we decode so our escape are interpreted correctly |
| 1536 | return _load_chat_template(chat_template, is_literal=True) |
| 1537 | |
| 1538 | |
| 1539 | _cached_load_chat_template = lru_cache(_load_chat_template) |
| 1540 | |
| 1541 | |
| 1542 | def load_chat_template( |
| 1543 | chat_template: Path | str | None, |
| 1544 | *, |
| 1545 | is_literal: bool = False, |
| 1546 | ) -> str | None: |
| 1547 | return _cached_load_chat_template(chat_template, is_literal=is_literal) |
| 1548 | |
| 1549 | |
| 1550 | def _get_interleaved_text_prompt( |
| 1551 | placeholder_storage: dict[str, list], texts: list[str] |
| 1552 | ) -> str: |
| 1553 | for idx, elem in enumerate(texts): |
| 1554 | if elem in placeholder_storage: |
| 1555 | slots = placeholder_storage[elem] |
| 1556 | if not slots: |
| 1557 | raise VLLMValidationError( |
| 1558 | f"Found more '{elem}' placeholders in input prompt than " |
| 1559 | "actual multimodal data items." |
| 1560 | ) |
| 1561 | texts[idx] = slots.pop(0) |
| 1562 | |
| 1563 | return "\n".join(texts) |
| 1564 | |
| 1565 | |
| 1566 | # TODO: Let user specify how to insert multimodal tokens into prompt |
| 1567 | # (similar to chat template) |
| 1568 | def _get_full_multimodal_text_prompt( |
| 1569 | placeholder_storage: dict[str, list], |
| 1570 | texts: list[str], |
| 1571 | interleave_strings: bool, |
| 1572 | multimodal_content_part_separator: str = "\n", |
| 1573 | ) -> str: |
| 1574 | """Combine multimodal prompts for a multimodal language model.""" |
| 1575 | # flatten storage to make it looks like |
| 1576 | # { |
| 1577 | # "<|image|>": 2, |
| 1578 | # "<|audio|>": 1 |
| 1579 | # } |
| 1580 | placeholder_counts = Counter( |
| 1581 | [v for elem in placeholder_storage.values() for v in elem] |
| 1582 | ) |
| 1583 | |
| 1584 | if interleave_strings: |
| 1585 | text_prompt = _get_interleaved_text_prompt(placeholder_storage, texts) |
| 1586 | else: |
| 1587 | text_prompt = "\n".join(texts) |
| 1588 | |
| 1589 | # Pass interleaved text further in case the user used image placeholders |
| 1590 | # himself, but forgot to disable the 'interleave_strings' flag |
| 1591 | |
| 1592 | # Look through the text prompt to check for missing placeholders |
| 1593 | missing_placeholders: list[str] = [] |
| 1594 | for placeholder in placeholder_counts: |
| 1595 | # For any existing placeholder in the text prompt, we leave it as is |
| 1596 | placeholder_counts[placeholder] -= text_prompt.count(placeholder) |
| 1597 | |
| 1598 | if placeholder_counts[placeholder] < 0: |
| 1599 | logger.error( |
| 1600 | "Placeholder count is negative! " |
| 1601 | "Ensure that the 'interleave_strings' flag is disabled " |
| 1602 | "(current value: %s) " |
| 1603 | "when manually placing image placeholders.", |
| 1604 | interleave_strings, |
| 1605 | ) |
| 1606 | logger.debug("Input prompt: %s", text_prompt) |
| 1607 | raise VLLMValidationError( |
| 1608 | f"Found more '{placeholder}' placeholders in input prompt than " |
| 1609 | "actual multimodal data items.", |
| 1610 | parameter="messages", |
| 1611 | ) |
| 1612 | |
| 1613 | missing_placeholders.extend([placeholder] * placeholder_counts[placeholder]) |
| 1614 | |
| 1615 | # NOTE: Default behaviour: we always add missing placeholders |
| 1616 | # at the front of the prompt, if interleave_strings=False |
| 1617 | if text_prompt: |
| 1618 | return multimodal_content_part_separator.join( |
| 1619 | missing_placeholders + [text_prompt] |
| 1620 | ) |
| 1621 | else: |
| 1622 | return multimodal_content_part_separator.join(missing_placeholders) |
| 1623 | |
| 1624 | |
| 1625 | # No need to validate using Pydantic again |
| 1626 | _TextParser = partial(cast, ChatCompletionContentPartTextParam) |
| 1627 | _ImageEmbedsParser = partial(cast, ChatCompletionContentPartImageEmbedsParam) |
| 1628 | _AudioEmbedsParser = partial(cast, ChatCompletionContentPartAudioEmbedsParam) |
| 1629 | _VideoEmbedsParser = partial(cast, ChatCompletionContentPartVideoEmbedsParam) |
| 1630 | _PromptEmbedsParser = partial(cast, ChatCompletionContentPartPromptEmbedsParam) |
| 1631 | _InputAudioParser = partial(cast, ChatCompletionContentPartInputAudioParam) |
| 1632 | _RefusalParser = partial(cast, ChatCompletionContentPartRefusalParam) |
| 1633 | _PILImageParser = partial(cast, CustomChatCompletionContentPILImageParam) |
| 1634 | _ThinkParser = partial(cast, CustomThinkCompletionContentParam) |
| 1635 | # Need to validate url objects |
| 1636 | _ImageParser = TypeAdapter(ChatCompletionContentPartImageParam).validate_python |
| 1637 | _AudioParser = TypeAdapter(ChatCompletionContentPartAudioParam).validate_python |
| 1638 | _VideoParser = TypeAdapter(ChatCompletionContentPartVideoParam).validate_python |
| 1639 | |
| 1640 | _ResponsesInputImageParser = TypeAdapter(ResponseInputImageParam).validate_python |
| 1641 | _ContentPart: TypeAlias = ( |
| 1642 | MultiModalEmbedsPayload | dict[str, str] | InputAudio | PILImage |
| 1643 | ) |
| 1644 | |
| 1645 | # Define a mapping from part types to their corresponding parsing functions. |
| 1646 | MM_PARSER_MAP: dict[ |
| 1647 | str, |
| 1648 | Callable[[ChatCompletionContentPartParam], _ContentPart], |
| 1649 | ] = { |
| 1650 | "text": lambda part: _TextParser(part).get("text", None), |
| 1651 | "thinking": lambda part: _ThinkParser(part).get("thinking", None), |
| 1652 | "input_text": lambda part: _TextParser(part).get("text", None), |
| 1653 | "output_text": lambda part: _TextParser(part).get("text", None), |
| 1654 | "input_image": lambda part: _ResponsesInputImageParser(part).get("image_url", None), |
| 1655 | "image_url": lambda part: _ImageParser(part).get("image_url", {}).get("url", None), |
| 1656 | "image_embeds": lambda part: _ImageEmbedsParser(part).get("image_embeds", None), |
| 1657 | "audio_embeds": lambda part: _AudioEmbedsParser(part).get("audio_embeds", None), |
| 1658 | "video_embeds": lambda part: _VideoEmbedsParser(part).get("video_embeds", None), |
| 1659 | "prompt_embeds": lambda part: _PromptEmbedsParser(part).get("data", None), |
| 1660 | "image_pil": lambda part: _PILImageParser(part).get("image_pil", None), |
| 1661 | "audio_url": lambda part: _AudioParser(part).get("audio_url", {}).get("url", None), |
| 1662 | "input_audio": lambda part: _InputAudioParser(part).get("input_audio", None), |
| 1663 | "refusal": lambda part: _RefusalParser(part).get("refusal", None), |
| 1664 | "video_url": lambda part: _VideoParser(part).get("video_url", {}).get("url", None), |
| 1665 | "tool_reference": lambda part: cast( |
| 1666 | CustomChatCompletionContentToolReferenceParam, part |
| 1667 | ).get("name", None), |
| 1668 | } |
| 1669 | |
| 1670 | |
| 1671 | def _collect_known_content_part_fields() -> frozenset[str]: |
| 1672 | fields: set[str] = set() |
| 1673 | stack: list[Any] = [ChatCompletionContentPartParam] |
| 1674 | while stack: |
| 1675 | node = stack.pop() |
| 1676 | if get_origin(node) in (Union, types.UnionType): |
| 1677 | stack.extend(get_args(node)) |
| 1678 | elif hasattr(node, "__required_keys__"): |
| 1679 | fields |= node.__required_keys__ | node.__optional_keys__ |
| 1680 | return frozenset(fields) |
| 1681 | |
| 1682 | |
| 1683 | _KNOWN_CONTENT_PART_FIELDS = _collect_known_content_part_fields() |
| 1684 | |
| 1685 | |
| 1686 | def _collect_extra_fields(part: dict[str, Any]) -> dict[str, Any]: |
| 1687 | return {k: v for k, v in part.items() if k not in _KNOWN_CONTENT_PART_FIELDS} |
| 1688 | |
| 1689 | |
| 1690 | def _parse_chat_message_content_mm_part( |
| 1691 | part: ChatCompletionContentPartParam, |
| 1692 | ) -> tuple[str, _ContentPart]: |
| 1693 | """Parses a given multi-modal content part based on its type. |
| 1694 | |
| 1695 | Args: |
| 1696 | part: A dict containing the content part, with a potential 'type' field. |
| 1697 | |
| 1698 | Returns: |
| 1699 | A tuple (part_type, content) where: |
| 1700 | - part_type: Type of the part (e.g., 'text', 'image_url'). |
| 1701 | - content: Parsed content (e.g., text, image URL). |
| 1702 | |
| 1703 | Raises: |
| 1704 | ValueError: If the 'type' field is missing and no direct URL is found. |
| 1705 | |
| 1706 | """ |
| 1707 | assert isinstance( |
| 1708 | part, dict |
| 1709 | ) # This is needed to avoid mypy errors: part.get() from str |
| 1710 | part_type = part.get("type", None) |
| 1711 | uuid = part.get("uuid", None) |
| 1712 | |
| 1713 | if isinstance(part_type, str) and part_type in MM_PARSER_MAP and uuid is None: # noqa: E501 |
| 1714 | content = MM_PARSER_MAP[part_type](part) |
| 1715 | |
| 1716 | # Special case for 'image_url.detail' |
| 1717 | # We only support 'auto', which is the default |
| 1718 | if part_type == "image_url" and part.get("detail", "auto") != "auto": |
| 1719 | logger.warning( |
| 1720 | "'image_url.detail' is currently not supported and will be ignored." |
| 1721 | ) |
| 1722 | |
| 1723 | return part_type, content |
| 1724 | |
| 1725 | # Handle missing 'type' but provided direct URL fields. |
| 1726 | # 'type' is required field by pydantic |
| 1727 | if part_type is None or uuid is not None: |
| 1728 | if "image_url" in part: |
| 1729 | image_params = cast(CustomChatCompletionContentSimpleImageParam, part) |
| 1730 | image_url = image_params.get("image_url", None) |
| 1731 | if isinstance(image_url, dict): |
| 1732 | # Can potentially happen if user provides a uuid |
| 1733 | # with url as a dict of {"url": url} |
| 1734 | image_url = image_url.get("url", None) |
| 1735 | return "image_url", image_url |
| 1736 | if "image_pil" in part: |
| 1737 | # "image_pil" could be None if UUID is provided. |
| 1738 | image_params = cast( # type: ignore |
| 1739 | CustomChatCompletionContentPILImageParam, part |
| 1740 | ) |
| 1741 | image_pil = image_params.get("image_pil", None) |
| 1742 | return "image_pil", image_pil |
| 1743 | if "image_embeds" in part: |
| 1744 | # "image_embeds" could be None if UUID is provided. |
| 1745 | image_params = cast( # type: ignore |
| 1746 | ChatCompletionContentPartImageEmbedsParam, part |
| 1747 | ) |
| 1748 | image_embeds = image_params.get("image_embeds", None) |
| 1749 | return "image_embeds", image_embeds |
| 1750 | if "audio_embeds" in part: |
| 1751 | # "audio_embeds" could be None if UUID is provided. |
| 1752 | audio_params = cast( # type: ignore[assignment] |
| 1753 | ChatCompletionContentPartAudioEmbedsParam, part |
| 1754 | ) |
| 1755 | audio_embeds = audio_params.get("audio_embeds", None) |
| 1756 | return "audio_embeds", audio_embeds |
| 1757 | if "video_embeds" in part: |
| 1758 | # "video_embeds" could be None if UUID is provided. |
| 1759 | video_embeds_params = cast( # type: ignore[assignment] |
| 1760 | ChatCompletionContentPartVideoEmbedsParam, part |
| 1761 | ) |
| 1762 | video_embeds = video_embeds_params.get("video_embeds", None) |
| 1763 | return "video_embeds", video_embeds |
| 1764 | if "prompt_embeds" in part: |
| 1765 | prompt_embeds_params = cast( # type: ignore[assignment] |
| 1766 | ChatCompletionContentPartPromptEmbedsParam, part |
| 1767 | ) |
| 1768 | return "prompt_embeds", prompt_embeds_params.get("data", None) |
| 1769 | if "audio_url" in part: |
| 1770 | audio_params = cast( # type: ignore[assignment] |
| 1771 | CustomChatCompletionContentSimpleAudioParam, part |
| 1772 | ) |
| 1773 | audio_url = audio_params.get("audio_url", None) |
| 1774 | if isinstance(audio_url, dict): |
| 1775 | # Can potentially happen if user provides a uuid |
| 1776 | # with url as a dict of {"url": url} |
| 1777 | audio_url = audio_url.get("url", None) |
| 1778 | return "audio_url", audio_url |
| 1779 | if "input_audio" in part: |
| 1780 | input_audio_params = _InputAudioParser(part).get("input_audio", None) |
| 1781 | return "input_audio", input_audio_params |
| 1782 | if "video_url" in part: |
| 1783 | video_params = cast(CustomChatCompletionContentSimpleVideoParam, part) |
| 1784 | video_url = video_params.get("video_url", None) |
| 1785 | if isinstance(video_url, dict): |
| 1786 | # Can potentially happen if user provides a uuid |
| 1787 | # with url as a dict of {"url": url} |
| 1788 | video_url = video_url.get("url", None) |
| 1789 | return "video_url", video_url |
| 1790 | if "tool_reference" in part: |
| 1791 | tool_reference_params = cast( |
| 1792 | CustomChatCompletionContentToolReferenceParam, part |
| 1793 | ) |
| 1794 | tool_reference = tool_reference_params.get("name", None) |
| 1795 | return "tool_reference", tool_reference |
| 1796 | # Raise an error if no 'type' or direct URL is found. |
| 1797 | raise VLLMValidationError( |
| 1798 | "Missing 'type' field in multimodal part.", parameter="type" |
| 1799 | ) |
| 1800 | |
| 1801 | if not isinstance(part_type, str): |
| 1802 | raise VLLMValidationError( |
| 1803 | "Invalid 'type' field in multimodal part.", parameter="type" |
| 1804 | ) |
| 1805 | return part_type, "unknown part_type content" |
| 1806 | |
| 1807 | |
| 1808 | PART_TYPES_TO_SKIP_NONE_CONTENT = ( |
| 1809 | "text", |
| 1810 | "refusal", |
| 1811 | ) |
| 1812 | |
| 1813 | # Content part types parsed as text rather than multimodal data. |
| 1814 | TEXT_PART_TYPES = frozenset( |
| 1815 | {"text", "input_text", "output_text", "refusal", "thinking"} |
| 1816 | ) |
| 1817 | # Content part types that carry no multimodal data. |
| 1818 | _TEXT_CONTENT_PART_TYPES = TEXT_PART_TYPES | {"tool_reference"} |
| 1819 | # Keys that mark a content part as multimodal, whatever its ``type``. |
| 1820 | _MEDIA_CONTENT_PART_KEYS = frozenset(MM_PARSER_MAP) - _TEXT_CONTENT_PART_TYPES |
| 1821 | |
| 1822 | |
| 1823 | def has_non_text_content(messages: Any) -> bool: |
| 1824 | """Whether any message in ``messages`` has a non-text content part. |
| 1825 | |
| 1826 | Only list content is inspected, so validated chat content that is a |
| 1827 | one-shot iterator is never consumed. |
| 1828 | """ |
| 1829 | if not isinstance(messages, list): |
| 1830 | return False |
| 1831 | for msg in messages: |
| 1832 | content = msg.get("content") if isinstance(msg, dict) else None |
| 1833 | if not isinstance(content, list): |
| 1834 | continue |
| 1835 | for part in content: |
| 1836 | if isinstance(part, dict) and ( |
| 1837 | any(key in part for key in _MEDIA_CONTENT_PART_KEYS) |
| 1838 | or not isinstance(part_type := part.get("type", "text"), str) |
| 1839 | or part_type not in _TEXT_CONTENT_PART_TYPES |
| 1840 | ): |
| 1841 | return True |
| 1842 | return False |
| 1843 | |
| 1844 | |
| 1845 | def _parse_chat_message_content_parts( |
| 1846 | role: str, |
| 1847 | parts: Iterable[ChatCompletionContentPartParam], |
| 1848 | mm_tracker: BaseMultiModalItemTracker, |
| 1849 | *, |
| 1850 | wrap_dicts: bool, |
| 1851 | interleave_strings: bool, |
| 1852 | mm_processor_kwargs: dict[str, Any] | None = None, |
| 1853 | multimodal_content_part_separator="\n", |
| 1854 | ) -> list[ConversationMessage]: |
| 1855 | content = list[_ContentPart]() |
| 1856 | |
| 1857 | mm_parser = mm_tracker.create_parser(mm_processor_kwargs=mm_processor_kwargs) |
| 1858 | |
| 1859 | for part in parts: |
| 1860 | parse_res = _parse_chat_message_content_part( |
| 1861 | part, |
| 1862 | mm_parser, |
| 1863 | wrap_dicts=wrap_dicts, |
| 1864 | interleave_strings=interleave_strings, |
| 1865 | ) |
| 1866 | if parse_res: |
| 1867 | content.append(parse_res) |
| 1868 | |
| 1869 | if wrap_dicts: |
| 1870 | # Parsing wraps images and texts as interleaved dictionaries |
| 1871 | return [ConversationMessage(role=role, content=content)] # type: ignore |
| 1872 | texts = cast(list[str], content) |
| 1873 | mm_placeholder_storage = mm_parser.mm_placeholder_storage() |
| 1874 | if mm_placeholder_storage: |
| 1875 | text_prompt = _get_full_multimodal_text_prompt( |
| 1876 | mm_placeholder_storage, |
| 1877 | texts, |
| 1878 | interleave_strings, |
| 1879 | multimodal_content_part_separator=multimodal_content_part_separator, |
| 1880 | ) |
| 1881 | else: |
| 1882 | text_prompt = "\n".join(texts) |
| 1883 | |
| 1884 | return [ConversationMessage(role=role, content=text_prompt)] |
| 1885 | |
| 1886 | |
| 1887 | def _reject_reserved_placeholder_in_text(text: str, model_config: ModelConfig) -> None: |
| 1888 | """Reject user-supplied text parts that contains the reserved `prompt_embeds` |
| 1889 | placeholder sentinel. |
| 1890 | |
| 1891 | When the server accepts `prompt_embeds`, the placeholder token is |
| 1892 | registered as a single unsplittable special token on the tokenizer. Any |
| 1893 | user text that happens to contain the literal sequence would tokenize to |
| 1894 | the same ID and be mistaken for a splice point by the renderer, letting a |
| 1895 | caller move or inject splice positions via plain text content. |
| 1896 | """ |
| 1897 | if model_config.enable_prompt_embeds and PROMPT_EMBEDS_PLACEHOLDER_TOKEN in text: |
| 1898 | raise VLLMValidationError( |
| 1899 | _RESERVED_PLACEHOLDER_IN_TEXT_ERROR.format( |
| 1900 | token=PROMPT_EMBEDS_PLACEHOLDER_TOKEN |
| 1901 | ), |
| 1902 | parameter="messages", |
| 1903 | ) |
| 1904 | |
| 1905 | |
| 1906 | def _parse_chat_message_content_part( |
| 1907 | part: ChatCompletionContentPartParam, |
| 1908 | mm_parser: BaseMultiModalContentParser, |
| 1909 | *, |
| 1910 | wrap_dicts: bool, |
| 1911 | interleave_strings: bool, |
| 1912 | ) -> _ContentPart | None: |
| 1913 | """Parses a single part of a conversation. If wrap_dicts is True, |
| 1914 | structured dictionary pieces for texts and images will be |
| 1915 | wrapped in dictionaries, i.e., {"type": "text", "text", ...} and |
| 1916 | {"type": "image"}, respectively. Otherwise multimodal data will be |
| 1917 | handled by mm_parser, and texts will be returned as strings to be joined |
| 1918 | with multimodal placeholders. |
| 1919 | """ |
| 1920 | if isinstance(part, str): # Handle plain text parts |
| 1921 | _reject_reserved_placeholder_in_text(part, mm_parser.model_config) |
| 1922 | if wrap_dicts: |
| 1923 | return {"type": "text", "text": part} |
| 1924 | return part |
| 1925 | # Handle structured dictionary parts |
| 1926 | part_type, content = _parse_chat_message_content_mm_part(part) |
| 1927 | # if part_type is text/refusal/image_url/audio_url/video_url/input_audio but |
| 1928 | # content is None, log a warning and skip |
| 1929 | if part_type in PART_TYPES_TO_SKIP_NONE_CONTENT and content is None: |
| 1930 | logger.warning( |
| 1931 | "Skipping multimodal part '%s' (type: '%s') " |
| 1932 | "with empty / unparsable content.", |
| 1933 | part, |
| 1934 | part_type, |
| 1935 | ) |
| 1936 | return None |
| 1937 | |
| 1938 | if part_type in TEXT_PART_TYPES: |
| 1939 | str_content = cast(str, content) |
| 1940 | _reject_reserved_placeholder_in_text(str_content, mm_parser.model_config) |
| 1941 | if wrap_dicts: |
| 1942 | result: dict[str, Any] = {"type": "text", "text": str_content} |
| 1943 | result.update(_collect_extra_fields(cast(dict[str, Any], part))) |
| 1944 | return result |
| 1945 | else: |
| 1946 | return str_content |
| 1947 | |
| 1948 | # For media items, if a user has provided one, use it. Otherwise, insert |
| 1949 | # a placeholder empty uuid. |
| 1950 | uuid = part.get("uuid", None) |
| 1951 | if uuid is not None: |
| 1952 | uuid = str(uuid) |
| 1953 | |
| 1954 | modality = None |
| 1955 | if part_type == "image_pil": |
| 1956 | image_content = cast(Image.Image, content) if content is not None else None |
| 1957 | mm_parser.parse_image_pil(image_content, uuid) |
| 1958 | modality = "image" |
| 1959 | elif part_type in ("image_url", "input_image"): |
| 1960 | str_content = cast(str, content) |
| 1961 | mm_parser.parse_image(str_content, uuid) |
| 1962 | modality = "image" |
| 1963 | elif part_type == "image_embeds": |
| 1964 | content = ( |
| 1965 | cast(MultiModalEmbedsPayload, content) if content is not None else None |
| 1966 | ) |
| 1967 | mm_parser.parse_image_embeds(content, uuid) |
| 1968 | modality = "image" |
| 1969 | elif part_type == "audio_embeds": |
| 1970 | content = ( |
| 1971 | cast(MultiModalEmbedsPayload, content) if content is not None else None |
| 1972 | ) |
| 1973 | mm_parser.parse_audio_embeds(content, uuid) |
| 1974 | modality = "audio" |
| 1975 | elif part_type == "video_embeds": |
| 1976 | content = ( |
| 1977 | cast(MultiModalEmbedsPayload, content) if content is not None else None |
| 1978 | ) |
| 1979 | mm_parser.parse_video_embeds(content, uuid) |
| 1980 | modality = "video" |
| 1981 | elif part_type == "prompt_embeds": |
| 1982 | if not content: |
| 1983 | raise VLLMValidationError( |
| 1984 | _PROMPT_EMBEDS_MISSING_DATA_ERROR, parameter="prompt_embeds" |
| 1985 | ) |
| 1986 | mm_parser.parse_prompt_embeds(cast(str, content)) |
| 1987 | modality = "prompt_embeds" |
| 1988 | elif part_type == "audio_url": |
| 1989 | str_content = cast(str, content) |
| 1990 | mm_parser.parse_audio(str_content, uuid) |
| 1991 | modality = "audio" |
| 1992 | elif part_type == "input_audio": |
| 1993 | dict_content = cast(InputAudio, content) |
| 1994 | mm_parser.parse_input_audio(dict_content, uuid) |
| 1995 | modality = "audio" |
| 1996 | elif part_type == "video_url": |
| 1997 | str_content = cast(str, content) |
| 1998 | mm_parser.parse_video(str_content, uuid) |
| 1999 | modality = "video" |
| 2000 | elif part_type == "tool_reference": |
| 2001 | # Tool references are not multimodal data — they reference deferred |
| 2002 | # tools and are passed through as-is for the chat template to expand. |
| 2003 | if wrap_dicts: |
| 2004 | return {"type": "tool_reference", "name": cast(str, content)} |
| 2005 | return cast(str, content) |
| 2006 | else: |
| 2007 | supported = sorted(MM_PARSER_MAP.keys() | set(PART_TYPES_TO_SKIP_NONE_CONTENT)) |
| 2008 | raise VLLMValidationError( |
| 2009 | f"Unsupported chat content part type: {part_type!r}. " |
| 2010 | f"Supported types: {', '.join(supported)}.", |
| 2011 | parameter="type", |
| 2012 | value=part_type, |
| 2013 | ) |
| 2014 | |
| 2015 | if wrap_dicts: |
| 2016 | if modality == "prompt_embeds": |
| 2017 | # Chat templates don't know about the "prompt_embeds" modality, |
| 2018 | # emit the single sentinel token as text so the template renders |
| 2019 | # it inline. The renderer later expands it to N tokens post-tokenize. |
| 2020 | return {"type": "text", "text": PROMPT_EMBEDS_PLACEHOLDER_TOKEN} |
| 2021 | result = {"type": modality} |
| 2022 | result.update(_collect_extra_fields(cast(dict[str, Any], part))) |
| 2023 | return result |
| 2024 | if modality == "prompt_embeds": |
| 2025 | # Emit the renderer token inline regardless of `interleave_strings`, |
| 2026 | # prompt_embeds are spliced at the token offset so position matters. |
| 2027 | # Falling back to front-padding via `missing_placeholders` would |
| 2028 | # reorder them relative to surrounding text. |
| 2029 | return PROMPT_EMBEDS_PLACEHOLDER_TOKEN |
| 2030 | return MODALITY_PLACEHOLDERS_MAP[modality] if interleave_strings else None |
| 2031 | |
| 2032 | |
| 2033 | # No need to validate using Pydantic again |
| 2034 | _AssistantParser = partial(cast, ChatCompletionAssistantMessageParam) |
| 2035 | _ToolParser = partial(cast, ChatCompletionToolMessageParam) |
| 2036 | |
| 2037 | |
| 2038 | _CLAUDE_CODE_BILLING_HEADER = "x-anthropic-billing-header" |
| 2039 | |
| 2040 | |
| 2041 | def _strip_claude_code_billing_header( |
| 2042 | parts: Iterable[ChatCompletionContentPartParam], |
| 2043 | ) -> list[ChatCompletionContentPartParam]: |
| 2044 | """Drop Claude Code's attribution header line from system text parts. |
| 2045 | |
| 2046 | Its ``cch`` value changes on every request, so a prompt that keeps it |
| 2047 | misses the prefix cache from the header on. Gateways that translate |
| 2048 | Anthropic requests to chat completions keep it; ``/v1/messages`` drops |
| 2049 | the same block. |
| 2050 | """ |
| 2051 | out: list[ChatCompletionContentPartParam] = [] |
| 2052 | for part in parts: |
| 2053 | if isinstance(part, str): |
| 2054 | text: object = part |
| 2055 | elif isinstance(part, dict) and part.get("type") == "text": |
| 2056 | text = part.get("text") |
| 2057 | else: |
| 2058 | text = None |
| 2059 | if not (isinstance(text, str) and text.startswith(_CLAUDE_CODE_BILLING_HEADER)): |
| 2060 | out.append(part) |
| 2061 | continue |
| 2062 | rest = text.partition("\n")[2] |
| 2063 | if not rest: |
| 2064 | continue |
| 2065 | out.append( |
| 2066 | cast(ChatCompletionContentPartParam, rest) |
| 2067 | if isinstance(part, str) |
| 2068 | else cast(ChatCompletionContentPartParam, {**part, "text": rest}) |
| 2069 | ) |
| 2070 | return out |
| 2071 | |
| 2072 | |
| 2073 | def _parse_chat_message_content( |
| 2074 | message: ChatCompletionMessageParam, |
| 2075 | mm_tracker: BaseMultiModalItemTracker, |
| 2076 | content_format: ChatTemplateContentFormat, |
| 2077 | interleave_strings: bool, |
| 2078 | mm_processor_kwargs: dict[str, Any] | None = None, |
| 2079 | ) -> list[ConversationMessage]: |
| 2080 | role = message["role"] |
| 2081 | content = message.get("content") |
| 2082 | reasoning = message.get("reasoning") |
| 2083 | |
| 2084 | if content is None: |
| 2085 | content = [] |
| 2086 | elif isinstance(content, str): |
| 2087 | content = [ChatCompletionContentPartTextParam(type="text", text=content)] |
| 2088 | if role == "system": |
| 2089 | content = _strip_claude_code_billing_header(content) # type: ignore[arg-type] |
| 2090 | result = _parse_chat_message_content_parts( |
| 2091 | role, |
| 2092 | content, # type: ignore |
| 2093 | mm_tracker, |
| 2094 | wrap_dicts=(content_format == "openai"), |
| 2095 | interleave_strings=interleave_strings, |
| 2096 | mm_processor_kwargs=mm_processor_kwargs, |
| 2097 | ) |
| 2098 | |
| 2099 | for result_msg in result: |
| 2100 | if role == "assistant": |
| 2101 | parsed_msg = _AssistantParser(message) |
| 2102 | |
| 2103 | # The 'tool_calls' is not None check ensures compatibility. |
| 2104 | # It's needed only if downstream code doesn't strictly |
| 2105 | # follow the OpenAI spec. |
| 2106 | if "tool_calls" in parsed_msg and parsed_msg["tool_calls"] is not None: |
| 2107 | result_msg["tool_calls"] = list(parsed_msg["tool_calls"]) |
| 2108 | # Include reasoning if present for interleaved thinking. |
| 2109 | if reasoning is not None: |
| 2110 | result_msg["reasoning"] = cast(str, reasoning) |
| 2111 | result_msg["reasoning_content"] = cast( |
| 2112 | str, reasoning |
| 2113 | ) # keep compatibility |
| 2114 | elif role == "tool": |
| 2115 | parsed_msg = _ToolParser(message) |
| 2116 | if "tool_call_id" in parsed_msg: |
| 2117 | result_msg["tool_call_id"] = parsed_msg["tool_call_id"] |
| 2118 | # Normalize tool message content from OpenAI array format to plain |
| 2119 | # string. Clients like Claude Code / Cursor send tool results as |
| 2120 | # [{"type": "text", "text": "..."}], but most chat templates only |
| 2121 | # handle string content for tool messages. |
| 2122 | # However, tool_reference items must be preserved as structured |
| 2123 | # dicts for the chat template to expand them. |
| 2124 | msg_content = result_msg.get("content") |
| 2125 | if isinstance(msg_content, list): |
| 2126 | has_non_text = any( |
| 2127 | isinstance(item, dict) and item.get("type") != "text" |
| 2128 | for item in msg_content |
| 2129 | ) |
| 2130 | if has_non_text: |
| 2131 | # Keep structured content (e.g., tool_reference) |
| 2132 | result_msg["content"] = msg_content |
| 2133 | else: |
| 2134 | texts = [ |
| 2135 | item.get("text", "") |
| 2136 | for item in msg_content |
| 2137 | if isinstance(item, dict) and item.get("type") == "text" |
| 2138 | ] |
| 2139 | result_msg["content"] = "\n".join(texts) if texts else "" |
| 2140 | |
| 2141 | if "name" in message and isinstance(message["name"], str): |
| 2142 | result_msg["name"] = message["name"] |
| 2143 | |
| 2144 | if "task" in message and isinstance(message["task"], str): |
| 2145 | result_msg["task"] = message["task"] |
| 2146 | |
| 2147 | if role == "developer": |
| 2148 | result_msg["tools"] = message.get("tools", None) |
| 2149 | return result |
| 2150 | |
| 2151 | |
| 2152 | def _postprocess_messages(messages: list[ConversationMessage]) -> None: |
| 2153 | # per the Transformers docs & maintainers, tool call arguments in |
| 2154 | # assistant-role messages with tool_calls need to be dicts not JSON str - |
| 2155 | # this is how tool-use chat templates will expect them moving forwards |
| 2156 | # so, for messages that have tool_calls, parse the string (which we get |
| 2157 | # from openAI format) to dict |
| 2158 | for message in messages: |
| 2159 | if message["role"] == "assistant" and "tool_calls" in message: |
| 2160 | tool_calls = message.get("tool_calls") |
| 2161 | if not isinstance(tool_calls, list): |
| 2162 | continue |
| 2163 | |
| 2164 | if len(tool_calls) == 0: |
| 2165 | # Drop empty tool_calls to keep templates on the normal assistant path. |
| 2166 | message.pop("tool_calls", None) |
| 2167 | continue |
| 2168 | |
| 2169 | for item in tool_calls: |
| 2170 | if not isinstance(item, dict): |
| 2171 | raise VLLMValidationError( |
| 2172 | "assistant tool_calls entries must be objects.", |
| 2173 | parameter="tool_calls", |
| 2174 | ) |
| 2175 | |
| 2176 | function = item.get("function") |
| 2177 | if item.get("type", "function") != "function" or not isinstance( |
| 2178 | function, dict |
| 2179 | ): |
| 2180 | raise VLLMValidationError( |
| 2181 | "chat completions only support assistant tool_calls " |
| 2182 | "of type 'function'.", |
| 2183 | parameter="tool_calls", |
| 2184 | ) |
| 2185 | |
| 2186 | # if arguments is None or empty string, set to {} |
| 2187 | if content := function.get("arguments"): |
| 2188 | if isinstance(content, dict): |
| 2189 | parsed = content |
| 2190 | else: |
| 2191 | if isinstance(content, str): |
| 2192 | try: |
| 2193 | parsed = json.loads(content) |
| 2194 | except json.JSONDecodeError: |
| 2195 | # A malformed `arguments` string lives in |
| 2196 | # conversation history, so failing the request |
| 2197 | # here would fail every subsequent turn too and |
| 2198 | # leave the conversation unrecoverable. Coerce |
| 2199 | # to an empty object so the turn can proceed. |
| 2200 | logger.warning( |
| 2201 | "Tool call %r has arguments that are not valid " |
| 2202 | "JSON (%d chars); coercing to an empty object " |
| 2203 | "so the conversation can continue.", |
| 2204 | function.get("name"), |
| 2205 | len(content), |
| 2206 | ) |
| 2207 | parsed = None |
| 2208 | else: |
| 2209 | parsed = content |
| 2210 | |
| 2211 | if not isinstance(parsed, dict): |
| 2212 | if parsed is not None: |
| 2213 | # Valid JSON, but not an object (e.g. "[]", |
| 2214 | # "42", "true"). |
| 2215 | # Chat templates require a mapping. |
| 2216 | logger.warning( |
| 2217 | "Tool call %r arguments decoded to %s, not a " |
| 2218 | "JSON object; coercing to an empty object.", |
| 2219 | function.get("name"), |
| 2220 | type(parsed).__name__, |
| 2221 | ) |
| 2222 | parsed = {} |
| 2223 | |
| 2224 | function["arguments"] = parsed |
| 2225 | else: |
| 2226 | function["arguments"] = {} |
| 2227 | |
| 2228 | |
| 2229 | def parse_chat_messages( |
| 2230 | messages: list[ChatCompletionMessageParam], |
| 2231 | model_config: ModelConfig, |
| 2232 | content_format: ChatTemplateContentFormat, |
| 2233 | media_io_kwargs: dict[str, dict[str, Any]] | None = None, |
| 2234 | mm_processor_kwargs: dict[str, Any] | None = None, |
| 2235 | ) -> tuple[ |
| 2236 | list[ConversationMessage], |
| 2237 | MultiModalDataDict | None, |
| 2238 | MultiModalUUIDDict | None, |
| 2239 | ]: |
| 2240 | conversation: list[ConversationMessage] = [] |
| 2241 | mm_tracker = MultiModalItemTracker( |
| 2242 | model_config, |
| 2243 | media_io_kwargs=media_io_kwargs, |
| 2244 | ) |
| 2245 | |
| 2246 | for msg in messages: |
| 2247 | sub_messages = _parse_chat_message_content( |
| 2248 | msg, |
| 2249 | mm_tracker, |
| 2250 | content_format, |
| 2251 | interleave_strings=( |
| 2252 | content_format == "string" |
| 2253 | and model_config.multimodal_config is not None |
| 2254 | and model_config.multimodal_config.interleave_mm_strings |
| 2255 | ), |
| 2256 | mm_processor_kwargs=mm_processor_kwargs, |
| 2257 | ) |
| 2258 | |
| 2259 | conversation.extend(sub_messages) |
| 2260 | |
| 2261 | _postprocess_messages(conversation) |
| 2262 | |
| 2263 | mm_data, mm_uuids = mm_tracker.resolve_items() |
| 2264 | |
| 2265 | return conversation, mm_data, mm_uuids |
| 2266 | |
| 2267 | |
| 2268 | async def parse_chat_messages_async( |
| 2269 | messages: list[ChatCompletionMessageParam], |
| 2270 | model_config: ModelConfig, |
| 2271 | content_format: ChatTemplateContentFormat, |
| 2272 | media_io_kwargs: dict[str, dict[str, Any]] | None = None, |
| 2273 | mm_processor_kwargs: dict[str, Any] | None = None, |
| 2274 | ) -> tuple[ |
| 2275 | list[ConversationMessage], |
| 2276 | MultiModalDataDict | None, |
| 2277 | MultiModalUUIDDict | None, |
| 2278 | ]: |
| 2279 | conversation: list[ConversationMessage] = [] |
| 2280 | mm_tracker = AsyncMultiModalItemTracker( |
| 2281 | model_config, |
| 2282 | media_io_kwargs=media_io_kwargs, |
| 2283 | ) |
| 2284 | |
| 2285 | for msg in messages: |
| 2286 | sub_messages = _parse_chat_message_content( |
| 2287 | msg, |
| 2288 | mm_tracker, |
| 2289 | content_format, |
| 2290 | interleave_strings=( |
| 2291 | content_format == "string" |
| 2292 | and model_config.multimodal_config is not None |
| 2293 | and model_config.multimodal_config.interleave_mm_strings |
| 2294 | ), |
| 2295 | mm_processor_kwargs=mm_processor_kwargs, |
| 2296 | ) |
| 2297 | |
| 2298 | conversation.extend(sub_messages) |
| 2299 | |
| 2300 | _postprocess_messages(conversation) |
| 2301 | |
| 2302 | mm_data, mm_uuids = await mm_tracker.resolve_items() |
| 2303 | |
| 2304 | return conversation, mm_data, mm_uuids |
| 2305 | |
| 2306 | |
| 2307 | _KIMI_MODEL_TYPES = ("kimi_k2", "kimi_k25", "kimi_k3") |
| 2308 | |
| 2309 | |
| 2310 | def get_tool_call_id_type(model_config: ModelConfig) -> str: |
| 2311 | """Return the tool-call ID type for a given model configuration.""" |
| 2312 | hf_overrides = getattr(model_config, "hf_overrides", None) |
| 2313 | hf_config = getattr(model_config, "hf_config", None) |
| 2314 | hf_text_config = getattr(model_config, "hf_text_config", None) |
| 2315 | model_types = ( |
| 2316 | getattr(hf_config, "model_type", None), |
| 2317 | getattr(hf_text_config, "model_type", None), |
| 2318 | ) |
| 2319 | if any(model_type in _KIMI_MODEL_TYPES for model_type in model_types) or ( |
| 2320 | isinstance(hf_overrides, dict) |
| 2321 | and hf_overrides.get("model_type") in _KIMI_MODEL_TYPES |
| 2322 | ): |
| 2323 | return "kimi_k2" |
| 2324 | return "random" |
| 2325 | |
| 2326 | |
| 2327 | def make_tool_call_id(id_type: str = "random", func_name=None, idx=None): |
| 2328 | if id_type == "kimi_k2": |
| 2329 | return f"functions.{func_name}:{idx}" |
| 2330 | else: |
| 2331 | # by default return random |
| 2332 | return f"chatcmpl-tool-{random_uuid()}" |