| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from collections.abc import Mapping |
| 5 | from pathlib import Path |
| 6 | from typing import Any, Literal, TypeAlias, TypeVar, final, overload |
| 7 | |
| 8 | import torch |
| 9 | from pydantic import ( |
| 10 | ConfigDict, |
| 11 | Field, |
| 12 | GetCoreSchemaHandler, |
| 13 | field_validator, |
| 14 | model_validator, |
| 15 | ) |
| 16 | from pydantic.dataclasses import dataclass |
| 17 | from pydantic_core import core_schema |
| 18 | |
| 19 | import vllm.envs as envs |
| 20 | from vllm.config.ec_transfer import ECTransferConfig |
| 21 | from vllm.config.utils import config |
| 22 | from vllm.logger import init_logger |
| 23 | from vllm.utils.hashing import safe_hash |
| 24 | from vllm.v1.attention.backends.registry import AttentionBackendEnum |
| 25 | |
| 26 | logger = init_logger(__name__) |
| 27 | |
| 28 | _T = TypeVar("_T") |
| 29 | |
| 30 | # HuggingFace processors accept a shared flat namespace in addition to the |
| 31 | # nested processor kwarg scopes ``text_kwargs`` / ``images_kwargs`` / |
| 32 | # ``videos_kwargs`` / ``audio_kwargs``. vLLM recognizes these scopes when |
| 33 | # resolving ``mm_processor_kwargs``. |
| 34 | _HF_PROCESSOR_KWARG_SCOPES = ( |
| 35 | "text_kwargs", |
| 36 | "images_kwargs", |
| 37 | "videos_kwargs", |
| 38 | "audio_kwargs", |
| 39 | ) |
| 40 | |
| 41 | |
| 42 | def _recursively_merge_mm_processor_kwargs( |
| 43 | defaults: Mapping[str, object], |
| 44 | overrides: Mapping[str, object], |
| 45 | ) -> dict[str, object]: |
| 46 | """Merge two mappings recursively, with ``overrides`` taking precedence. |
| 47 | |
| 48 | Nested values are merged only when both values are mappings; otherwise the |
| 49 | override replaces the existing value. |
| 50 | """ |
| 51 | merged = dict(defaults) |
| 52 | |
| 53 | for key, value in overrides.items(): |
| 54 | default_value = merged.get(key) |
| 55 | if isinstance(default_value, Mapping) and isinstance(value, Mapping): |
| 56 | merged[key] = _recursively_merge_mm_processor_kwargs(default_value, value) |
| 57 | else: |
| 58 | merged[key] = value |
| 59 | |
| 60 | return merged |
| 61 | |
| 62 | |
| 63 | def _merge_mm_processor_scope( |
| 64 | config_flat: Mapping[str, object], |
| 65 | config_scoped: Mapping[str, object], |
| 66 | inference_flat: Mapping[str, object], |
| 67 | inference_scoped: Mapping[str, object], |
| 68 | ) -> dict[str, object]: |
| 69 | """Merge kwargs for one existing HuggingFace processor kwarg scope. |
| 70 | |
| 71 | Flat kwargs contribute only to keys present in either the configured or |
| 72 | inference-time scope. For those keys, precedence is: |
| 73 | configured flat < configured scoped < inference flat < inference scoped. |
| 74 | |
| 75 | Nested mappings are merged recursively; otherwise the higher-priority value |
| 76 | replaces the lower-priority value. |
| 77 | """ |
| 78 | # Flat kwargs contribute only to keys present in this scope in either input. |
| 79 | scoped_keys = config_scoped.keys() | inference_scoped.keys() |
| 80 | config_flat_for_scope = { |
| 81 | key: value for key, value in config_flat.items() if key in scoped_keys |
| 82 | } |
| 83 | inference_flat_for_scope = { |
| 84 | key: value for key, value in inference_flat.items() if key in scoped_keys |
| 85 | } |
| 86 | |
| 87 | # Merge from lowest to highest priority: |
| 88 | # configured flat -> configured scoped -> inference flat -> inference scoped. |
| 89 | merged = _recursively_merge_mm_processor_kwargs( |
| 90 | config_flat_for_scope, config_scoped |
| 91 | ) |
| 92 | merged = _recursively_merge_mm_processor_kwargs(merged, inference_flat_for_scope) |
| 93 | merged = _recursively_merge_mm_processor_kwargs(merged, inference_scoped) |
| 94 | return merged |
| 95 | |
| 96 | |
| 97 | def _merge_mm_processor_kwargs( |
| 98 | config_kwargs: Mapping[str, object], |
| 99 | inference_kwargs: Mapping[str, object], |
| 100 | ) -> dict[str, object]: |
| 101 | """Merge configured and inference-time multi-modal processor kwargs. |
| 102 | |
| 103 | The merge is performed in two parts: |
| 104 | |
| 105 | 1. The shared flat kwargs are merged recursively, with inference-time values |
| 106 | taking precedence. |
| 107 | 2. Each existing HuggingFace processor kwarg scope is merged separately. |
| 108 | For keys present in either the configured or inference-time scope, |
| 109 | precedence is: |
| 110 | configured flat < configured scoped < inference flat < inference scoped. |
| 111 | |
| 112 | For example: |
| 113 | |
| 114 | configured = { |
| 115 | "size": {"shortest_edge": 256}, |
| 116 | "videos_kwargs": { |
| 117 | "size": {"longest_edge": 1024}, |
| 118 | "num_frames": 16, |
| 119 | }, |
| 120 | } |
| 121 | inference = { |
| 122 | "size": {"shortest_edge": 512}, |
| 123 | "videos_kwargs": {"fps": 4}, |
| 124 | } |
| 125 | |
| 126 | produces: |
| 127 | |
| 128 | { |
| 129 | "size": {"shortest_edge": 512}, |
| 130 | "videos_kwargs": { |
| 131 | "size": { |
| 132 | "shortest_edge": 512, |
| 133 | "longest_edge": 1024, |
| 134 | }, |
| 135 | "num_frames": 16, |
| 136 | "fps": 4, |
| 137 | }, |
| 138 | } |
| 139 | |
| 140 | Scoped values do not modify the shared flat kwargs or sibling scopes. |
| 141 | Missing scopes, None-valued scopes, and empty scope mappings are treated as |
| 142 | absent and do not create a scope. |
| 143 | """ |
| 144 | # Merge the shared flat kwargs first; nested scopes are handled separately |
| 145 | # below. |
| 146 | config_flat = { |
| 147 | key: value |
| 148 | for key, value in config_kwargs.items() |
| 149 | if key not in _HF_PROCESSOR_KWARG_SCOPES |
| 150 | } |
| 151 | inference_flat = { |
| 152 | key: value |
| 153 | for key, value in inference_kwargs.items() |
| 154 | if key not in _HF_PROCESSOR_KWARG_SCOPES |
| 155 | } |
| 156 | merged = _recursively_merge_mm_processor_kwargs(config_flat, inference_flat) |
| 157 | |
| 158 | # Merge each nested processor kwarg scope present in either input. |
| 159 | for scoped_key in _HF_PROCESSOR_KWARG_SCOPES: |
| 160 | config_scope_value = config_kwargs.get(scoped_key) |
| 161 | inference_scope_value = inference_kwargs.get(scoped_key) |
| 162 | |
| 163 | # Treat missing scopes, None-valued scopes, and empty scope mappings as |
| 164 | # absent. |
| 165 | if isinstance(config_scope_value, Mapping) and not config_scope_value: |
| 166 | config_scope_value = None |
| 167 | if isinstance(inference_scope_value, Mapping) and not inference_scope_value: |
| 168 | inference_scope_value = None |
| 169 | |
| 170 | # Skip adding the scope to the merged result when it is absent from both |
| 171 | # sources. |
| 172 | if config_scope_value is None and inference_scope_value is None: |
| 173 | continue |
| 174 | |
| 175 | # Handle the inference scope first because it has higher precedence; only |
| 176 | # mapping-valued scopes participate in the recursive scoped merge. |
| 177 | inference_scope_kwargs: Mapping[str, object] |
| 178 | if inference_scope_value is None: |
| 179 | inference_scope_kwargs = {} |
| 180 | elif not isinstance(inference_scope_value, Mapping): |
| 181 | # Non-mapping scopes cannot participate in the recursive scoped merge; |
| 182 | # the higher-priority inference value replaces the configured scope. |
| 183 | merged[scoped_key] = inference_scope_value |
| 184 | continue |
| 185 | else: |
| 186 | inference_scope_kwargs = inference_scope_value |
| 187 | |
| 188 | # Handle the configured scope next because it has lower precedence; only |
| 189 | # mapping-valued scopes participate in the recursive scoped merge. |
| 190 | if isinstance(config_scope_value, Mapping): |
| 191 | config_scope_kwargs = config_scope_value |
| 192 | elif inference_scope_value is None: |
| 193 | # Non-mapping scopes cannot participate in the recursive scoped merge; |
| 194 | # preserve the configured value when inference contributes nothing. |
| 195 | merged[scoped_key] = config_scope_value |
| 196 | continue |
| 197 | else: |
| 198 | config_scope_kwargs = {} |
| 199 | |
| 200 | # Merge this scope together with the matching flat kwargs. |
| 201 | merged[scoped_key] = _merge_mm_processor_scope( |
| 202 | config_flat, |
| 203 | config_scope_kwargs, |
| 204 | inference_flat, |
| 205 | inference_scope_kwargs, |
| 206 | ) |
| 207 | |
| 208 | return merged |
| 209 | |
| 210 | |
| 211 | @dataclass |
| 212 | class BaseDummyOptions: |
| 213 | """Base options for generating dummy data during profiling.""" |
| 214 | |
| 215 | count: int = Field(999, ge=0) |
| 216 | |
| 217 | |
| 218 | @dataclass(config=ConfigDict(extra="forbid")) |
| 219 | class VideoDummyOptions(BaseDummyOptions): |
| 220 | """Options for generating dummy video data during profiling.""" |
| 221 | |
| 222 | num_frames: int | None = Field(None, gt=0) |
| 223 | width: int | None = Field(None, gt=0) |
| 224 | height: int | None = Field(None, gt=0) |
| 225 | |
| 226 | |
| 227 | @dataclass(config=ConfigDict(extra="forbid")) |
| 228 | class ImageDummyOptions(BaseDummyOptions): |
| 229 | """Options for generating dummy image data during profiling.""" |
| 230 | |
| 231 | width: int | None = Field(None, gt=0) |
| 232 | height: int | None = Field(None, gt=0) |
| 233 | |
| 234 | |
| 235 | @dataclass(config=ConfigDict(extra="forbid")) |
| 236 | class AudioDummyOptions(BaseDummyOptions): |
| 237 | """Options for generating dummy audio data during profiling.""" |
| 238 | |
| 239 | length: int | None = Field(None, gt=0) |
| 240 | |
| 241 | |
| 242 | @final |
| 243 | class MultiModalDummyOptions(dict[str, BaseDummyOptions]): |
| 244 | """Dummy data options for each modality. |
| 245 | |
| 246 | Lookups of the modalities predefined by vLLM return their own options |
| 247 | class, while any other modality returns |
| 248 | [`BaseDummyOptions`][vllm.config.multimodal.BaseDummyOptions]. |
| 249 | """ |
| 250 | |
| 251 | @overload # type: ignore[override] |
| 252 | def get(self, key: Literal["image"], /) -> ImageDummyOptions | None: ... |
| 253 | |
| 254 | @overload |
| 255 | def get(self, key: Literal["image"], default: _T, /) -> ImageDummyOptions | _T: ... |
| 256 | |
| 257 | @overload |
| 258 | def get(self, key: Literal["video"], /) -> VideoDummyOptions | None: ... |
| 259 | |
| 260 | @overload |
| 261 | def get(self, key: Literal["video"], default: _T, /) -> VideoDummyOptions | _T: ... |
| 262 | |
| 263 | @overload |
| 264 | def get(self, key: Literal["audio"], /) -> AudioDummyOptions | None: ... |
| 265 | |
| 266 | @overload |
| 267 | def get(self, key: Literal["audio"], default: _T, /) -> AudioDummyOptions | _T: ... |
| 268 | |
| 269 | @overload |
| 270 | def get(self, key: str, /) -> BaseDummyOptions | None: ... |
| 271 | |
| 272 | @overload |
| 273 | def get(self, key: str, default: _T, /) -> BaseDummyOptions | _T: ... |
| 274 | |
| 275 | def get(self, key: str, default: object = None, /) -> object: |
| 276 | return super().get(key, default) |
| 277 | |
| 278 | @classmethod |
| 279 | def __get_pydantic_core_schema__( |
| 280 | cls, |
| 281 | source_type: Any, |
| 282 | handler: GetCoreSchemaHandler, |
| 283 | ) -> core_schema.CoreSchema: |
| 284 | return core_schema.no_info_after_validator_function( |
| 285 | cls, handler.generate_schema(dict[str, BaseDummyOptions]) |
| 286 | ) |
| 287 | |
| 288 | |
| 289 | MMEncoderTPMode = Literal["weights", "data"] |
| 290 | MMCacheType = Literal["shm", "lru"] |
| 291 | VideoPruningMethod = Literal["evs", "vidcom2"] |
| 292 | MMTensorIPC = Literal["direct_rpc", "torch_shm"] |
| 293 | MMHasherAlgorithm = Literal["blake3", "sha256", "sha512"] |
| 294 | MMProcessorDevice: TypeAlias = str |
| 295 | """`"auto"`, `"cpu"`, or the platform's own accelerator name |
| 296 | (`current_platform.device_type`, e.g. `"cuda"` on CUDA and ROCm, |
| 297 | `"xpu"` on XPU). Validated against that set by the CLI.""" |
| 298 | |
| 299 | |
| 300 | @config |
| 301 | class MultiModalConfig: |
| 302 | """Controls the behavior of multimodal models.""" |
| 303 | |
| 304 | language_model_only: bool = False |
| 305 | """If True, disables all multimodal inputs by setting all modality limits to 0. |
| 306 | Equivalent to setting `--limit-mm-per-prompt` to 0 for every modality.""" |
| 307 | limit_per_prompt: MultiModalDummyOptions = Field( |
| 308 | default_factory=MultiModalDummyOptions |
| 309 | ) |
| 310 | """The maximum number of input items and options allowed per |
| 311 | prompt for each modality. |
| 312 | |
| 313 | Defaults to 999 for each modality. |
| 314 | |
| 315 | Legacy format (count only): |
| 316 | {"image": 16, "video": 2} |
| 317 | |
| 318 | Configurable format (with options): |
| 319 | {"video": {"count": 1, "num_frames": 32, "width": 512, "height": 512}, |
| 320 | "image": {"count": 5, "width": 512, "height": 512}} |
| 321 | |
| 322 | Mixed format (combining both): |
| 323 | {"image": 16, "video": {"count": 1, "num_frames": 32, "width": 512, |
| 324 | "height": 512}} |
| 325 | """ |
| 326 | enable_mm_embeds: bool = False |
| 327 | """If `True`, enables passing multimodal embeddings: |
| 328 | for `LLM` class, this refers to tensor inputs under `multi_modal_data`; |
| 329 | for the OpenAI-compatible server, this refers to chat messages with content |
| 330 | `"type": "*_embeds"`. |
| 331 | |
| 332 | When enabled with `--limit-mm-per-prompt` set to 0 for a modality, |
| 333 | precomputed embeddings skip count validation for that modality, |
| 334 | saving memory by not loading encoder modules while still enabling |
| 335 | embeddings as an input. Limits greater than 0 still apply to embeddings. |
| 336 | |
| 337 | WARNING: The vLLM engine may crash if incorrect shape of embeddings is passed. |
| 338 | Only enable this flag for trusted users!""" |
| 339 | media_io_kwargs: dict[str, dict[str, Any]] = Field(default_factory=dict) |
| 340 | """Additional args passed to process media inputs, keyed by modalities. |
| 341 | For example, to set num_frames for video, set |
| 342 | `--media-io-kwargs '{"video": {"num_frames": 40} }'`""" |
| 343 | mm_processor_kwargs: dict[str, object] | None = None |
| 344 | """Arguments to be forwarded to the model's processor for multi-modal data, |
| 345 | e.g., image processor. Overrides for the multi-modal processor obtained |
| 346 | from `transformers.AutoProcessor.from_pretrained`. |
| 347 | |
| 348 | The available overrides depend on the model that is being run. |
| 349 | |
| 350 | For example, for Phi-3-Vision: |
| 351 | `{"num_crops": 4}`.""" |
| 352 | mm_device_do_normalize: bool | None = True |
| 353 | """ |
| 354 | Move the do_normalize computation in the mm preprocessing to before the ViT, |
| 355 | and let the device do it, so that CPU computation can be saved. |
| 356 | """ |
| 357 | mm_processor_cache_gb: float = Field(default=4, ge=0) |
| 358 | """The size (in GiB) of the multi-modal processor cache, which is used to |
| 359 | avoid re-processing past multi-modal inputs. |
| 360 | |
| 361 | This cache is duplicated for each API process and engine core process, |
| 362 | resulting in a total memory usage of |
| 363 | `mm_processor_cache_gb * (api_server_count + data_parallel_size)`. |
| 364 | |
| 365 | A single processed item larger than this budget is served uncached |
| 366 | (with a warning) instead of failing. Raise this value to cache such items. |
| 367 | |
| 368 | Set to `0` to disable this cache completely (not recommended).""" |
| 369 | mm_processor_cache_type: MMCacheType = "lru" |
| 370 | """Type of cache to use for the multi-modal preprocessor/mapper. If `shm`, |
| 371 | use shared memory FIFO cache. If `lru`, use mirrored LRU cache.""" |
| 372 | mm_hasher_algorithm: MMHasherAlgorithm = "blake3" |
| 373 | """Hash algorithm to use for multi-modal input caching. Use `"sha256"` or |
| 374 | `"sha512"` for FIPS-compliant deployments.""" |
| 375 | mm_shm_cache_max_object_size_mb: int = Field(default=128, ge=0) |
| 376 | """Size limit (in MiB) for each object stored in the multi-modal processor |
| 377 | shared memory cache. Only effective when `mm_processor_cache_type` is |
| 378 | `"shm"`.""" |
| 379 | mm_encoder_only: bool = False |
| 380 | """ |
| 381 | When enabled, skips the language component of the model. |
| 382 | |
| 383 | This is usually only valid in disaggregated Encoder process. |
| 384 | """ |
| 385 | mm_encoder_tp_mode: MMEncoderTPMode = "weights" |
| 386 | """Indicates how to optimize multi-modal encoder inference using tensor |
| 387 | parallelism (TP). |
| 388 | |
| 389 | - `"weights"`: Within the same vLLM engine, split the weights of |
| 390 | each layer across TP ranks. (default TP behavior) |
| 391 | - `"data"`: Within the same vLLM engine, split the batched input data |
| 392 | across TP ranks to process the data in parallel, while hosting |
| 393 | the full weights on each TP rank. |
| 394 | This batch-level DP is not to be confused with API request-level |
| 395 | DP (which is controlled by `--data-parallel-size`). |
| 396 | This is only supported on a per-model basis and falls back to |
| 397 | `"weights"` if the encoder does not support DP.""" |
| 398 | mm_encoder_attn_backend: AttentionBackendEnum | None = None |
| 399 | """Optional override for the multi-modal encoder attention backend when |
| 400 | using vision transformers. Accepts any value from |
| 401 | `vllm.v1.attention.backends.registry.AttentionBackendEnum` (e.g. `FLASH_ATTN`).""" |
| 402 | mm_encoder_attn_dtype: Literal["fp8"] | None = None |
| 403 | """Optional dtype override for ViT encoder attention. Set to `"fp8"` to |
| 404 | enable FP8 quantization via the FlashInfer cuDNN backend. When set to |
| 405 | `"fp8"` without a scale file, dynamic scaling is used automatically. |
| 406 | See docs/features/quantization/fp8_vit_attn.md for details.""" |
| 407 | mm_encoder_fp8_scale_path: str | None = None |
| 408 | """Path to a JSON file containing per-layer FP8 Q/K/V scales for ViT |
| 409 | encoder attention. When provided (with `mm_encoder_attn_dtype="fp8"`), |
| 410 | static scaling is used. When omitted, dynamic scaling is used.""" |
| 411 | mm_encoder_fp8_scale_save_path: str | None = None |
| 412 | """When set with dynamic FP8 scaling (`mm_encoder_attn_dtype="fp8"` |
| 413 | and no `mm_encoder_fp8_scale_path`), saves the calibrated scales to |
| 414 | this file after the amax history buffer is full. The saved file can |
| 415 | then be used as `mm_encoder_fp8_scale_path` in subsequent runs.""" |
| 416 | mm_encoder_fp8_scale_save_margin: float = Field(default=1.5, gt=0.0) |
| 417 | """Safety margin multiplied onto scales when auto-saving. A value > 1 |
| 418 | leaves headroom so that inputs with larger activations than the |
| 419 | calibration set do not overflow FP8 range. Default 1.5.""" |
| 420 | interleave_mm_strings: bool = False |
| 421 | """Enable fully interleaved support for multimodal prompts, while using |
| 422 | --chat-template-content-format=string.""" |
| 423 | skip_mm_profiling: bool = False |
| 424 | """When enabled, skips multimodal memory profiling and only profiles with |
| 425 | language backbone model during engine initialization. |
| 426 | |
| 427 | This reduces engine startup time but shifts the responsibility to users for |
| 428 | estimating the peak memory usage of the activation of multimodal encoder and |
| 429 | embedding cache.""" |
| 430 | video_pruning_rate: float | None = Field(default=None, ge=0.0, lt=1.0) |
| 431 | """Fraction of video tokens to prune from each video. Value sits in range |
| 432 | [0;1); pruning is enabled when it is greater than 0. The pruning algorithm |
| 433 | is selected by `video_pruning_method`. |
| 434 | """ |
| 435 | video_pruning_method: VideoPruningMethod = "evs" |
| 436 | """Video token pruning algorithm applied when `video_pruning_rate` > 0: |
| 437 | - "evs": Efficient Video Sampling. |
| 438 | - "vidcom2": Video Compression Commander. |
| 439 | """ |
| 440 | mm_tensor_ipc: MMTensorIPC = "direct_rpc" |
| 441 | """IPC (inter-process communication) method for multimodal tensors. |
| 442 | - "direct_rpc": Use msgspec serialization via RPC |
| 443 | - "torch_shm": Use torch.multiprocessing shared memory for zero-copy IPC |
| 444 | Defaults to "direct_rpc". """ |
| 445 | allow_missing_mm_embeddings: bool = False |
| 446 | """Whether a pre-computed-embedding input may omit the `*_embeds` tensor. |
| 447 | |
| 448 | In an encode/prefill/decode (EPD) deployment the encoder instance publishes |
| 449 | embeddings through the EC connector. An EC consumer loads those embeddings |
| 450 | from the connector, while a KV consumer receives the resulting prompt KV |
| 451 | cache. Their requests only need the grid/size metadata that sizes the |
| 452 | placeholder range. |
| 453 | |
| 454 | Derived, not user-settable: `VllmConfig.__post_init__` sets this to True |
| 455 | on EC and KV consumers. Everywhere else it stays False so that a request |
| 456 | which forgets its embeddings still fails fast in the frontend, with a clear |
| 457 | error, rather than deep inside the model.""" |
| 458 | |
| 459 | mm_ipc_gpu_memory_gb: float = Field(default=0, ge=0) |
| 460 | """Amount of GPU memory (in GiB) sequestered on the engine's device for |
| 461 | GPU-side multimodal work in the API-server (frontend) process, such as |
| 462 | hardware video decoding. |
| 463 | |
| 464 | This budget is carved out of the engine's KV-cache memory so the headroom |
| 465 | physically exists, and frontend GPU decode paths acquire from a blocking |
| 466 | byte-counting semaphore of this size before allocating on the device. |
| 467 | |
| 468 | Set to `0` (default) to disable frontend GPU multimodal memory gating.""" |
| 469 | |
| 470 | @field_validator("limit_per_prompt", mode="before") |
| 471 | @classmethod |
| 472 | def _validate_limit_per_prompt( |
| 473 | cls, |
| 474 | value: dict[str, int | dict[str, int]], |
| 475 | ) -> MultiModalDummyOptions: |
| 476 | out = MultiModalDummyOptions() |
| 477 | |
| 478 | for k, v in value.items(): |
| 479 | # Handle legacy format where only count is specified |
| 480 | if isinstance(v, int): |
| 481 | v = {"count": v} |
| 482 | |
| 483 | # Convert to the appropriate DummyOptions subclass |
| 484 | if k == "video": |
| 485 | out[k] = VideoDummyOptions(**v) |
| 486 | elif k == "image": |
| 487 | out[k] = ImageDummyOptions(**v) |
| 488 | elif k == "audio": |
| 489 | out[k] = AudioDummyOptions(**v) |
| 490 | else: |
| 491 | out[k] = BaseDummyOptions(**v) |
| 492 | |
| 493 | return out |
| 494 | |
| 495 | @field_validator("mm_encoder_attn_backend", mode="before") |
| 496 | @classmethod |
| 497 | def _validate_mm_encoder_attn_backend( |
| 498 | cls, value: str | AttentionBackendEnum | None |
| 499 | ) -> AttentionBackendEnum | None: |
| 500 | if isinstance(value, str) and value.upper() == "XFORMERS": |
| 501 | raise ValueError( |
| 502 | "Attention backend 'XFORMERS' has been removed (See PR #29262 for " |
| 503 | "details). Please select a supported attention backend." |
| 504 | ) |
| 505 | |
| 506 | if value is None or isinstance(value, AttentionBackendEnum): |
| 507 | return value |
| 508 | |
| 509 | assert isinstance(value, str), ( |
| 510 | "mm_encoder_attn_backend must be a string or an AttentionBackendEnum." |
| 511 | ) |
| 512 | return AttentionBackendEnum[value.upper()] |
| 513 | |
| 514 | @model_validator(mode="after") |
| 515 | def _validate_multimodal_config(self): |
| 516 | if self.mm_processor_cache_type != "shm" and ( |
| 517 | self.mm_shm_cache_max_object_size_mb |
| 518 | != MultiModalConfig.mm_shm_cache_max_object_size_mb |
| 519 | ): |
| 520 | raise ValueError( |
| 521 | "'mm_shm_cache_max_object_size_mb' should only be set when " |
| 522 | "'mm_processor_cache_type' is 'shm'." |
| 523 | ) |
| 524 | # Validate FP8 scale path combinations. |
| 525 | if self.mm_encoder_attn_dtype != "fp8" and ( |
| 526 | self.mm_encoder_fp8_scale_path is not None |
| 527 | or self.mm_encoder_fp8_scale_save_path is not None |
| 528 | ): |
| 529 | raise ValueError( |
| 530 | "'mm_encoder_fp8_scale_path' and " |
| 531 | "'mm_encoder_fp8_scale_save_path' require " |
| 532 | "'mm_encoder_attn_dtype' to be 'fp8'." |
| 533 | ) |
| 534 | if ( |
| 535 | self.mm_encoder_fp8_scale_path is not None |
| 536 | and self.mm_encoder_fp8_scale_save_path is not None |
| 537 | ): |
| 538 | raise ValueError( |
| 539 | "'mm_encoder_fp8_scale_save_path' cannot be used with " |
| 540 | "'mm_encoder_fp8_scale_path' (saving requires dynamic scaling)." |
| 541 | ) |
| 542 | |
| 543 | # Validate file paths exist. |
| 544 | if self.mm_encoder_fp8_scale_path is not None: |
| 545 | scale_path = Path(self.mm_encoder_fp8_scale_path) |
| 546 | if not scale_path.is_file(): |
| 547 | raise FileNotFoundError(f"FP8 scale file not found: {scale_path}") |
| 548 | if self.mm_encoder_fp8_scale_save_path is not None: |
| 549 | save_parent = Path(self.mm_encoder_fp8_scale_save_path).parent |
| 550 | if not save_parent.is_dir(): |
| 551 | raise FileNotFoundError( |
| 552 | f"Parent directory for FP8 scale save path not found: {save_parent}" |
| 553 | ) |
| 554 | return self |
| 555 | |
| 556 | @staticmethod |
| 557 | def fold_mm_processor_device( |
| 558 | mm_processor_kwargs: dict[str, Any] | None, |
| 559 | mm_processor_device: MMProcessorDevice | None, |
| 560 | ) -> dict[str, Any] | None: |
| 561 | """Fold the `mm_processor_device` convenience flag into the kwargs. |
| 562 | |
| 563 | The flag keeps no state of its own: `mm_processor_kwargs["device"]` is |
| 564 | the only representation of where the processor runs, so an explicit |
| 565 | `device` there always wins and `"auto"` stays unresolved for |
| 566 | `VllmConfig`, which is where the EC role needed to resolve it lives. |
| 567 | |
| 568 | Args: |
| 569 | mm_processor_kwargs: The kwargs as given, or None. |
| 570 | mm_processor_device: The flag's value, or None when unset. |
| 571 | |
| 572 | Returns: |
| 573 | The kwargs to build the config with, unchanged unless the flag adds |
| 574 | a `device`. |
| 575 | |
| 576 | """ |
| 577 | if mm_processor_device in (None, "auto"): |
| 578 | return mm_processor_kwargs |
| 579 | if (mm_processor_kwargs or {}).get("device") is not None: |
| 580 | return mm_processor_kwargs |
| 581 | |
| 582 | from vllm.platforms import current_platform |
| 583 | |
| 584 | # Any explicit value other than "cpu" means "the accelerator", so a |
| 585 | # programmatically-set "cuda" still works on a platform whose device type |
| 586 | # is named differently ("xpu"), and degrades to CPU where there is none. |
| 587 | device = ( |
| 588 | "cpu" |
| 589 | if mm_processor_device == "cpu" |
| 590 | else (current_platform.device_type or "cpu") |
| 591 | ) |
| 592 | return {**(mm_processor_kwargs or {}), "device": device} |
| 593 | |
| 594 | def get_mm_processor_device_type(self) -> str | None: |
| 595 | """The torch device type `mm_processor_kwargs["device"]` names. |
| 596 | |
| 597 | `mm_processor_kwargs` is untyped, so `device` may be any form torch |
| 598 | accepts -- `"cuda"`, `"cuda:1"`, `torch.device(...)`, or a bare index. |
| 599 | Normalising through torch rather than parsing the string keeps the |
| 600 | non-string forms from slipping past a caller's comparison. |
| 601 | |
| 602 | Returns: |
| 603 | The device type, or None when no device is requested. |
| 604 | |
| 605 | Raises: |
| 606 | ValueError: If `device` is not something `torch.device` accepts. |
| 607 | `validate_mm_processor_device` is what surfaces this during |
| 608 | startup, so the value is only parsed once. |
| 609 | |
| 610 | """ |
| 611 | device = (self.mm_processor_kwargs or {}).get("device") |
| 612 | if device is None: |
| 613 | return None |
| 614 | try: |
| 615 | return torch.device(device).type # type: ignore[arg-type] |
| 616 | except (RuntimeError, TypeError, ValueError): |
| 617 | raise ValueError( |
| 618 | f'Invalid "device" in mm_processor_kwargs: {device!r}. Expected a ' |
| 619 | 'torch device such as "cpu", "cuda" or "cuda:0".' |
| 620 | ) from None |
| 621 | |
| 622 | def validate_mm_processor_device(self, ec_config: ECTransferConfig | None) -> None: |
| 623 | """Check `mm_processor_kwargs["device"]` for this deployment. |
| 624 | |
| 625 | The only place the requested device is validated, so it runs even on a |
| 626 | CPU-only platform: the value is parsed before any early return. |
| 627 | |
| 628 | Args: |
| 629 | ec_config: The deployment's EC config, or None when it is not an |
| 630 | encode/prefill/decode deployment. Passed in because it is not |
| 631 | reachable from here, and because a field assigned after |
| 632 | construction would not re-trigger this config's validators. |
| 633 | |
| 634 | Raises: |
| 635 | ValueError: If the requested device is not a torch device, or if it |
| 636 | is the accelerator on an instance that also runs the language |
| 637 | model. |
| 638 | |
| 639 | """ |
| 640 | from vllm.platforms import current_platform |
| 641 | |
| 642 | device_type = self.get_mm_processor_device_type() |
| 643 | accelerator = current_platform.device_type |
| 644 | if device_type is None or accelerator in ("", "cpu"): |
| 645 | return |
| 646 | if device_type != accelerator: |
| 647 | return |
| 648 | |
| 649 | if ec_config is None or not ec_config.is_encode_only: |
| 650 | raise ValueError( |
| 651 | f"Cannot run the multi-modal processor on {device_type!r}: this " |
| 652 | "instance also runs the language model. The processor would " |
| 653 | "share the device with the model's forward pass, so its " |
| 654 | "transform kernels contend with that compute, and because it " |
| 655 | "runs in the API-server process its allocations are outside the " |
| 656 | "memory the engine profiled for its KV cache -- risking OOM or " |
| 657 | "a silently shrunken cache.\n" |
| 658 | "Accelerator preprocessing is only supported on an encode-only " |
| 659 | "instance of an encode/prefill/decode deployment (an EC " |
| 660 | "producer that is not also a consumer), which runs no forward " |
| 661 | "pass and allocates no KV cache.\n" |
| 662 | 'Use --mm-processor-device=cpu, or drop "device" from ' |
| 663 | "--mm-processor-kwargs." |
| 664 | ) |
| 665 | |
| 666 | logger.info_once( |
| 667 | "Running the multi-modal processor on %s. Override with " |
| 668 | "--mm-processor-device=cpu.", |
| 669 | device_type, |
| 670 | ) |
| 671 | |
| 672 | def compute_hash(self) -> str: |
| 673 | """WARNING: Whenever a new field is added to this config, |
| 674 | ensure that it is included in the factors list if |
| 675 | it affects the computation graph. |
| 676 | |
| 677 | Provide a hash that uniquely identifies all the configs |
| 678 | that affect the structure of the computation |
| 679 | graph from input ids/embeddings to the final hidden states, |
| 680 | excluding anything before input ids/embeddings and after |
| 681 | the final hidden states. |
| 682 | """ |
| 683 | factors: list[Any] = [ |
| 684 | self.mm_encoder_attn_backend.name |
| 685 | if self.mm_encoder_attn_backend is not None |
| 686 | else None, |
| 687 | self.mm_encoder_tp_mode, |
| 688 | self.mm_encoder_attn_dtype, |
| 689 | self.mm_encoder_fp8_scale_path, |
| 690 | self.mm_device_do_normalize, |
| 691 | ] |
| 692 | hash_str = safe_hash(str(factors).encode(), usedforsecurity=False).hexdigest() |
| 693 | return hash_str |
| 694 | |
| 695 | def get_limit_per_prompt(self, modality: str) -> int: |
| 696 | """Get the maximum number of input items allowed per prompt |
| 697 | for the given modality (backward compatible). |
| 698 | """ |
| 699 | if self.language_model_only: |
| 700 | return 0 |
| 701 | |
| 702 | limit_data = self.limit_per_prompt.get(modality) |
| 703 | |
| 704 | if limit_data is None: |
| 705 | # Unspecified modality is set to 999 by default |
| 706 | return 999 |
| 707 | |
| 708 | return limit_data.count |
| 709 | |
| 710 | def merge_mm_processor_kwargs( |
| 711 | self, |
| 712 | inference_kwargs: Mapping[str, object], |
| 713 | ) -> dict[str, object]: |
| 714 | """Get the keyword arguments to pass to the multi-modal processor |
| 715 | according to the extra arguments passed during inference. |
| 716 | |
| 717 | Nested mappings are merged recursively, with inference-time values taking |
| 718 | precedence over configured values. |
| 719 | """ |
| 720 | merged = _merge_mm_processor_kwargs( |
| 721 | self.mm_processor_kwargs or {}, |
| 722 | inference_kwargs, |
| 723 | ) |
| 724 | if self.mm_device_do_normalize: |
| 725 | # The model expects raw pixels and normalises on device; |
| 726 | # overriding these per request would silently double-normalise. |
| 727 | for key in ("do_normalize", "do_rescale"): |
| 728 | if key in inference_kwargs: |
| 729 | logger.warning_once( |
| 730 | "mm_device_do_normalize is enabled; ignoring " |
| 731 | "per-request mm_processor_kwargs[%r].", |
| 732 | key, |
| 733 | ) |
| 734 | merged[key] = False |
| 735 | return merged |
| 736 | |
| 737 | def use_gpu_video_backend(self) -> bool: |
| 738 | """Return whether the configured video loader or codec uses the GPU.""" |
| 739 | from vllm.multimodal.video import VIDEO_LOADER_REGISTRY |
| 740 | |
| 741 | video_kwargs = self.media_io_kwargs.get("video", {}) |
| 742 | video_loader_backend = ( |
| 743 | video_kwargs.get("video_backend") or envs.VLLM_VIDEO_LOADER_BACKEND |
| 744 | ) |
| 745 | codec_backend = video_kwargs.get("backend") |
| 746 | return VIDEO_LOADER_REGISTRY.backend_requires_gpu(video_loader_backend) or ( |
| 747 | codec_backend is not None |
| 748 | and VIDEO_LOADER_REGISTRY.backend_requires_gpu(codec_backend) |
| 749 | ) |
| 750 | |
| 751 | def is_multimodal_pruning_enabled(self): |
| 752 | return self.get_video_pruning_spec() is not None |
| 753 | |
| 754 | def get_video_pruning_spec(self) -> tuple[VideoPruningMethod, float] | None: |
| 755 | """Return `(method, rate)` when video pruning is enabled, else None. |
| 756 | `rate` is the fraction of video tokens to prune.""" |
| 757 | if self.video_pruning_rate is not None and self.video_pruning_rate > 0: |
| 758 | return (self.video_pruning_method, float(self.video_pruning_rate)) |
| 759 | return None |