| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import time |
| 4 | from collections.abc import Callable, Mapping |
| 5 | from contextlib import contextmanager |
| 6 | from dataclasses import dataclass, field |
| 7 | from functools import cached_property |
| 8 | from typing import TYPE_CHECKING, Any, get_type_hints, overload |
| 9 | |
| 10 | import torch |
| 11 | from typing_extensions import TypeVar |
| 12 | |
| 13 | from vllm.exceptions import VLLMValidationError |
| 14 | from vllm.inputs import MultiModalDataDict |
| 15 | from vllm.logger import init_logger |
| 16 | from vllm.multimodal.parse import ( |
| 17 | DictEmbeddingItems, |
| 18 | EmbeddingItems, |
| 19 | MultiModalDataItems, |
| 20 | MultiModalDataParser, |
| 21 | ) |
| 22 | from vllm.tokenizers import TokenizerLike |
| 23 | from vllm.transformers_utils.processor import cached_processor_from_config |
| 24 | from vllm.utils.func_utils import get_allowed_kwarg_only_overrides |
| 25 | from vllm.utils.jsontree import JSONTree, json_map_leaves |
| 26 | from vllm.utils.mistral import is_mistral_tokenizer |
| 27 | |
| 28 | if TYPE_CHECKING: |
| 29 | from transformers.configuration_utils import PreTrainedConfig |
| 30 | from transformers.feature_extraction_utils import BatchFeature |
| 31 | from transformers.processing_utils import ProcessorMixin |
| 32 | |
| 33 | from vllm.config import ModelConfig |
| 34 | from vllm.renderers import TokenizeParams |
| 35 | else: |
| 36 | PreTrainedConfig = object |
| 37 | BatchFeature = object |
| 38 | ProcessorMixin = object |
| 39 | |
| 40 | ModelConfig = object |
| 41 | TokenizeParams = object |
| 42 | |
| 43 | logger = init_logger(__name__) |
| 44 | |
| 45 | # HuggingFace processors accept a shared flat namespace in addition to the |
| 46 | # nested processor kwarg scopes ``text_kwargs`` / ``images_kwargs`` / |
| 47 | # ``videos_kwargs`` / ``audio_kwargs``. vLLM recognizes these scopes when |
| 48 | # resolving ``mm_processor_kwargs``. |
| 49 | _HF_PROCESSOR_KWARG_SCOPES = ( |
| 50 | "text_kwargs", |
| 51 | "images_kwargs", |
| 52 | "videos_kwargs", |
| 53 | "audio_kwargs", |
| 54 | ) |
| 55 | |
| 56 | |
| 57 | def _merge_scoped_mm_processor_value( |
| 58 | flat_value: object, |
| 59 | scoped_value: object, |
| 60 | ) -> object: |
| 61 | """Merge a flat value with an existing, more specific scoped value. |
| 62 | |
| 63 | Mappings are merged recursively; otherwise the scoped value replaces the |
| 64 | flat value. |
| 65 | """ |
| 66 | if isinstance(flat_value, Mapping) and isinstance(scoped_value, Mapping): |
| 67 | # Merge recursively so scoped leaves override matching flat leaves |
| 68 | # without dropping unrelated flat entries. |
| 69 | merged = dict(flat_value) |
| 70 | for key, value in scoped_value.items(): |
| 71 | if key in merged: |
| 72 | merged[key] = _merge_scoped_mm_processor_value(merged[key], value) |
| 73 | else: |
| 74 | merged[key] = value |
| 75 | return merged |
| 76 | return scoped_value |
| 77 | |
| 78 | |
| 79 | def _resolve_mm_processor_kwargs( |
| 80 | kwargs: Mapping[str, object], |
| 81 | supported_mm_processor_kwargs: Mapping[str, set[str]] | None = None, |
| 82 | ) -> dict[str, object]: |
| 83 | """Resolve flat processor kwargs into HuggingFace processor kwarg scopes. |
| 84 | |
| 85 | With ``supported_mm_processor_kwargs``, each scope lists the flat keys it |
| 86 | supports. Matching flat keys are routed to every scope that supports them, |
| 87 | existing scoped values take precedence, and keys not supported by any scope |
| 88 | remain flat. |
| 89 | |
| 90 | Without ``supported_mm_processor_kwargs``, flat keys already represented in |
| 91 | at least one existing mapping-valued scope are removed from the shared flat |
| 92 | namespace; all other flat keys are left unchanged because there is no |
| 93 | information to determine which scopes should receive them. |
| 94 | """ |
| 95 | resolved = dict(kwargs) |
| 96 | |
| 97 | # Without ``supported_mm_processor_kwargs``, remove flat entries already |
| 98 | # represented in at least one existing mapping-valued scope. There is no |
| 99 | # information to route the remaining flat entries. |
| 100 | if supported_mm_processor_kwargs is None: |
| 101 | # Collect all keys represented in existing mapping-valued scopes. |
| 102 | # Non-mapping scope values do not contain scoped kwargs and are left |
| 103 | # untouched. |
| 104 | represented_scoped_keys: set[str] = set() |
| 105 | for scoped_key in _HF_PROCESSOR_KWARG_SCOPES: |
| 106 | scoped_kwargs = resolved.get(scoped_key) |
| 107 | if isinstance(scoped_kwargs, Mapping): |
| 108 | represented_scoped_keys.update(scoped_kwargs) |
| 109 | |
| 110 | # Remove flat entries for keys already represented in a scope. |
| 111 | for key in represented_scoped_keys: |
| 112 | if key not in _HF_PROCESSOR_KWARG_SCOPES: |
| 113 | resolved.pop(key, None) |
| 114 | return resolved |
| 115 | |
| 116 | # With ``supported_mm_processor_kwargs``, collect the flat kwargs before |
| 117 | # routing each one to the processor kwarg scopes that support it. |
| 118 | flat_kwargs = { |
| 119 | key: value |
| 120 | for key, value in resolved.items() |
| 121 | if key not in _HF_PROCESSOR_KWARG_SCOPES |
| 122 | } |
| 123 | |
| 124 | for key, flat_value in flat_kwargs.items(): |
| 125 | # Find every processor kwarg scope that supports this flat key. If none |
| 126 | # supports it, keep the key in the shared flat namespace. |
| 127 | supported_scopes = [ |
| 128 | scoped_key |
| 129 | for scoped_key in _HF_PROCESSOR_KWARG_SCOPES |
| 130 | if key in supported_mm_processor_kwargs.get(scoped_key, set()) |
| 131 | ] |
| 132 | if not supported_scopes: |
| 133 | continue |
| 134 | |
| 135 | # Add the flat value to each supported destination scope. |
| 136 | for scoped_key in supported_scopes: |
| 137 | # Start from a copy of the existing scoped kwargs, or an empty scope |
| 138 | # if absent. |
| 139 | if scoped_key in resolved: |
| 140 | scoped_kwargs = resolved[scoped_key] |
| 141 | # A scope must be a mapping before keys can be added to it. Do |
| 142 | # not replace an explicit non-mapping value. |
| 143 | if not isinstance(scoped_kwargs, Mapping): |
| 144 | raise TypeError(f"`{scoped_key}` must be a mapping") |
| 145 | scoped_kwargs = dict(scoped_kwargs) |
| 146 | else: |
| 147 | scoped_kwargs = {} |
| 148 | |
| 149 | # Add the flat value to this scope. Existing scoped values take |
| 150 | # precedence. |
| 151 | if key in scoped_kwargs: |
| 152 | scoped_kwargs[key] = _merge_scoped_mm_processor_value( |
| 153 | flat_value, scoped_kwargs[key] |
| 154 | ) |
| 155 | else: |
| 156 | # Assign the flat value to this scope. For mappings, assign a |
| 157 | # copy so the same mapping object is not shared between the flat |
| 158 | # value and multiple scopes. |
| 159 | scoped_kwargs[key] = ( |
| 160 | dict(flat_value) if isinstance(flat_value, Mapping) else flat_value |
| 161 | ) |
| 162 | resolved[scoped_key] = scoped_kwargs |
| 163 | |
| 164 | # After routing to every destination, remove the flat copy. |
| 165 | resolved.pop(key) |
| 166 | |
| 167 | return resolved |
| 168 | |
| 169 | |
| 170 | @dataclass |
| 171 | class TimingContext: |
| 172 | """Helper class to record execution times during multi-modal processing.""" |
| 173 | |
| 174 | enabled: bool = True |
| 175 | """If disabled, `TimingContext.record` becomes a no-op.""" |
| 176 | |
| 177 | stage_secs: dict[str, float] = field(default_factory=dict) |
| 178 | """The execution time (in seconds) for each processing stage.""" |
| 179 | |
| 180 | @property |
| 181 | def total_secs(self) -> float: |
| 182 | return sum(self.stage_secs.values()) |
| 183 | |
| 184 | @contextmanager |
| 185 | def record(self, stage: str): |
| 186 | """Record the execution time for a processing stage.""" |
| 187 | if not self.enabled: |
| 188 | yield |
| 189 | return |
| 190 | |
| 191 | start_time = time.perf_counter() |
| 192 | try: |
| 193 | yield |
| 194 | finally: |
| 195 | elapsed = time.perf_counter() - start_time |
| 196 | self.stage_secs.setdefault(stage, 0.0) |
| 197 | self.stage_secs[stage] += elapsed |
| 198 | |
| 199 | def get_stats_dict(self): |
| 200 | stats_dict = { |
| 201 | f"{stage}_secs": time_s for stage, time_s in self.stage_secs.items() |
| 202 | } |
| 203 | stats_dict["preprocessor_total_secs"] = self.total_secs |
| 204 | |
| 205 | return stats_dict |
| 206 | |
| 207 | |
| 208 | _T = TypeVar("_T") |
| 209 | _C = TypeVar("_C", bound=PreTrainedConfig, default=PreTrainedConfig) |
| 210 | _P = TypeVar("_P", bound=ProcessorMixin, default=ProcessorMixin) |
| 211 | |
| 212 | |
| 213 | @dataclass(frozen=True) |
| 214 | class InputProcessingContext: |
| 215 | """Contains information about the model which may be used to |
| 216 | modify the inputs. |
| 217 | """ |
| 218 | |
| 219 | model_config: ModelConfig |
| 220 | """The configuration of the model.""" |
| 221 | |
| 222 | tokenizer: TokenizerLike | None |
| 223 | """The tokenizer used to tokenize the inputs.""" |
| 224 | |
| 225 | def get_tokenizer(self) -> TokenizerLike: |
| 226 | if self.tokenizer is None: |
| 227 | raise ValueError( |
| 228 | "You cannot pass text prompts when `skip_tokenizer_init=True`" |
| 229 | ) |
| 230 | |
| 231 | return self.tokenizer |
| 232 | |
| 233 | @overload |
| 234 | def get_hf_config(self, /) -> PreTrainedConfig: ... |
| 235 | |
| 236 | @overload |
| 237 | def get_hf_config( |
| 238 | self, |
| 239 | typ: type[_C] | tuple[type[_C], ...], |
| 240 | /, |
| 241 | ) -> _C: ... |
| 242 | |
| 243 | def get_hf_config( |
| 244 | self, |
| 245 | typ: type[Any] | tuple[type[Any], ...] | None = None, |
| 246 | /, |
| 247 | ) -> Any: |
| 248 | """Get the HuggingFace configuration |
| 249 | (`transformers.PreTrainedConfig`) of the model, |
| 250 | additionally checking its type. |
| 251 | |
| 252 | Raises: |
| 253 | TypeError: If the configuration is not of the specified type. |
| 254 | |
| 255 | """ |
| 256 | if typ is None: |
| 257 | from transformers.configuration_utils import PreTrainedConfig |
| 258 | |
| 259 | typ = PreTrainedConfig |
| 260 | |
| 261 | hf_config = self.model_config.hf_config |
| 262 | if not isinstance(hf_config, typ): |
| 263 | raise TypeError( |
| 264 | "Invalid type of HuggingFace config. " |
| 265 | f"Expected type: {typ}, but " |
| 266 | f"found type: {type(hf_config)}" |
| 267 | ) |
| 268 | |
| 269 | return hf_config |
| 270 | |
| 271 | def get_hf_image_processor_config(self) -> dict[str, Any]: |
| 272 | """Get the HuggingFace image processor configuration of the model.""" |
| 273 | return self.model_config.hf_image_processor_config |
| 274 | |
| 275 | def get_mm_config(self): |
| 276 | """Get the multimodal config of the model. |
| 277 | |
| 278 | Raises: |
| 279 | RuntimeError: If the model is not a multimodal model. |
| 280 | |
| 281 | """ |
| 282 | mm_config = self.model_config.multimodal_config |
| 283 | if mm_config is None: |
| 284 | raise RuntimeError("Not a multimodal model") |
| 285 | |
| 286 | return mm_config |
| 287 | |
| 288 | @overload |
| 289 | def get_hf_processor(self, /, **kwargs: object) -> ProcessorMixin: ... |
| 290 | |
| 291 | @overload |
| 292 | def get_hf_processor( |
| 293 | self, |
| 294 | typ: type[_P] | tuple[type[_P], ...], |
| 295 | /, |
| 296 | **kwargs: object, |
| 297 | ) -> _P: ... |
| 298 | |
| 299 | def get_hf_processor( |
| 300 | self, |
| 301 | typ: type[Any] | tuple[type[Any], ...] | None = None, |
| 302 | /, |
| 303 | **kwargs: object, |
| 304 | ) -> Any: |
| 305 | """Get the HuggingFace processor |
| 306 | (`transformers.ProcessorMixin`) of the model, |
| 307 | additionally checking its type. |
| 308 | |
| 309 | Raises: |
| 310 | TypeError: If the processor is not of the specified type. |
| 311 | |
| 312 | """ |
| 313 | if typ is None: |
| 314 | from transformers.processing_utils import ProcessorMixin |
| 315 | |
| 316 | typ = ProcessorMixin |
| 317 | |
| 318 | tokenizer = self.tokenizer |
| 319 | if is_mistral_tokenizer(tokenizer): |
| 320 | tokenizer = tokenizer.transformers_tokenizer # type: ignore[union-attr] |
| 321 | |
| 322 | merged_kwargs = self.get_merged_mm_kwargs(kwargs) |
| 323 | merged_kwargs.pop("tokenizer", None) |
| 324 | |
| 325 | return cached_processor_from_config( |
| 326 | self.model_config, |
| 327 | processor_cls=typ, |
| 328 | tokenizer=tokenizer, |
| 329 | **merged_kwargs, |
| 330 | ) |
| 331 | |
| 332 | def init_processor( |
| 333 | self, |
| 334 | typ: type[_T], |
| 335 | /, |
| 336 | **kwargs: object, |
| 337 | ) -> _T: |
| 338 | """Initialize a HuggingFace-like processor class, merging the |
| 339 | keyword arguments with those in the model's configuration. |
| 340 | """ |
| 341 | merged_kwargs = self.get_merged_mm_kwargs(kwargs) |
| 342 | |
| 343 | return typ(**merged_kwargs) |
| 344 | |
| 345 | def _postprocess_output( |
| 346 | self, |
| 347 | output: JSONTree, |
| 348 | ) -> JSONTree: |
| 349 | # "torch_shm" puts tensors on a torch.multiprocessing queue, which |
| 350 | # shares device tensors by CUDA IPC handle, so a device-side processor |
| 351 | # can hand `pixel_values` straight to the worker. Every other transport |
| 352 | # serializes host bytes, so the result has to be copied back first. |
| 353 | keep_on_device = ( |
| 354 | self.model_config.get_multimodal_config().mm_tensor_ipc == "torch_shm" |
| 355 | ) |
| 356 | |
| 357 | def _postprocess_one(x: object): |
| 358 | if not isinstance(x, torch.Tensor): |
| 359 | return x |
| 360 | |
| 361 | # Bind to a Tensor-typed local: reassigning the `object`-typed |
| 362 | # parameter would discard the isinstance narrowing. |
| 363 | tensor = x |
| 364 | |
| 365 | # This mimics the behavior of transformers.BatchFeature |
| 366 | if tensor.is_floating_point(): |
| 367 | tensor = tensor.to(dtype=self.model_config.dtype) |
| 368 | |
| 369 | if not tensor.is_cpu and not keep_on_device: |
| 370 | tensor = tensor.cpu() |
| 371 | |
| 372 | return tensor |
| 373 | |
| 374 | return json_map_leaves(_postprocess_one, output) |
| 375 | |
| 376 | def get_merged_mm_kwargs( |
| 377 | self, |
| 378 | kwargs: Mapping[str, object], |
| 379 | *, |
| 380 | supported_mm_processor_kwargs: Mapping[str, set[str]] | None = None, |
| 381 | ) -> dict[str, Any]: |
| 382 | """Merge configured and request ``mm_processor_kwargs``. |
| 383 | |
| 384 | When ``supported_mm_processor_kwargs`` is provided, flat keys are matched |
| 385 | against the keys supported by each HuggingFace processor kwarg scope. |
| 386 | Matching flat keys are routed to every scope that supports them and |
| 387 | removed from the shared flat namespace; keys not supported by any scope |
| 388 | remain flat. |
| 389 | |
| 390 | Without ``supported_mm_processor_kwargs``, flat keys already represented |
| 391 | in at least one existing mapping-valued scope are removed from the shared |
| 392 | flat namespace; all other flat keys are left unchanged because there is |
| 393 | no information to determine which scopes should receive them. |
| 394 | """ |
| 395 | mm_config = self.model_config.get_multimodal_config() |
| 396 | merged = mm_config.merge_mm_processor_kwargs(kwargs) |
| 397 | return _resolve_mm_processor_kwargs( |
| 398 | merged, |
| 399 | supported_mm_processor_kwargs=supported_mm_processor_kwargs, |
| 400 | ) |
| 401 | |
| 402 | def call_hf_processor( |
| 403 | self, |
| 404 | hf_processor: Callable[..., BatchFeature] | ProcessorMixin, |
| 405 | data: Mapping[str, object], |
| 406 | kwargs: Mapping[str, object] = {}, |
| 407 | ) -> BatchFeature: |
| 408 | """Call `hf_processor` on the prompt `data` |
| 409 | (text, image, audio...) with configurable options `kwargs`. |
| 410 | """ |
| 411 | assert callable(hf_processor) |
| 412 | |
| 413 | merged_kwargs = self.get_merged_mm_kwargs(kwargs) |
| 414 | |
| 415 | allowed_kwargs = get_allowed_kwarg_only_overrides( |
| 416 | hf_processor, |
| 417 | merged_kwargs, |
| 418 | requires_kw_only=False, |
| 419 | allow_var_kwargs=True, |
| 420 | ) |
| 421 | allowed_kwargs.setdefault("return_tensors", "pt") |
| 422 | |
| 423 | try: |
| 424 | output = hf_processor(**data, **allowed_kwargs) |
| 425 | except Exception as exc: |
| 426 | msg = ( |
| 427 | f"Failed to apply {type(hf_processor).__name__} " |
| 428 | f"on data={data} with kwargs={allowed_kwargs}" |
| 429 | ) |
| 430 | |
| 431 | raise ValueError(msg) from exc |
| 432 | |
| 433 | # this emulates output.to(dtype=self.model_config.dtype) |
| 434 | from transformers.feature_extraction_utils import BatchFeature |
| 435 | |
| 436 | if isinstance(output, BatchFeature): |
| 437 | output_ = self._postprocess_output(output.data) |
| 438 | return BatchFeature(output_) # type: ignore |
| 439 | |
| 440 | logger.warning_once( |
| 441 | "%s did not return `BatchFeature`. " |
| 442 | "Make sure to match the behaviour of `ProcessorMixin` when " |
| 443 | "implementing custom processors.", |
| 444 | type(hf_processor).__name__, |
| 445 | ) |
| 446 | |
| 447 | return self._postprocess_output(output) # type: ignore |
| 448 | |
| 449 | |
| 450 | class BaseProcessingInfo: |
| 451 | """Base class to provide the information necessary for data processing.""" |
| 452 | |
| 453 | def __init__(self, ctx: InputProcessingContext) -> None: |
| 454 | super().__init__() |
| 455 | |
| 456 | self.ctx = ctx |
| 457 | |
| 458 | @property |
| 459 | def model_id(self) -> str: |
| 460 | return self.ctx.model_config.model |
| 461 | |
| 462 | def get_tokenizer(self) -> TokenizerLike: |
| 463 | return self.ctx.get_tokenizer() |
| 464 | |
| 465 | def get_hf_config(self) -> PreTrainedConfig: |
| 466 | return self.ctx.get_hf_config() |
| 467 | |
| 468 | def get_hf_processor(self, **kwargs: object) -> ProcessorMixin: |
| 469 | """Subclasses can override this method to handle |
| 470 | specific kwargs from model config or user inputs. |
| 471 | """ |
| 472 | return self.ctx.get_hf_processor(**kwargs) |
| 473 | |
| 474 | def get_supported_mm_processor_kwargs(self) -> dict[str, set[str]]: |
| 475 | """Return supported kwarg names for each HF processor kwargs scope.""" |
| 476 | processor = self.get_hf_processor() |
| 477 | processor_kwargs = get_type_hints(processor.valid_processor_kwargs) |
| 478 | |
| 479 | supported = { |
| 480 | "text_kwargs": set(processor_kwargs["text_kwargs"].__annotations__), |
| 481 | } |
| 482 | if "image" in self.supported_mm_limits: |
| 483 | supported["images_kwargs"] = set( |
| 484 | processor.image_processor.valid_kwargs.__annotations__ |
| 485 | ) |
| 486 | if "video" in self.supported_mm_limits: |
| 487 | supported["videos_kwargs"] = set( |
| 488 | processor.video_processor.valid_kwargs.__annotations__ |
| 489 | ) |
| 490 | if "audio" in self.supported_mm_limits: |
| 491 | supported["audio_kwargs"] = set( |
| 492 | processor_kwargs["audio_kwargs"].__annotations__ |
| 493 | ) |
| 494 | |
| 495 | return supported |
| 496 | |
| 497 | @cached_property |
| 498 | def supported_mm_processor_kwargs(self) -> dict[str, set[str]]: |
| 499 | """Supported kwarg names for each HF processor kwargs scope.""" |
| 500 | return self.get_supported_mm_processor_kwargs() |
| 501 | |
| 502 | def _merge_and_resolve_mm_processor_kwargs( |
| 503 | self, |
| 504 | mm_kwargs: Mapping[str, object], |
| 505 | ) -> dict[str, Any]: |
| 506 | """Merge configured and request ``mm_processor_kwargs``. |
| 507 | |
| 508 | Flat kwargs are routed into the HuggingFace processor kwarg scopes that |
| 509 | support them after the configured/request merge. When a routed flat value |
| 510 | conflicts with an existing scoped value, the scoped value takes precedence. |
| 511 | """ |
| 512 | return self.ctx.get_merged_mm_kwargs( |
| 513 | mm_kwargs, |
| 514 | supported_mm_processor_kwargs=self.supported_mm_processor_kwargs, |
| 515 | ) |
| 516 | |
| 517 | def get_default_tok_params(self) -> TokenizeParams: |
| 518 | """Construct the default parameters for tokenization.""" |
| 519 | from vllm.renderers import TokenizeParams |
| 520 | |
| 521 | model_config = self.ctx.model_config |
| 522 | encoder_config = model_config.encoder_config or {} |
| 523 | |
| 524 | return TokenizeParams( |
| 525 | max_total_tokens=model_config.max_model_len, |
| 526 | do_lower_case=encoder_config.get("do_lower_case", False), |
| 527 | add_special_tokens=True, |
| 528 | ) |
| 529 | |
| 530 | @cached_property |
| 531 | def default_tok_params(self) -> TokenizeParams: |
| 532 | return self.get_default_tok_params() |
| 533 | |
| 534 | def _get_expected_hidden_size(self) -> int | None: |
| 535 | """Get expected hidden size for embedding validation if `mm_embeds` are enabled. |
| 536 | |
| 537 | This validates hidden dimensions to prevent a vulnerability where embeddings |
| 538 | with correct `ndim` but wrong `shape` could cause crashes at inference time. |
| 539 | """ |
| 540 | model_config = self.ctx.model_config |
| 541 | mm_config = model_config.get_multimodal_config() |
| 542 | |
| 543 | if mm_config.enable_mm_embeds: |
| 544 | return model_config.get_inputs_embeds_size() |
| 545 | |
| 546 | return None |
| 547 | |
| 548 | @property |
| 549 | def allow_missing_mm_embeddings(self) -> bool: |
| 550 | """Whether pre-computed embedding tensors may be omitted.""" |
| 551 | mm_config = self.ctx.model_config.multimodal_config |
| 552 | return mm_config is not None and mm_config.allow_missing_mm_embeddings |
| 553 | |
| 554 | def get_data_parser(self) -> MultiModalDataParser: |
| 555 | """Constructs a parser to preprocess multi-modal data items |
| 556 | before passing them to |
| 557 | [`_get_hf_mm_inputs`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs]. |
| 558 | |
| 559 | You can support additional modalities by creating a subclass |
| 560 | of [`MultiModalDataParser`][vllm.multimodal.parse.MultiModalDataParser] |
| 561 | that has additional subparsers. |
| 562 | """ |
| 563 | return MultiModalDataParser( |
| 564 | expected_hidden_size=self._get_expected_hidden_size(), |
| 565 | allow_missing_mm_embeddings=self.allow_missing_mm_embeddings, |
| 566 | ) |
| 567 | |
| 568 | @cached_property |
| 569 | def data_parser(self) -> MultiModalDataParser: |
| 570 | return self.get_data_parser() |
| 571 | |
| 572 | @property |
| 573 | def skip_prompt_length_check(self) -> bool: |
| 574 | return False |
| 575 | |
| 576 | def get_supported_mm_limits(self) -> Mapping[str, int | None]: |
| 577 | """Return the maximum supported number of items for each modality. |
| 578 | |
| 579 | A value of `None` means unlimited number of items. |
| 580 | |
| 581 | Omitting a modality from the returned dictionary means that |
| 582 | it is not supported at all. |
| 583 | """ |
| 584 | raise NotImplementedError |
| 585 | |
| 586 | @cached_property |
| 587 | def supported_mm_limits(self) -> Mapping[str, int | None]: |
| 588 | """The maximum supported number of items for each modality.""" |
| 589 | return self.get_supported_mm_limits() |
| 590 | |
| 591 | @cached_property |
| 592 | def allowed_mm_limits(self) -> Mapping[str, int]: |
| 593 | """The maximum allowed number of items for each modality.""" |
| 594 | mm_config = self.ctx.get_mm_config() |
| 595 | |
| 596 | allowed_limits = dict[str, int]() |
| 597 | for modality, supported_limit in self.supported_mm_limits.items(): |
| 598 | user_limit = mm_config.get_limit_per_prompt(modality) |
| 599 | |
| 600 | allowed_limits[modality] = ( |
| 601 | user_limit |
| 602 | if supported_limit is None |
| 603 | else min(user_limit, supported_limit) |
| 604 | ) |
| 605 | |
| 606 | return allowed_limits |
| 607 | |
| 608 | def validate_num_items(self, modality: str, num_items: int) -> None: |
| 609 | """Raise `ValueError` if the number of input items for the given modality |
| 610 | is invalid. |
| 611 | """ |
| 612 | supported_limit = self.supported_mm_limits.get(modality, 0) |
| 613 | allowed_limit = self.allowed_mm_limits.get(modality, 0) |
| 614 | |
| 615 | if supported_limit is None: |
| 616 | supported_limit = allowed_limit |
| 617 | |
| 618 | limit = min(supported_limit, allowed_limit) |
| 619 | |
| 620 | if num_items > limit: |
| 621 | msg = f"At most {limit} {modality}(s) may be provided in one prompt." |
| 622 | |
| 623 | if num_items <= supported_limit: |
| 624 | msg += " Set `--limit-mm-per-prompt` to increase this limit." |
| 625 | |
| 626 | raise VLLMValidationError(msg, parameter=modality) |
| 627 | |
| 628 | def parse_mm_data( |
| 629 | self, |
| 630 | mm_data: MultiModalDataDict, |
| 631 | *, |
| 632 | validate: bool = True, |
| 633 | ) -> MultiModalDataItems: |
| 634 | """Normalize [`MultiModalDataDict`][vllm.inputs.MultiModalDataDict] |
| 635 | to [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems] |
| 636 | before passing them to |
| 637 | [`_get_hf_mm_inputs`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs]. |
| 638 | """ |
| 639 | mm_items = self.data_parser.parse_mm_data(mm_data) |
| 640 | |
| 641 | if validate: |
| 642 | mm_config = self.ctx.get_mm_config() |
| 643 | |
| 644 | for modality, items in mm_items.items(): |
| 645 | if isinstance(items, (EmbeddingItems, DictEmbeddingItems)): |
| 646 | if not mm_config.enable_mm_embeds: |
| 647 | raise ValueError( |
| 648 | f"You must set `--enable-mm-embeds` to input " |
| 649 | f"`{modality}_embeds`" |
| 650 | ) |
| 651 | if mm_config.get_limit_per_prompt(modality) == 0: |
| 652 | logger.debug( |
| 653 | "Skipping count validation for modality " |
| 654 | "'%s' (embeddings with limit=0)", |
| 655 | modality, |
| 656 | ) |
| 657 | continue |
| 658 | self.validate_num_items(modality, len(items)) |
| 659 | |
| 660 | return mm_items |
| 661 | |
| 662 | def get_mm_max_tokens_per_item( |
| 663 | self, |
| 664 | seq_len: int, |
| 665 | mm_counts: Mapping[str, int], |
| 666 | ) -> Mapping[str, int] | None: |
| 667 | """Return the maximum number of tokens per item of for each modality. |
| 668 | |
| 669 | When `None` (the default) is returned, vLLM will generate dummy inputs |
| 670 | (images/videos) at maximum possible sizes and process them to determine |
| 671 | the maximum token count per modality. |
| 672 | |
| 673 | This approach works but can be very slow for certain models (e.g., |
| 674 | Qwen2.5-VL), leading to very long startup time. For better performance, |
| 675 | each model can override this method to return pre-computed maximum token |
| 676 | counts, avoiding the need for dummy input generation and processing. |
| 677 | |
| 678 | Note: |
| 679 | The maximum number of tokens per item of each modality returned |
| 680 | from this function should respect the model's maximum sequence |
| 681 | length and the maximum number of items of each modality allowed, |
| 682 | and agree with dummy inputs (images/videos) at maximum possible |
| 683 | sizes. |
| 684 | |
| 685 | """ |
| 686 | return None |