@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c4afdb69cc

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

Move `MultiModalConfig` from `config/init.py` to `config/multimodal.py` (#24659)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

Harry Mellor committed Sep 15, 2025 at 18:43 UTC c4afdb69cc22a23d23886528147b956796c1000c

11 files changed +229 -247

tests/entrypoints/openai/test_lora_resolvers.py

+1 -1

@@ -9,7 +9,7 @@ from unittest.mock import MagicMock

10 import pytest

12 -from vllm.config import MultiModalConfig

12 +from vllm.config.multimodal import MultiModalConfig

13 from vllm.engine.multiprocessing.client import MQLLMEngineClient

14 from vllm.entrypoints.openai.protocol import CompletionRequest, ErrorResponse

15 from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion

tests/entrypoints/openai/test_serving_chat.py

+1 -1

@@ -12,7 +12,7 @@ from unittest.mock import MagicMock

12 import pytest

13 import pytest_asyncio

15 -from vllm.config import MultiModalConfig

15 +from vllm.config.multimodal import MultiModalConfig

16 from vllm.engine.multiprocessing.client import MQLLMEngineClient

17 from vllm.entrypoints.openai.protocol import ChatCompletionRequest

18 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat

tests/multimodal/test_cache.py

+12 -9

@@ -7,6 +7,7 @@ import pytest

7 import torch

9 from vllm.config import ModelConfig, ParallelConfig, VllmConfig

10 +from vllm.multimodal import MULTIMODAL_REGISTRY

11 from vllm.multimodal.cache import (MultiModalCache,

12 MultiModalProcessorCacheItem,

13 MultiModalProcessorCacheItemMetadata,

@@ -17,7 +18,6 @@ from vllm.multimodal.inputs import (MultiModalFieldElem, MultiModalKwargsItem,

18 MultiModalKwargsItems,

19 MultiModalSharedField)

20 from vllm.multimodal.processing import PromptInsertion

20 -from vllm.multimodal.registry import MultiModalRegistry

23 def _dummy_elem(

@@ -96,7 +96,9 @@ def _create_vllm_config(

96 enable_ipc: bool,

97 ):

98 return VllmConfig(

99 - model_config=ModelConfig(mm_processor_cache_gb=mm_processor_cache_gb),

99 + model_config=ModelConfig(

100 + model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf",

101 + mm_processor_cache_gb=mm_processor_cache_gb),

102 parallel_config=ParallelConfig(

103 data_parallel_size=1 if enable_ipc else 2),

104 )

@@ -113,15 +115,16 @@ def _compare_caches(

115 n_iter: int = 100,

116 seed: int = 0,

117 ):

116 - mm_registry = MultiModalRegistry()

117 - cache_0_p0 = processor_cache_from_config(config_0, mm_registry)

118 - cache_0_p1 = engine_receiver_cache_from_config(config_0, mm_registry)

119 - cache_1_p0 = processor_cache_from_config(config_1, mm_registry)

120 - cache_1_p1 = engine_receiver_cache_from_config(config_1, mm_registry)

118 + cache_0_p0 = processor_cache_from_config(config_0, MULTIMODAL_REGISTRY)

119 + cache_0_p1 = engine_receiver_cache_from_config(config_0,

120 + MULTIMODAL_REGISTRY)

121 + cache_1_p0 = processor_cache_from_config(config_1, MULTIMODAL_REGISTRY)

122 + cache_1_p1 = engine_receiver_cache_from_config(config_1,

123 + MULTIMODAL_REGISTRY)

124

125 cache_size_gb = max(

123 - config_0.model_config.mm_processor_cache_gb,

124 - config_1.model_config.mm_processor_cache_gb,

126 + config_0.model_config.multimodal_config.mm_processor_cache_gb,

127 + config_1.model_config.multimodal_config.mm_processor_cache_gb,

128 )

129 item_size_gb = int(cache_size_gb / item_capacity)

130

tests/test_config.py

+2 -2

@@ -6,9 +6,9 @@ from dataclasses import MISSING, Field, asdict, dataclass, field

6 import pytest

8 from vllm.compilation.backends import VllmBackend

9 -from vllm.config import (ModelConfig, PoolerConfig, VllmConfig, get_field,

10 - update_config)

9 +from vllm.config import ModelConfig, PoolerConfig, VllmConfig, update_config

10 from vllm.config.load import LoadConfig

11 +from vllm.config.utils import get_field

12 from vllm.model_executor.layers.pooler import PoolingType

13 from vllm.platforms import current_platform

tests/v1/engine/test_processor_multi_modal_uuids.py

+1 -1

@@ -31,7 +31,7 @@ def _mk_processor(monkeypatch,

31 raising=True)

32 monkeypatch.setattr(ModelConfig,

33 "__post_init__",

34 - lambda self: None,

34 + lambda self, *args: None,

35 raising=True)

36 monkeypatch.setattr(UnspecifiedPlatform,

37 "is_async_output_supported",

vllm/config/__init__.py

+56 -221

@@ -11,13 +11,12 @@ import json

11 import os

12 import textwrap

13 import warnings

14 -from collections.abc import Mapping

14 from contextlib import contextmanager

16 -from dataclasses import MISSING, Field, field, fields, is_dataclass, replace

15 +from dataclasses import InitVar, field, fields, is_dataclass, replace

16 from functools import cached_property, lru_cache

17 from importlib.util import find_spec

19 -from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Literal, Optional,

20 - Protocol, TypeVar, Union, cast, get_args)

18 +from typing import (TYPE_CHECKING, Any, Callable, Literal, Optional, Protocol,

19 + TypeVar, Union, cast, get_args)

21 import regex as re

22 import torch

@@ -37,6 +36,8 @@ from vllm.config.kv_events import KVEventsConfig

36 from vllm.config.kv_transfer import KVTransferConfig

37 from vllm.config.load import LoadConfig

38 from vllm.config.lora import LoRAConfig

39 +from vllm.config.multimodal import (MMCacheType, MMEncoderTPMode,

40 + MultiModalConfig)

41 from vllm.config.parallel import (DistributedExecutorBackend, EPLBConfig,

42 ParallelConfig)

43 from vllm.config.scheduler import SchedulerConfig, SchedulerPolicy

@@ -238,31 +239,12 @@ def get_attr_docs(cls: type[Any]) -> dict[str, str]:

239 return out

240

241

241 -def get_field(cls: ConfigType, name: str) -> Field:

242 - """Get the default factory field of a dataclass by name. Used for getting

243 - default factory fields in `EngineArgs`."""

244 - if not is_dataclass(cls):

245 - raise TypeError("The given class is not a dataclass.")

246 - cls_fields = {f.name: f for f in fields(cls)}

247 - if name not in cls_fields:

248 - raise ValueError(f"Field '{name}' not found in {cls.__name__}.")

249 - named_field: Field = cls_fields[name]

250 - if (default_factory := named_field.default_factory) is not MISSING:

251 - return field(default_factory=default_factory)

252 - if (default := named_field.default) is not MISSING:

253 - return field(default=default)

254 - raise ValueError(

255 - f"{cls.__name__}.{name} must have a default value or default factory.")

256 -

257 -

242 def is_init_field(cls: ConfigType, name: str) -> bool:

243 return next(f for f in fields(cls) if f.name == name).init

244

245

246 TokenizerMode = Literal["auto", "slow", "mistral", "custom"]

247 ModelDType = Literal["auto", "half", "float16", "bfloat16", "float", "float32"]

264 -MMEncoderTPMode = Literal["weights", "data"]

265 -MMCacheType = Literal["shm", "lru"]

248

249

250 class LogprobsMode(enum.Enum):

@@ -407,20 +389,6 @@ class ModelConfig:

389 that this name(s) will also be used in `model_name` tag content of

390 prometheus metrics, if multiple names provided, metrics tag will take the

391 first one."""

410 - limit_mm_per_prompt: dict[str, int] = field(default_factory=dict)

411 - """Maximum number of data items per modality per prompt. Only applicable

412 - for multimodal models."""

413 - interleave_mm_strings: bool = False

414 - """Enable fully interleaved support for multimodal prompts, while using

415 - --chat-template-content-format=string. Defaults to False."""

416 - skip_mm_profiling: bool = False

417 - """When enabled, skips multimodal memory profiling and only profiles with

418 - language backbone model during engine initialization.

419 - """

420 - media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)

421 - """Additional args passed to process media inputs, keyed by modalities.

422 - For example, to set num_frames for video, set

423 - `--media-io-kwargs '{"video": {"num_frames": 40} }'` """

392 use_async_output_proc: bool = True

393 """Whether to use async output processor."""

394 config_format: Union[str, ConfigFormat] = "auto"

@@ -436,41 +404,6 @@ class ModelConfig:

404 hf_overrides: HfOverrides = field(default_factory=dict)

405 """If a dictionary, contains arguments to be forwarded to the Hugging Face

406 config. If a callable, it is called to update the HuggingFace config."""

439 - mm_processor_kwargs: Optional[dict[str, Any]] = None

440 - """Arguments to be forwarded to the model's processor for multi-modal data,

441 - e.g., image processor. Overrides for the multi-modal processor obtained

442 - from `AutoProcessor.from_pretrained`. The available overrides depend on the

443 - model that is being run. For example, for Phi-3-Vision: `{"num_crops": 4}`.

444 - """

445 - mm_processor_cache_gb: float = 4

446 - """The size (in GiB) of the multi-modal processor cache, which is used to

447 - avoid re-processing past multi-modal inputs.

448 -

449 - This cache is duplicated for each API process and engine core process,

450 - resulting in a total memory usage of

451 - `mm_processor_cache_gb * (api_server_count + data_parallel_size)`.

452 -

453 - Set to `0` to disable this cache completely (not recommended)."""

454 - mm_processor_cache_type: MMCacheType = "lru"

455 - """Type of cache to use for the multi-modal preprocessor/mapper. If `shm`,

456 - use shared memory FIFO cache. If `lru`, use mirrored LRU cache."""

457 - mm_shm_cache_max_object_size_mb: int = 128

458 - """Size limit (in MiB) for each object stored in the multi-modal processor

459 - shared memory cache. Only effective when `mm_processor_cache_type` is

460 - `"shm"`."""

461 - mm_encoder_tp_mode: MMEncoderTPMode = "weights"

462 - """Indicates how to optimize multi-modal encoder inference using

463 - tensor parallelism (TP).

464 -

465 - - `"weights"`: Within the same vLLM engine, split the weights of

466 - each layer across TP ranks. (default TP behavior)

467 - - `"data"`: Within the same vLLM engine, split the batched input data

468 - across TP ranks to process the data in parallel, while hosting

469 - the full weights on each TP rank.

470 - This batch-level DP is not to be confused with API request-level

471 - DP (which is controlled by `--data-parallel-size`).

472 - This is only supported on a per-model basis and falls back to

473 - `"weights"` if the encoder does not support DP."""

407 pooler_config: Optional["PoolerConfig"] = field(init=False)

408 """Pooler config which controls the behaviour of output pooling in pooling

409 models."""

@@ -513,6 +446,18 @@ class ModelConfig:

446 io_processor_plugin: Optional[str] = None

447 """IOProcessor plugin name to load at model startup"""

448

449 + # Multimodal config and init vars

450 + multimodal_config: Optional[MultiModalConfig] = None

451 + limit_mm_per_prompt: InitVar[Optional[dict[str, int]]] = None

452 + media_io_kwargs: InitVar[Optional[dict[str, dict[str, Any]]]] = None

453 + mm_processor_kwargs: InitVar[Optional[dict[str, Any]]] = None

454 + mm_processor_cache_gb: InitVar[Optional[float]] = None

455 + mm_processor_cache_type: InitVar[Optional[MMCacheType]] = None

456 + mm_shm_cache_max_object_size_mb: InitVar[Optional[int]] = None

457 + mm_encoder_tp_mode: InitVar[Optional[MMEncoderTPMode]] = None

458 + interleave_mm_strings: InitVar[Optional[bool]] = None

459 + skip_mm_profiling: InitVar[Optional[bool]] = None

460 +

461 def compute_hash(self) -> str:

462 """

463 WARNING: Whenever a new field is added to this config,

@@ -546,7 +491,18 @@ class ModelConfig:

491 assert_hashable(str_factors)

492 return hashlib.sha256(str(factors).encode()).hexdigest()

493

549 - def __post_init__(self) -> None:

494 + def __post_init__(

495 + self,

496 + # Multimodal config init vars

497 + limit_mm_per_prompt: Optional[dict[str, int]],

498 + media_io_kwargs: Optional[dict[str, dict[str, Any]]],

499 + mm_processor_kwargs: Optional[dict[str, Any]],

500 + mm_processor_cache_gb: Optional[float],

501 + mm_processor_cache_type: Optional[MMCacheType],

502 + mm_shm_cache_max_object_size_mb: Optional[int],

503 + mm_encoder_tp_mode: Optional[MMEncoderTPMode],

504 + interleave_mm_strings: Optional[bool],

505 + skip_mm_profiling: Optional[bool]) -> None:

506 # Set the default seed to 0 in V1.

507 # NOTE(woosuk): In V0, we set the default seed to None because the

508 # driver worker shares the same process as the user process, and thus

@@ -777,7 +733,33 @@ class ModelConfig:

733

734 self.original_max_model_len = self.max_model_len

735 self.max_model_len = self.get_and_verify_max_len(self.max_model_len)

780 - self.multimodal_config = self._init_multimodal_config()

736 + # Init multimodal config if needed

737 + if self._model_info.supports_multimodal:

738 + if (mm_encoder_tp_mode == "data" and

739 + not self._model_info.supports_multimodal_encoder_tp_data):

740 + logger.warning_once(

741 + "This model does not support `--mm-encoder-tp-mode data`. "

742 + "Falling back to `--mm-encoder-tp-mode weights`.")

743 + mm_encoder_tp_mode = "weights"

744 +

745 + mm_config_kwargs = dict(

746 + limit_per_prompt=limit_mm_per_prompt,

747 + media_io_kwargs=media_io_kwargs,

748 + mm_processor_kwargs=mm_processor_kwargs,

749 + mm_processor_cache_gb=mm_processor_cache_gb,

750 + mm_processor_cache_type=mm_processor_cache_type,

751 + mm_shm_cache_max_object_size_mb=mm_shm_cache_max_object_size_mb,

752 + mm_encoder_tp_mode=mm_encoder_tp_mode,

753 + interleave_mm_strings=interleave_mm_strings,

754 + skip_mm_profiling=skip_mm_profiling,

755 + )

756 +

757 + mm_config_kwargs = {

758 + k: v

759 + for k, v in mm_config_kwargs.items() if v is not None

760 + }

761 +

762 + self.multimodal_config = MultiModalConfig(**mm_config_kwargs)

763

764 if self.disable_sliding_window:

765 # Set after get_and_verify_max_len to ensure that max_model_len

@@ -875,30 +857,6 @@ class ModelConfig:

857 ignore_pattern=["*.pt", "*.safetensors", "*.bin", "*.tensors"])

858 self.tokenizer = object_storage_tokenizer.dir

859

878 - def _init_multimodal_config(self) -> Optional["MultiModalConfig"]:

879 - if self._model_info.supports_multimodal:

880 - if (self.mm_encoder_tp_mode == "data" and

881 - not self._model_info.supports_multimodal_encoder_tp_data):

882 - logger.warning_once(

883 - "This model does not support `--mm-encoder-tp-mode data`. "

884 - "Falling back to `--mm-encoder-tp-mode weights`.")

885 - self.mm_encoder_tp_mode = "weights"

886 -

887 - return MultiModalConfig(

888 - limit_per_prompt=self.limit_mm_per_prompt,

889 - media_io_kwargs=self.media_io_kwargs,

890 - mm_processor_kwargs=self.mm_processor_kwargs,

891 - mm_processor_cache_gb=self.mm_processor_cache_gb,

892 - mm_processor_cache_type=self.mm_processor_cache_type,

893 - mm_shm_cache_max_object_size_mb=self.

894 - mm_shm_cache_max_object_size_mb,

895 - mm_encoder_tp_mode=self.mm_encoder_tp_mode,

896 - interleave_mm_strings=self.interleave_mm_strings,

897 - skip_mm_profiling=self.skip_mm_profiling,

898 - )

899 -

900 - return None

901 -

860 def _get_encoder_config(self):

861 return get_sentence_transformer_tokenizer_config(

862 self.model, self.revision)

@@ -2417,129 +2375,6 @@ class SpeculativeConfig:

2375 return f"SpeculativeConfig({method=}, {model=}, {num_spec_tokens=})"

2376

2377

2420 -@config

2421 -@dataclass

2422 -class MultiModalConfig:

2423 - """Controls the behavior of multimodal models."""

2424 -

2425 - limit_per_prompt: dict[str, int] = \

2426 - cast(dict[str, int], get_field(ModelConfig, "limit_mm_per_prompt"))

2427 - """

2428 - The maximum number of input items allowed per prompt for each modality.

2429 - Defaults to 1 (V0) or 999 (V1) for each modality.

2430 -

2431 - For example, to allow up to 16 images and 2 videos per prompt:

2432 - `{"image": 16, "video": 2}`

2433 - """

2434 -

2435 - media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)

2436 - """Additional args passed to process media inputs, keyed by modalities.

2437 - For example, to set num_frames for video, set

2438 - `--media-io-kwargs '{"video": {"num_frames": 40} }'` """

2439 -

2440 - mm_processor_kwargs: Optional[dict[str, object]] = None

2441 - """

2442 - Overrides for the multi-modal processor obtained from

2443 - `transformers.AutoProcessor.from_pretrained`.

2444 -

2445 - The available overrides depend on the model that is being run.

2446 -

2447 - For example, for Phi-3-Vision:

2448 - `{"num_crops": 4}`.

2449 - """

2450 -

2451 - mm_processor_cache_gb: float = 4

2452 - """

2453 - The size (in GiB) of the multi-modal processor cache, which is used to

2454 -

2455 - This cache is duplicated for each API process and engine core process,

2456 - resulting in a total memory usage of

2457 - `mm_processor_cache_gb * (api_server_count + data_parallel_size)`.

2458 -

2459 - Set to `0` to disable this cache completely (not recommended).

2460 - """

2461 -

2462 - mm_processor_cache_type: MMCacheType = "lru"

2463 - """Type of cache to use for the multi-modal preprocessor/mapper. If `shm`,

2464 - use shared memory FIFO cache. If `lru`, use mirrored LRU cache."""

2465 -

2466 - mm_shm_cache_max_object_size_mb: int = 128

2467 - """Size limit (in MiB) for each object stored in the multi-modal processor

2468 - shared memory cache. Only effective when `mm_processor_cache_type` is

2469 - `"shm"`."""

2470 -

2471 - mm_encoder_tp_mode: MMEncoderTPMode = "weights"

2472 - """

2473 - Indicates how to optimize multi-modal encoder inference using

2474 - tensor parallelism (TP).

2475 -

2476 - - `"weights"`: Within the same vLLM engine, split the weights of

2477 - each layer across TP ranks. (default TP behavior)

2478 - - `"data"`: Within the same vLLM engine, split the batched input data

2479 - across TP ranks to process the data in parallel, while hosting

2480 - the full weights on each TP rank.

2481 - This batch-level DP is not to be confused with API request-level

2482 - DP (which is controlled by `--data-parallel-size`).

2483 - This is only supported on a per-model basis and falls back to

2484 - `"weights"` if the encoder does not support DP.

2485 - """

2486 -

2487 - interleave_mm_strings: bool = False

2488 - """

2489 - Enable fully interleaved support for multimodal prompts.

2490 - """

2491 -

2492 - skip_mm_profiling: bool = False

2493 - """

2494 - When enabled, skips multimodal memory profiling and only profiles with

2495 - language backbone model during engine initialization.

2496 -

2497 - This reduces engine startup time but shifts the responsibility to users for

2498 - estimating the peak memory usage of the activation of multimodal encoder and

2499 - embedding cache.

2500 - """

2501 -

2502 - def compute_hash(self) -> str:

2503 - """

2504 - WARNING: Whenever a new field is added to this config,

2505 - ensure that it is included in the factors list if

2506 - it affects the computation graph.

2507 -

2508 - Provide a hash that uniquely identifies all the configs

2509 - that affect the structure of the computation

2510 - graph from input ids/embeddings to the final hidden states,

2511 - excluding anything before input ids/embeddings and after

2512 - the final hidden states.

2513 - """

2514 - # no factors to consider.

2515 - # this config will not affect the computation graph.

2516 - factors: list[Any] = []

2517 - hash_str = hashlib.md5(str(factors).encode(),

2518 - usedforsecurity=False).hexdigest()

2519 - return hash_str

2520 -

2521 - def get_limit_per_prompt(self, modality: str) -> int:

2522 - """

2523 - Get the maximum number of input items allowed per prompt

2524 - for the given modality.

2525 - """

2526 - return self.limit_per_prompt.get(

2527 - modality,

2528 - 999 if envs.VLLM_USE_V1 else 1,

2529 - )

2530 -

2531 - def merge_mm_processor_kwargs(

2532 - self,

2533 - inference_kwargs: Mapping[str, object],

2534 - ) -> dict[str, object]:

2535 - """

2536 - Get the keyword arguments to pass to the multi-modal processor

2537 - according to the extra arguments passed during inference.

2538 - """

2539 - kwargs = self.mm_processor_kwargs or {}

2540 - return kwargs | dict(inference_kwargs)

2541 -

2542 -

2378 @config

2379 @dataclass

2380 class PoolerConfig:

vllm/config/multimodal.py new

+120

@@ -0,0 +1,120 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

4 +import hashlib

5 +from collections.abc import Mapping

6 +from dataclasses import field

7 +from typing import Any, Literal, Optional

8 +

9 +from pydantic.dataclasses import dataclass

10 +

11 +import vllm.envs as envs

12 +from vllm.config.utils import config

13 +

14 +MMEncoderTPMode = Literal["weights", "data"]

15 +MMCacheType = Literal["shm", "lru"]

16 +

17 +

18 +@config

19 +@dataclass

20 +class MultiModalConfig:

21 + """Controls the behavior of multimodal models."""

22 +

23 + limit_per_prompt: dict[str, int] = field(default_factory=dict)

24 + """The maximum number of input items allowed per prompt for each modality.

25 + Defaults to 1 (V0) or 999 (V1) for each modality.

26 +

27 + For example, to allow up to 16 images and 2 videos per prompt:

28 + `{"image": 16, "video": 2}`"""

29 + media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)

30 + """Additional args passed to process media inputs, keyed by modalities.

31 + For example, to set num_frames for video, set

32 + `--media-io-kwargs '{"video": {"num_frames": 40} }'`"""

33 + mm_processor_kwargs: Optional[dict[str, object]] = None

34 + """Arguments to be forwarded to the model's processor for multi-modal data,

35 + e.g., image processor. Overrides for the multi-modal processor obtained

36 + from `transformers.AutoProcessor.from_pretrained`.

37 +

38 + The available overrides depend on the model that is being run.

39 +

40 + For example, for Phi-3-Vision:

41 + `{"num_crops": 4}`."""

42 + mm_processor_cache_gb: float = 4

43 + """The size (in GiB) of the multi-modal processor cache, which is used to

44 + avoid re-processing past multi-modal inputs.

45 +

46 + This cache is duplicated for each API process and engine core process,

47 + resulting in a total memory usage of

48 + `mm_processor_cache_gb * (api_server_count + data_parallel_size)`.

49 +

50 + Set to `0` to disable this cache completely (not recommended)."""

51 + mm_processor_cache_type: MMCacheType = "lru"

52 + """Type of cache to use for the multi-modal preprocessor/mapper. If `shm`,

53 + use shared memory FIFO cache. If `lru`, use mirrored LRU cache."""

54 + mm_shm_cache_max_object_size_mb: int = 128

55 + """Size limit (in MiB) for each object stored in the multi-modal processor

56 + shared memory cache. Only effective when `mm_processor_cache_type` is

57 + `"shm"`."""

58 + mm_encoder_tp_mode: MMEncoderTPMode = "weights"

59 + """Indicates how to optimize multi-modal encoder inference using tensor

60 + parallelism (TP).

61 +

62 + - `"weights"`: Within the same vLLM engine, split the weights of

63 + each layer across TP ranks. (default TP behavior)\n

64 + - `"data"`: Within the same vLLM engine, split the batched input data

65 + across TP ranks to process the data in parallel, while hosting

66 + the full weights on each TP rank.

67 + This batch-level DP is not to be confused with API request-level

68 + DP (which is controlled by `--data-parallel-size`).

69 + This is only supported on a per-model basis and falls back to

70 + `"weights"` if the encoder does not support DP."""

71 + interleave_mm_strings: bool = False

72 + """Enable fully interleaved support for multimodal prompts, while using

73 + --chat-template-content-format=string."""

74 + skip_mm_profiling: bool = False

75 + """When enabled, skips multimodal memory profiling and only profiles with

76 + language backbone model during engine initialization.

77 +

78 + This reduces engine startup time but shifts the responsibility to users for

79 + estimating the peak memory usage of the activation of multimodal encoder and

80 + embedding cache."""

81 +

82 + def compute_hash(self) -> str:

83 + """

84 + WARNING: Whenever a new field is added to this config,

85 + ensure that it is included in the factors list if

86 + it affects the computation graph.

87 +

88 + Provide a hash that uniquely identifies all the configs

89 + that affect the structure of the computation

90 + graph from input ids/embeddings to the final hidden states,

91 + excluding anything before input ids/embeddings and after

92 + the final hidden states.

93 + """

94 + # no factors to consider.

95 + # this config will not affect the computation graph.

96 + factors: list[Any] = []

97 + hash_str = hashlib.md5(str(factors).encode(),

98 + usedforsecurity=False).hexdigest()

99 + return hash_str

100 +

101 + def get_limit_per_prompt(self, modality: str) -> int:

102 + """

103 + Get the maximum number of input items allowed per prompt

104 + for the given modality.

105 + """

106 + return self.limit_per_prompt.get(

107 + modality,

108 + 999 if envs.VLLM_USE_V1 else 1,

109 + )

110 +

111 + def merge_mm_processor_kwargs(

112 + self,

113 + inference_kwargs: Mapping[str, object],

114 + ) -> dict[str, object]:

115 + """

116 + Get the keyword arguments to pass to the multi-modal processor

117 + according to the extra arguments passed during inference.

118 + """

119 + kwargs = self.mm_processor_kwargs or {}

120 + return kwargs | dict(inference_kwargs)

vllm/config/utils.py

+18

@@ -1,6 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 +from dataclasses import MISSING, Field, field, fields, is_dataclass

5 from typing import TYPE_CHECKING, TypeVar

7 if TYPE_CHECKING:

@@ -27,3 +28,20 @@ def config(cls: ConfigT) -> ConfigT:

28 script, which is invoked during the pre-commit checks.

29 """

30 return cls

31 +

32 +

33 +def get_field(cls: ConfigType, name: str) -> Field:

34 + """Get the default factory field of a dataclass by name. Used for getting

35 + default factory fields in `EngineArgs`."""

36 + if not is_dataclass(cls):

37 + raise TypeError("The given class is not a dataclass.")

38 + cls_fields = {f.name: f for f in fields(cls)}

39 + if name not in cls_fields:

40 + raise ValueError(f"Field '{name}' not found in {cls.__name__}.")

41 + named_field: Field = cls_fields[name]

42 + if (default_factory := named_field.default_factory) is not MISSING:

43 + return field(default_factory=default_factory)

44 + if (default := named_field.default) is not MISSING:

45 + return field(default=default)

46 + raise ValueError(

47 + f"{cls.__name__}.{name} must have a default value or default factory.")

vllm/engine/arg_utils.py

+8 -6

@@ -27,12 +27,14 @@ from vllm.config import (BlockSize, CacheConfig, CacheDType, CompilationConfig,

27 DistributedExecutorBackend, EPLBConfig,

28 GuidedDecodingBackend, HfOverrides, KVEventsConfig,

29 KVTransferConfig, LoadConfig, LogprobsMode,

30 - LoRAConfig, MambaDType, MMCacheType, MMEncoderTPMode,

31 - ModelConfig, ModelDType, ModelImpl, MultiModalConfig,

32 - ObservabilityConfig, ParallelConfig, PoolerConfig,

33 - PrefixCachingHashAlgo, RunnerOption, SchedulerConfig,

34 - SchedulerPolicy, SpeculativeConfig, TaskOption,

35 - TokenizerMode, VllmConfig, get_attr_docs, get_field)

30 + LoRAConfig, MambaDType, MMEncoderTPMode, ModelConfig,

31 + ModelDType, ModelImpl, ObservabilityConfig,

32 + ParallelConfig, PoolerConfig, PrefixCachingHashAlgo,

33 + RunnerOption, SchedulerConfig, SchedulerPolicy,

34 + SpeculativeConfig, TaskOption, TokenizerMode,

35 + VllmConfig, get_attr_docs)

36 +from vllm.config.multimodal import MMCacheType, MultiModalConfig

37 +from vllm.config.utils import get_field

38 from vllm.logger import init_logger

39 from vllm.platforms import CpuArchEnum, current_platform

40 from vllm.plugins import load_general_plugins

vllm/entrypoints/chat_utils.py

+6 -3

@@ -800,9 +800,10 @@ class MultiModalContentParser(BaseMultiModalContentParser):

800 super().__init__()

801

802 self._tracker = tracker

803 -

803 + multimodal_config = self._tracker.model_config.multimodal_config

804 + media_io_kwargs = getattr(multimodal_config, "media_io_kwargs", None)

805 self._connector = MediaConnector(

805 - media_io_kwargs=self._tracker._model_config.media_io_kwargs,

806 + media_io_kwargs=media_io_kwargs,

807 allowed_local_media_path=tracker.allowed_local_media_path,

808 )

809

@@ -883,8 +884,10 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):

884 super().__init__()

885

886 self._tracker = tracker

887 + multimodal_config = self._tracker.model_config.multimodal_config

888 + media_io_kwargs = getattr(multimodal_config, "media_io_kwargs", None)

889 self._connector = MediaConnector(

887 - media_io_kwargs=self._tracker._model_config.media_io_kwargs,

890 + media_io_kwargs=media_io_kwargs,

891 allowed_local_media_path=tracker.allowed_local_media_path,

892 )

893

vllm/model_executor/models/transformers.py

+4 -3

@@ -229,7 +229,8 @@ class MultiModalProcessingInfo(BaseProcessingInfo):

229 def get_max_image_tokens(self) -> int:

230 width, height = self.get_max_image_size()

231 processor = self.get_hf_processor()

232 - mm_processor_kwargs = self.ctx.model_config.mm_processor_kwargs or {}

232 + multimodal_config = self.ctx.model_config.multimodal_config

233 + mm_processor_kwargs = multimodal_config.mm_processor_kwargs or {}

234 mm_tokens = processor._get_num_multimodal_tokens(

235 image_sizes=([height, width], ), **mm_processor_kwargs)

236 image_tokens = mm_tokens["num_image_tokens"][0]

@@ -380,8 +381,8 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):

381 # Below tested on Llava. Prompts and `mm_token_type_ids` are always bs=1

382 mm_positions = torch.where(mm_token_type_ids == 1)[1]

383 images = mm_items.get_items("image", ImageProcessorItems)

383 - mm_processor_kwargs = (self.info.ctx.model_config.mm_processor_kwargs

384 - or {})

384 + multimodal_config = self.info.ctx.model_config.multimodal_config

385 + mm_processor_kwargs = multimodal_config.mm_processor_kwargs or {}

386 image_sizes = []

387 for item_idx in range(len(images)):

388 image_size = images.get_image_size(item_idx)

Move `MultiModalConfig` from `config/__init__.py` to `config/multimodal.py` (#24659)

Move `MultiModalConfig` from `config/init.py` to `config/multimodal.py` (#24659)