@tduncan / Nvidia-Optimization-Stack-VLLM / commits / c8824a1e42

[MM] Move `supports_multimodal_inputs` and cache out of registry (#57913)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Sep 21, 2026 at 22:43 UTC c8824a1e42bbd674751f808d657d58024c68fb0f

36 files changed +784 -707

tests/config/test_multimodal_config.py

+46 -54

@@ -1,12 +1,14 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 +import copy

5 from unittest.mock import MagicMock, patch

7 import pytest

8 import torch

9 from transformers import PretrainedConfig

11 +from tests.models.utils import build_model_context

12 from vllm.config.ec_transfer import ECRole, ECTransferConfig

13 from vllm.config.model import ModelConfig

14 from vllm.config.multimodal import MultiModalConfig

@@ -17,6 +19,26 @@ from vllm.transformers_utils.model_arch_config_convertor import (

19 from vllm.v1.attention.backends.registry import AttentionBackendEnum

22 +@pytest.mark.parametrize(

23 + "model_id,limit_mm_per_prompt,expected",

24 + [

25 + ("Qwen/Qwen2-0.5B-Instruct", {}, False),

26 + ("Qwen/Qwen2.5-VL-3B-Instruct", {}, True),

27 + ("Qwen/Qwen2.5-VL-3B-Instruct", {"image": 0, "video": 0}, False),

28 + ("Qwen/Qwen2.5-VL-3B-Instruct", {"image": 0}, True),

29 + ],

30 +)

31 +@pytest.mark.core_model

32 +def test_supports_multimodal_inputs(model_id, limit_mm_per_prompt, expected):

33 + """Test supports_multimodal_inputs returns correct boolean for various

34 + configs."""

35 + ctx = build_model_context(

36 + model_id,

37 + limit_mm_per_prompt=limit_mm_per_prompt,

38 + )

39 + assert ctx.model_config.supports_multimodal_inputs is expected

40 +

41 +

42 def test_mm_encoder_attn_backend_str_conversion():

43 config = MultiModalConfig(mm_encoder_attn_backend="FLASH_ATTN") # type: ignore[arg-type]

44 assert config.mm_encoder_attn_backend == AttentionBackendEnum.FLASH_ATTN

@@ -84,59 +106,33 @@ def test_mm_encoder_attn_dtype_hash_updates(tmp_path):

106 assert fp8_hash != fp8_static_hash

107

108

109 +_MULTIMODAL_MODEL = "llava-hf/llava-1.5-7b-hf"

110 +_TEXT_ONLY_MODEL = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"

111 +

112 +

113 def _make_mm_prefix_model_config(

114 + model: str = _MULTIMODAL_MODEL,

115 *,

116 language_model_only: bool = False,

117 ) -> ModelConfig:

91 - model_config = MagicMock(spec=ModelConfig)

92 - model_config.multimodal_config = MultiModalConfig(

93 - language_model_only=language_model_only

94 - )

95 - # Bind real helper methods onto the mock.

96 - model_config._supports_multimodal_for_mm_prefix = (

97 - ModelConfig._supports_multimodal_for_mm_prefix.__get__(

98 - model_config, ModelConfig

99 - )

100 - )

101 - return model_config

102 -

103 -

104 -@pytest.mark.parametrize("supports_mm", [True, False])

105 -def test_supports_multimodal_for_mm_prefix_uses_registry(supports_mm: bool):

106 - model_config = _make_mm_prefix_model_config()

107 -

108 - with patch(

109 - "vllm.multimodal.MULTIMODAL_REGISTRY.supports_multimodal_inputs",

110 - return_value=supports_mm,

111 - ) as mocked:

112 - assert model_config._supports_multimodal_for_mm_prefix() is supports_mm

113 - mocked.assert_called_once_with(model_config)

114 -

115 - # Sticky cache — registry must not be consulted again.

116 - with patch(

117 - "vllm.multimodal.MULTIMODAL_REGISTRY.supports_multimodal_inputs",

118 - side_effect=AssertionError("should use cache"),

119 - ):

120 - assert model_config._supports_multimodal_for_mm_prefix() is supports_mm

118 + return ModelConfig(model, language_model_only=language_model_only)

119

120

121 def test_supports_multimodal_for_mm_prefix_before_multimodal_config():

124 - model_config = _make_mm_prefix_model_config()

125 - model_config.multimodal_config = None

122 + """A text-only model never builds a multimodal config, so the early

123 + return must neither clear mm_prefix nor write the sticky cache."""

124 + model_config = _make_mm_prefix_model_config(_TEXT_ONLY_MODEL)

125 + assert model_config.multimodal_config is None

126

127 assert model_config._supports_multimodal_for_mm_prefix() is True

128 - assert not hasattr(model_config, "_supports_multimodal_inputs_cached")

128 + assert not getattr(model_config, "_supports_multimodal_inputs_cache", None)

129

130

131 def test_language_model_only_disables_via_supports_multimodal_inputs():

132 """language_model_only zeros all limits, so registry reports text-only."""

133 model_config = _make_mm_prefix_model_config(language_model_only=True)

134

135 - with patch(

136 - "vllm.multimodal.MULTIMODAL_REGISTRY.supports_multimodal_inputs",

137 - return_value=False,

138 - ):

139 - assert model_config._supports_multimodal_for_mm_prefix() is False

135 + assert model_config._supports_multimodal_for_mm_prefix() is False

136

137

138 def test_convertor_clears_mm_prefix_when_multimodal_disabled():

@@ -159,23 +155,19 @@ def test_convertor_clears_mm_prefix_when_multimodal_disabled():

155 def test_sticky_cache_survives_text_subconfig_regeneration():

156 """with_hf_config deepcopies the cached decision onto text submodules."""

157 model_config = _make_mm_prefix_model_config()

162 - with patch(

163 - "vllm.multimodal.MULTIMODAL_REGISTRY.supports_multimodal_inputs",

164 - return_value=False,

165 - ):

166 - assert model_config._supports_multimodal_for_mm_prefix() is False

158 + assert model_config._supports_multimodal_for_mm_prefix() is True

159

168 - # Simulate deepcopy onto a Gemma4ForCausalLM-like config that would

169 - # otherwise fail registry lookup / return False incorrectly.

170 - text_config = _make_mm_prefix_model_config()

171 - text_config._supports_multimodal_inputs_cached = (

172 - model_config._supports_multimodal_inputs_cached

173 - )

174 - with patch(

175 - "vllm.multimodal.MULTIMODAL_REGISTRY.supports_multimodal_inputs",

176 - side_effect=AssertionError("must not re-query registry"),

177 - ):

178 - assert text_config._supports_multimodal_for_mm_prefix() is False

160 + # `with_hf_config` deep-copies this config and swaps `hf_config` for a

161 + # text-only submodule (e.g. Gemma4ForCausalLM).

162 + text_config = copy.deepcopy(model_config)

163 + text_config.hf_config = model_config.hf_text_config

164 + assert text_config._supports_multimodal_for_mm_prefix() is True

165 +

166 + # Without the copied cache the submodule architecture has no registered

167 + # multimodal processor, so re-querying the registry wrongly settles on

168 + # text-only and would clear mm_prefix.

169 + del text_config._supports_multimodal_inputs_cache

170 + assert text_config._supports_multimodal_for_mm_prefix() is False

171

172

173 @pytest.mark.parametrize(

tests/entrypoints/generate/generative_scoring/test_generative_scoring.py

+4 -1

@@ -60,7 +60,10 @@ class MockModelConfig:

60 encoder_config = None

61 generation_config: str = "auto"

62 media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)

63 - skip_tokenizer_init = False

63 + skip_tokenizer_init: bool = False

64 + is_encoder_decoder: bool = False

65 + is_multimodal_model: bool = False

66 + supports_multimodal_inputs: bool = False

67 vocab_size = 151936

69 def get_diff_sampling_param(self):

tests/entrypoints/openai/chat_completion/test_chat_error.py

+2 -1

@@ -56,9 +56,10 @@ class MockModelConfig:

56 encoder_config = None

57 generation_config: str = "auto"

58 media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)

59 - skip_tokenizer_init = False

59 + skip_tokenizer_init: bool = False

60 is_encoder_decoder: bool = False

61 is_multimodal_model: bool = False

62 + supports_multimodal_inputs: bool = False

63 renderer_num_workers: int = 1

65 def get_diff_sampling_param(self):

tests/entrypoints/openai/chat_completion/test_serving_chat.py

@@ -556,6 +556,7 @@ class MockModelConfig:

556 skip_tokenizer_init: bool = False

557 is_encoder_decoder: bool = False

558 is_multimodal_model: bool = False

559 + supports_multimodal_inputs: bool = False

560 renderer_num_workers: int = 1

561 enable_prompt_embeds: bool = False

562

tests/entrypoints/openai/completion/test_completion_error.py

+2 -1

@@ -61,9 +61,10 @@ class MockModelConfig:

61 encoder_config = None

62 generation_config: str = "auto"

63 media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)

64 - skip_tokenizer_init = False

64 + skip_tokenizer_init: bool = False

65 is_encoder_decoder: bool = False

66 is_multimodal_model: bool = False

67 + supports_multimodal_inputs: bool = False

68 renderer_num_workers: int = 1

70 def get_diff_sampling_param(self):

tests/entrypoints/openai/completion/test_lora_resolvers.py

@@ -54,6 +54,7 @@ class MockModelConfig:

54 skip_tokenizer_init: bool = False

55 is_encoder_decoder: bool = False

56 is_multimodal_model: bool = False

57 + supports_multimodal_inputs: bool = False

58 renderer_num_workers: int = 1

60 def get_diff_sampling_param(self):

tests/entrypoints/openai/test_render_parity.py

+2 -1

@@ -86,9 +86,10 @@ class MockModelConfig:

86 generation_config: str = "auto"

87 override_generation_config: dict[str, Any] = field(default_factory=dict)

88 media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)

89 - skip_tokenizer_init = False

89 + skip_tokenizer_init: bool = False

90 is_encoder_decoder: bool = False

91 is_multimodal_model: bool = False

92 + supports_multimodal_inputs: bool = False

93 renderer_num_workers: int = 1

94 enable_prompt_embeds: bool = False

tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py

+2 -1

@@ -61,9 +61,10 @@ class MockModelConfig:

61 encoder_config = None

62 generation_config: str = "auto"

63 media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)

64 - skip_tokenizer_init = False

64 + skip_tokenizer_init: bool = False

65 is_encoder_decoder: bool = False

66 is_multimodal_model: bool = False

67 + supports_multimodal_inputs: bool = False

68 renderer_num_workers: int = 1

70 def get_diff_sampling_param(self):

tests/entrypoints/serve/tokenize/test_serving_tokenization.py

+2 -1

@@ -56,9 +56,10 @@ class MockModelConfig:

56 encoder_config = None

57 generation_config: str = "auto"

58 media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)

59 - skip_tokenizer_init = False

59 + skip_tokenizer_init: bool = False

60 is_encoder_decoder: bool = False

61 is_multimodal_model: bool = False

62 + supports_multimodal_inputs: bool = False

63 renderer_num_workers: int = 1

65 def get_diff_sampling_param(self):

tests/models/multimodal/processing/test_audio_in_video.py

+2 -2

@@ -22,7 +22,7 @@ import numpy as np

22 import pytest

24 from vllm.multimodal import MULTIMODAL_REGISTRY

25 -from vllm.multimodal.cache import MultiModalProcessorSenderCache

25 +from vllm.multimodal.cache import LruKeyReplicatedSenderCache

27 from ....multimodal.utils import random_audio, random_video

28 from ...utils import build_model_context

@@ -68,7 +68,7 @@ def test_audio_in_video_cache_correctness(model_id: str, num_videos: int) -> Non

68 # Sender cache: on a cache hit returns (None, prompt_updates) for each

69 # item, setting mm_kwargs["video"] = [None] – the exact condition that

70 # triggered the original bug.

71 - sender_cache = MultiModalProcessorSenderCache(ctx.model_config)

71 + sender_cache = LruKeyReplicatedSenderCache(ctx.model_config)

73 video_token_id = processor.info.get_hf_config().video_token_id

tests/multimodal/test_cache.py

+26 -23

@@ -10,20 +10,23 @@ import torch

11 from vllm.config import ModelConfig, ParallelConfig, VllmConfig

12 from vllm.config.multimodal import MultiModalConfig

13 -from vllm.multimodal import MULTIMODAL_REGISTRY

13 from vllm.multimodal.cache import (

14 BaseMultiModalProcessorCache,

15 BaseMultiModalReceiverCache,

17 - MultiModalCache,

16 + LruKeyReplicatedReceiverCache,

17 + LruKeyReplicatedSenderCache,

18 MultiModalCacheMissError,

19 - MultiModalProcessorCacheInItem,

20 - MultiModalProcessorCacheItem,

21 - MultiModalProcessorCacheItemMetadata,

19 MultiModalProcessorOnlyCache,

23 - MultiModalProcessorSenderCache,

24 - MultiModalReceiverCache,

20 ShmObjectStoreReceiverCache,

21 ShmObjectStoreSenderCache,

22 + engine_receiver_cache_from_config,

23 + processor_cache_from_config,

24 +)

25 +from vllm.multimodal.cache.base import (

26 + MultiModalCache,

27 + MultiModalProcessorCacheInItem,

28 + MultiModalProcessorCacheItem,

29 + MultiModalProcessorCacheItemMetadata,

30 )

31 from vllm.multimodal.hasher import MultiModalHasher

32 from vllm.multimodal.inputs import (

@@ -137,10 +140,10 @@ def _compare_caches(

140 n_iter: int = 100,

141 seed: int = 0,

142 ):

140 - cache_0_p0 = MULTIMODAL_REGISTRY.processor_cache_from_config(config_0)

141 - cache_0_p1 = MULTIMODAL_REGISTRY.engine_receiver_cache_from_config(config_0)

142 - cache_1_p0 = MULTIMODAL_REGISTRY.processor_cache_from_config(config_1)

143 - cache_1_p1 = MULTIMODAL_REGISTRY.engine_receiver_cache_from_config(config_1)

143 + cache_0_p0 = processor_cache_from_config(config_0)

144 + cache_0_p1 = engine_receiver_cache_from_config(config_0)

145 + cache_1_p0 = processor_cache_from_config(config_1)

146 + cache_1_p1 = engine_receiver_cache_from_config(config_1)

147

148 cache_size_gb = max(

149 config_0.model_config.multimodal_config.mm_processor_cache_gb,

@@ -275,8 +278,8 @@ def test_oversized_item_is_served_uncached():

278 assert p0_only.get_and_update_item((small, []), "small")[0] is small

279 assert p0_only.is_cached_item("small")

280

278 - p0 = MultiModalProcessorSenderCache(model_config) # type: ignore[arg-type]

279 - p1 = MultiModalReceiverCache(model_config) # type: ignore[arg-type]

281 + p0 = LruKeyReplicatedSenderCache(model_config) # type: ignore[arg-type]

282 + p1 = LruKeyReplicatedReceiverCache(model_config) # type: ignore[arg-type]

283 assert p0.get_and_update_item((item, []), "big")[0] is item

284 assert not p0.is_cached_item("big")

285 assert p1.get_and_update_item(item, "big") is item

@@ -296,8 +299,8 @@ def test_mm_cache_miss_raises_and_recovers():

299 model or network.

300 """

301 model_config = _StubModelConfig(mm_processor_cache_gb=1)

299 - p0 = MultiModalProcessorSenderCache(model_config) # type: ignore[arg-type]

300 - p1 = MultiModalReceiverCache(model_config) # type: ignore[arg-type]

302 + p0 = LruKeyReplicatedSenderCache(model_config) # type: ignore[arg-type]

303 + p1 = LruKeyReplicatedReceiverCache(model_config) # type: ignore[arg-type]

304

305 mm_hash = "image_A"

306 item = MultiModalKwargsItem.dummy(nbytes=64)

@@ -338,7 +341,7 @@ def test_receiver_cache_replaces_stale_item_when_payload_resent():

341 and kills EngineCore. Prefer the fresh payload and keep it for later hits.

342 """

343 model_config = _StubModelConfig(mm_processor_cache_gb=1)

341 - p1 = MultiModalReceiverCache(model_config) # type: ignore[arg-type]

344 + p1 = LruKeyReplicatedReceiverCache(model_config) # type: ignore[arg-type]

345 small = MultiModalKwargsItem.dummy(nbytes=64)

346 large = MultiModalKwargsItem.dummy(nbytes=256)

347 mm_hash = "shared-id"

@@ -351,7 +354,7 @@ def test_receiver_cache_replaces_stale_item_when_payload_resent():

354 def test_receiver_cache_features_keep_resent_payload():

355 """EngineCore updates features through get_and_update_features."""

356 model_config = _StubModelConfig(mm_processor_cache_gb=1)

354 - p1 = MultiModalReceiverCache(model_config) # type: ignore[arg-type]

357 + p1 = LruKeyReplicatedReceiverCache(model_config) # type: ignore[arg-type]

358 small = MultiModalKwargsItem.dummy(nbytes=64)

359 large = MultiModalKwargsItem.dummy(nbytes=256)

360 mm_hash = "shared-id"

@@ -384,7 +387,7 @@ def test_mm_cache_miss_batches_all_drifted_hashes():

387 non-drifted items in the same request are still ingested.

388 """

389 model_config = _StubModelConfig(mm_processor_cache_gb=1)

387 - p1 = MultiModalReceiverCache(model_config) # type: ignore[arg-type]

390 + p1 = LruKeyReplicatedReceiverCache(model_config) # type: ignore[arg-type]

391 item = MultiModalKwargsItem.dummy(nbytes=64)

392

393 def _feature(

@@ -582,8 +585,8 @@ def test_cache_eviction_lru_cache():

585 model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf",

586 mm_processor_cache_gb=6 / GiB_bytes,

587 )

585 - sender_cache = MultiModalProcessorSenderCache(model_config)

586 - receiver_cache = MultiModalReceiverCache(model_config)

588 + sender_cache = LruKeyReplicatedSenderCache(model_config)

589 + receiver_cache = LruKeyReplicatedReceiverCache(model_config)

590

591 _run_test_cache_eviction_lru(sender_cache, receiver_cache, base_item_size=1)

592

@@ -775,7 +778,7 @@ def test_processor_cache_shared_across_loras():

778 model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf",

779 mm_processor_cache_gb=1,

780 )

778 - receiver_cache = MultiModalReceiverCache(model_config)

781 + receiver_cache = LruKeyReplicatedReceiverCache(model_config)

782

783 base_mm_hash = "image_hash_abc123"

784 lora_a_identifier = f"12345:{base_mm_hash}"

@@ -827,8 +830,8 @@ async def test_release_kv_cache_resends_mm_payload(use_async, release_error):

830 )

831 model_config = ctx.model_config

832

830 - sender = MultiModalProcessorSenderCache(model_config)

831 - receiver = MultiModalReceiverCache(model_config)

833 + sender = LruKeyReplicatedSenderCache(model_config)

834 + receiver = LruKeyReplicatedReceiverCache(model_config)

835 item = _dummy_item({"pixel_values": 16})

836 mm_hash = "image_A"

837 payload, _ = sender.get_and_update_item((item, []), mm_hash)

tests/multimodal/test_registry.py

-22

@@ -10,31 +10,9 @@ import pytest

11 from vllm.multimodal import MULTIMODAL_REGISTRY

13 -from ..models.utils import build_model_context

14 -

13 pytestmark = pytest.mark.cpu_test

18 -@pytest.mark.parametrize(

19 - "model_id,limit_mm_per_prompt,expected",

20 - [

21 - ("Qwen/Qwen2-0.5B-Instruct", {}, False),

22 - ("Qwen/Qwen2.5-VL-3B-Instruct", {}, True),

23 - ("Qwen/Qwen2.5-VL-3B-Instruct", {"image": 0, "video": 0}, False),

24 - ("Qwen/Qwen2.5-VL-3B-Instruct", {"image": 0}, True),

25 - ],

26 -)

27 -@pytest.mark.core_model

28 -def test_supports_multimodal_inputs(model_id, limit_mm_per_prompt, expected):

29 - """Test supports_multimodal_inputs returns correct boolean for various

30 - configs."""

31 - ctx = build_model_context(

32 - model_id,

33 - limit_mm_per_prompt=limit_mm_per_prompt,

34 - )

35 - assert MULTIMODAL_REGISTRY.supports_multimodal_inputs(ctx.model_config) is expected

36 -

37 -

16 def test_create_processor_error_uses_served_model_name():

17 model_config = SimpleNamespace(

18 is_multimodal_model=False,

tests/renderers/test_cohere.py

@@ -997,6 +997,7 @@ class _MockModelConfig:

997 skip_tokenizer_init: bool = True

998 is_encoder_decoder: bool = False

999 is_multimodal_model: bool = False

1000 + supports_multimodal_inputs: bool = False

1001 renderer_num_workers: int = 1

1002

1003

tests/renderers/test_completions.py

@@ -39,6 +39,7 @@ class MockModelConfig:

39 skip_tokenizer_init: bool = False

40 is_encoder_decoder: bool = False

41 is_multimodal_model: bool = False

42 + supports_multimodal_inputs: bool = False

43 renderer_num_workers: int = 1

44 hidden_size: int = 768

45 dtype: torch.dtype = torch.float32

tests/renderers/test_kimi_k3.py

@@ -40,6 +40,7 @@ class MockHFConfig:

40 class MockModelConfig:

41 runner_type: str = "generate"

42 is_multimodal_model: bool = False

43 + supports_multimodal_inputs: bool = False

44 multimodal_config: Any = None

45 hf_config: MockHFConfig = field(default_factory=MockHFConfig)

46 allowed_local_media_path: str = ""

tests/renderers/test_mistral.py

@@ -37,6 +37,7 @@ class MockModelConfig:

37 skip_tokenizer_init: bool = False

38 is_encoder_decoder: bool = False

39 is_multimodal_model: bool = False

40 + supports_multimodal_inputs: bool = False

41 renderer_num_workers: int = 1

tests/v1/kv_connector/unit/test_handshake_pp_aggregation.py

+5 -6

@@ -79,11 +79,6 @@ def _run_engine_core_handshake(

79 "resolve_kv_cache_block_sizes",

80 lambda kv_cache_config, vllm_config: (16, 16),

81 )

82 - monkeypatch.setattr(

83 - engine_core_module,

84 - "MULTIMODAL_REGISTRY",

85 - SimpleNamespace(engine_receiver_cache_from_config=lambda vllm_config: None),

86 - )

82 monkeypatch.setattr(engine_core_module, "freeze_gc_heap", lambda: None)

83 monkeypatch.setattr(

84 engine_core_module, "maybe_attach_gc_debug_callback", lambda: None

@@ -105,7 +100,11 @@ def _run_engine_core_handshake(

100 speculative_config=None,

101 ec_transfer_config=None,

102 max_concurrent_batches=1,

108 - model_config=SimpleNamespace(runner_type="generate", is_diffusion=False),

103 + model_config=SimpleNamespace(

104 + runner_type="generate",

105 + is_diffusion=False,

106 + supports_multimodal_inputs=False,

107 + ),

108 cache_config=SimpleNamespace(

109 enable_prefix_caching=False,

110 prefix_caching_hash_algo="builtin",

tests/v1/streaming_input/test_scheduler_streaming.py

@@ -53,6 +53,7 @@ def create_scheduler() -> Scheduler:

53 vllm_config.model_config = MagicMock()

54 vllm_config.model_config.skip_tokenizer_init = True

55 vllm_config.model_config.is_multimodal_model = False

56 + vllm_config.model_config.supports_multimodal_inputs = False

57 vllm_config.model_config.is_encoder_decoder = False

58 vllm_config.model_config.is_diffusion = False

59 vllm_config.model_config.max_model_len = 1024

tests/v1/worker/test_gpu_autoregressive_speculator.py

+19 -31

@@ -148,8 +148,16 @@ def test_speculator_uses_draft_model_hidden_size(monkeypatch, hc_mult, expected)

148

149

150 def test_mm_support_configured_after_model_load(monkeypatch):

151 - target_model_config = object()

152 - draft_model_config = object()

151 + checked_configs = []

152 +

153 + class _ModelConfig:

154 + @property

155 + def supports_multimodal_inputs(self):

156 + checked_configs.append(self)

157 + return True

158 +

159 + target_model_config = _ModelConfig()

160 + draft_model_config = _ModelConfig()

161 vllm_config = SimpleNamespace(model_config=target_model_config)

162 draft_model = _MultimodalDraftModel()

163

@@ -164,19 +172,8 @@ def test_mm_support_configured_after_model_load(monkeypatch):

172 speculator.supports_mm_inputs = False

173 speculator.use_acceptance_estimator = False

174

167 - checked_configs = []

168 -

169 - def supports_multimodal_inputs(model_config):

170 - checked_configs.append(model_config)

171 - return True

172 -

175 monkeypatch.setattr(DraftModelSpeculator, "__init__", init_base)

176 _mock_base_model_load(monkeypatch)

175 - monkeypatch.setattr(

176 - base_spec_module.MULTIMODAL_REGISTRY,

177 - "supports_multimodal_inputs",

178 - supports_multimodal_inputs,

179 - )

177

178 speculator = _TestSpeculator(vllm_config, torch.device("cpu"))

179

@@ -198,7 +195,9 @@ def test_load_model_keeps_mm_support_for_capable_drafter(monkeypatch):

195 speculator.supports_mm_inputs = False

196 speculator.inputs_embeds = None

197 speculator.use_acceptance_estimator = False

201 - speculator.vllm_config = SimpleNamespace(model_config=object())

198 + speculator.vllm_config = SimpleNamespace(

199 + model_config=SimpleNamespace(supports_multimodal_inputs=True)

200 + )

201 speculator.max_num_tokens = 4

202 speculator.hidden_size = 3

203 speculator.dtype = torch.float32

@@ -206,11 +205,6 @@ def test_load_model_keeps_mm_support_for_capable_drafter(monkeypatch):

205 draft_model = _MultimodalDraftModel()

206 speculator.test_draft_model = draft_model

207 _mock_base_model_load(monkeypatch)

209 - monkeypatch.setattr(

210 - base_spec_module.MULTIMODAL_REGISTRY,

211 - "supports_multimodal_inputs",

212 - lambda model_config: True,

213 - )

208

209 speculator.load_model(torch.nn.Module())

210

@@ -223,16 +217,13 @@ def test_load_model_disables_mm_support_for_text_only_drafter(monkeypatch):

217 speculator.supports_mm_inputs = False

218 speculator.inputs_embeds = None

219 speculator.use_acceptance_estimator = False

226 - speculator.vllm_config = SimpleNamespace(model_config=object())

220 + speculator.vllm_config = SimpleNamespace(

221 + model_config=SimpleNamespace(supports_multimodal_inputs=True)

222 + )

223 draft_model = _TextOnlyDraftModel()

224 speculator.test_draft_model = draft_model

225 warning_messages = []

226 _mock_base_model_load(monkeypatch)

231 - monkeypatch.setattr(

232 - base_spec_module.MULTIMODAL_REGISTRY,

233 - "supports_multimodal_inputs",

234 - lambda model_config: True,

235 - )

227 monkeypatch.setattr(

228 base_spec_module.logger,

229 "warning_once",

@@ -257,7 +248,9 @@ def test_multi_module_mm_support_configured_after_model_load(monkeypatch):

248 speculator.inputs_embeds = None

249 speculator.use_acceptance_estimator = False

250 speculator.cached_draft_input_embeds = None

260 - speculator.vllm_config = SimpleNamespace(model_config=object())

251 + speculator.vllm_config = SimpleNamespace(

252 + model_config=SimpleNamespace(supports_multimodal_inputs=True)

253 + )

254 speculator.max_num_tokens = 4

255 speculator.max_num_reqs = 2

256 speculator.num_speculative_steps = 3

@@ -271,11 +264,6 @@ def test_multi_module_mm_support_configured_after_model_load(monkeypatch):

264 "load_draft_model",

265 lambda self, target_model, target_attn_layer_names: draft_model,

266 )

274 - monkeypatch.setattr(

275 - base_spec_module.MULTIMODAL_REGISTRY,

276 - "supports_multimodal_inputs",

277 - lambda model_config: True,

278 - )

267

268 speculator.load_model(torch.nn.Module())

269

vllm/config/model.py

+80 -8

@@ -865,6 +865,80 @@ class ModelConfig:

865 self._verify_quantization()

866 self._verify_cuda_graph()

867

868 + def _supports_multimodal_inputs(self) -> bool:

869 + """Checks if the model supports multimodal inputs.

870 + Returns True if the model is multimodal with any non-zero supported

871 + modalities, otherwise returns False, effectively running in

872 + text-only mode.

873 + """

874 + if not self.is_multimodal_model:

875 + return False

876 +

877 + from vllm.multimodal import MULTIMODAL_REGISTRY

878 +

879 + mm_config = self.get_multimodal_config()

880 + try:

881 + info = MULTIMODAL_REGISTRY.get_processing_info(self)

882 + except ValueError:

883 + # Speculative drafters for multimodal targets (e.g. Qwen3_5MTP,

884 + # Exaone4_5_MTP, MiMoV2OmniMTP) declare `SupportsMultiModal` so

885 + # that they can consume the embeddings merged by the target model,

886 + # but they never run a multi-modal processor of their own. Running

887 + # in text-only mode is the expected outcome for them, not a

888 + # misconfiguration worth warning about.

889 + if self.runner_type != "draft":

890 + logger.warning_once(

891 + "Model %s is treated as multimodal but has no registered "

892 + "multimodal processor; running in text-only mode.",

893 + self.model,

894 + )

895 + return False

896 +

897 + # Check if all supported modalities have limit == 0

898 + if all(

899 + mm_config.get_limit_per_prompt(modality) == 0

900 + for modality in info.supported_mm_limits

901 + ):

902 + # If enable_mm_embeds is True, we still need MM infrastructure

903 + # to process pre-computed embeddings even though encoder won't run

904 + if mm_config.enable_mm_embeds:

905 + return True

906 +

907 + logger.info_once(

908 + "All limits of multimodal modalities supported by the model "

909 + "are set to 0, running in text-only mode."

910 + )

911 + return False

912 +

913 + return True

914 +

915 + def _cached_supports_multimodal_inputs(self) -> bool:

916 + cache = getattr(self, "_supports_multimodal_inputs_cache", None)

917 + if cache is None:

918 + self._supports_multimodal_inputs_cache = cache = {}

919 +

920 + mm_config = self.multimodal_config

921 + if mm_config is None:

922 + mm_cache_key = None

923 + else:

924 + limits_per_prompt = {

925 + modality: mm_config.get_limit_per_prompt(modality)

926 + for modality in mm_config.limit_per_prompt

927 + }

928 + mm_cache_key = (

929 + mm_config.language_model_only,

930 + tuple(limits_per_prompt.items()),

931 + mm_config.enable_mm_embeds,

932 + )

933 +

934 + cache_key = (self.is_multimodal_model, self.runner_type, mm_cache_key)

935 + if cache_key in cache:

936 + return cache[cache_key]

937 +

938 + supports_mm = self._supports_multimodal_inputs()

939 + cache[cache_key] = supports_mm

940 + return supports_mm

941 +

942 def _supports_multimodal_for_mm_prefix(self) -> bool:

943 """Whether multimodal inputs can still appear for this deployment.

944

@@ -881,24 +955,18 @@ class ModelConfig:

955 vision modality is still enabled (e.g. ``image=0`` but video allowed).

956 The deep-copied cache preserves the top-level decision instead.

957 """

884 - cached = getattr(self, "_supports_multimodal_inputs_cached", None)

885 - if cached is not None:

886 - return cached

887 -

958 if self.multimodal_config is None:

959 # Early call before multimodal init — do not clear mm_prefix yet.

960 return True

961

892 - from vllm.multimodal import MULTIMODAL_REGISTRY

893 -

894 - supports_mm = MULTIMODAL_REGISTRY.supports_multimodal_inputs(self)

895 - self._supports_multimodal_inputs_cached = supports_mm

962 + supports_mm = self._cached_supports_multimodal_inputs()

963 if not supports_mm:

964 logger.info_once(

965 "Disabled mm_prefix attention mode because multimodal inputs "

966 "are configuration-disabled. Attention backends without "

967 "mm_prefix support may now be selected."

968 )

969 +

970 return supports_mm

971

972 def get_model_arch_config(self) -> ModelArchitectureConfig:

@@ -1859,6 +1927,10 @@ class ModelConfig:

1927 def is_multimodal_raw_input_only_model(self) -> bool:

1928 return self._model_info.supports_multimodal_raw_input_only

1929

1930 + @property

1931 + def supports_multimodal_inputs(self) -> bool:

1932 + return self._cached_supports_multimodal_inputs()

1933 +

1934 @property

1935 def requires_raw_input_tokens(self) -> bool:

1936 return self._model_info.requires_raw_input_tokens

vllm/multimodal/cache/__init__.py new

+38

@@ -0,0 +1,38 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

4 +from .base import (

5 + BaseMultiModalProcessorCache,

6 + BaseMultiModalReceiverCache,

7 + MultiModalCacheMissError,

8 + MultiModalProcessorOnlyCache,

9 +)

10 +from .factories import (

11 + engine_receiver_cache_from_config,

12 + processor_cache_from_config,

13 + processor_only_cache_from_config,

14 + worker_receiver_cache_from_config,

15 +)

16 +from .lru import (

17 + LruKeyReplicatedReceiverCache,

18 + LruKeyReplicatedSenderCache,

19 +)

20 +from .shm import (

21 + ShmObjectStoreReceiverCache,

22 + ShmObjectStoreSenderCache,

23 +)

24 +

25 +__all__ = [

26 + "MultiModalCacheMissError",

27 + "BaseMultiModalProcessorCache",

28 + "BaseMultiModalReceiverCache",

29 + "MultiModalProcessorOnlyCache",

30 + "LruKeyReplicatedSenderCache",

31 + "LruKeyReplicatedReceiverCache",

32 + "ShmObjectStoreReceiverCache",

33 + "ShmObjectStoreSenderCache",

34 + "processor_cache_from_config",

35 + "processor_only_cache_from_config",

36 + "engine_receiver_cache_from_config",

37 + "worker_receiver_cache_from_config",

38 +]

vllm/multimodal/cache/base.py renamed

+6 -361

@@ -4,26 +4,19 @@ import operator

4 import sys

5 from abc import ABC, abstractmethod

6 from collections.abc import Mapping, Sequence

7 -from multiprocessing.synchronize import Lock as LockType

8 -from typing import TYPE_CHECKING, Generic, TypeAlias, TypeVar, cast

7 +from typing import TYPE_CHECKING, Generic, TypeAlias, TypeVar

9 import torch

10 from typing_extensions import override

13 -import vllm.envs as envs

14 -from vllm.distributed.device_communicators.shm_object_storage import (

15 - MsgpackSerde,

16 - SingleWriterShmObjectStorage,

17 - SingleWriterShmRingBuffer,

18 -)

12 +from vllm.config import ModelConfig

13 from vllm.logger import init_logger

14 from vllm.utils.cache import CacheInfo, LRUCache

15 from vllm.utils.jsontree import json_count_leaves, json_map_leaves, json_reduce_leaves

22 -from vllm.utils.mem_constants import GiB_bytes, MiB_bytes

16 +from vllm.utils.mem_constants import GiB_bytes

17 from vllm.utils.mem_utils import format_gib

25 -from .inputs import (

26 - MultiModalBatchedField,

19 +from ..inputs import (

20 MultiModalFeatureSpec,

21 MultiModalFieldElem,

22 MultiModalKwargsItem,

@@ -32,9 +25,7 @@ from .inputs import (

25 )

27 if TYPE_CHECKING:

35 - from vllm.config import ModelConfig, VllmConfig

36 -

37 - from .processing.processor import ResolvedPromptUpdate

28 + from ..processing.processor import ResolvedPromptUpdate

30 logger = init_logger(__name__)

@@ -399,7 +390,7 @@ class MultiModalProcessorOnlyCache(BaseMultiModalProcessorCache):

390 tensor data and metadata) into the cache, and return the input.

391 """

392

402 - def __init__(self, model_config: "ModelConfig") -> None:

393 + def __init__(self, model_config: ModelConfig) -> None:

394 super().__init__()

395

396 mm_config = model_config.get_multimodal_config()

@@ -440,216 +431,6 @@ class MultiModalProcessorOnlyCache(BaseMultiModalProcessorCache):

431 return self._cache.stat(delta=delta)

432

433

443 -class MultiModalProcessorSenderCache(BaseMultiModalProcessorCache):

444 - """The cache which is used on P0 when IPC caching is enabled.

445 -

446 - How to update each item:

447 -

448 - - If the item is already in the cache, clear the input to avoid

449 - unnecessary IPC.

450 -

451 - - If the item is not in the cache, store the metadata of that item so

452 - that the eviction policy remains the same as the cache on P1,

453 - and return the input.

454 - By only storing the metadata, we avoid keeping the data itself in

455 - memory inside P0.

456 - """

457 -

458 - def __init__(self, model_config: "ModelConfig") -> None:

459 - super().__init__()

460 -

461 - mm_config = model_config.get_multimodal_config()

462 -

463 - self._cache = MultiModalCache.get_lru_cache(

464 - mm_config.mm_processor_cache_gb,

465 - MultiModalProcessorCacheItemMetadata,

466 - )

467 -

468 - @override

469 - def is_cached_item(self, mm_hash: str) -> bool:

470 - return mm_hash in self._cache

471 -

472 - @override

473 - def get_and_update_item(

474 - self,

475 - mm_item: MultiModalProcessorCacheInItem,

476 - mm_hash: str,

477 - ) -> MultiModalProcessorCacheOutItem:

478 - if (cached_item := self._cache.get(mm_hash)) is not None:

479 - return None, cached_item.prompt_updates

480 -

481 - assert mm_item is not None, f"Expected a cached item for {mm_hash=}"

482 -

483 - self.cache_if_fits(

484 - self._cache, mm_hash, MultiModalProcessorCacheItemMetadata(*mm_item)

485 - )

486 - return mm_item

487 -

488 - @override

489 - def touch_sender_cache_item(self, mm_hash: str) -> None:

490 - self._cache.touch(mm_hash)

491 -

492 - @override

493 - def clear_cache(self) -> None:

494 - self._cache.clear()

495 -

496 - @override

497 - def make_stats(self, *, delta: bool = False) -> CacheInfo:

498 - return self._cache.stat(delta=delta)

499 -

500 - @override

501 - def invalidate(self, mm_hash: str) -> None:

502 - # Drop our stale shadow entry so the next request for this hash re-sends the

503 - # data and repopulates P1 (see MultiModalCacheMissError).

504 - self._cache.pop(mm_hash, None)

505 -

506 -

507 -class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):

508 - """The cache which is used on P0 when IPC caching is enabled.

509 -

510 - How to update each item:

511 -

512 - - If the item is already in the cache, clear the input to avoid

513 - unnecessary IPC.

514 -

515 - - If the item is not in the cache, store the data in shared memory.

516 - """

517 -

518 - def __init__(self, vllm_config: "VllmConfig") -> None:

519 - super().__init__()

520 -

521 - self.world_size = vllm_config.parallel_config.world_size

522 - mm_config = vllm_config.model_config.get_multimodal_config()

523 -

524 - ring_buffer = SingleWriterShmRingBuffer(

525 - data_buffer_size=int(mm_config.mm_processor_cache_gb * GiB_bytes),

526 - name=envs.VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME,

527 - create=True, # sender is the writer

528 - )

529 - self._shm_cache = SingleWriterShmObjectStorage(

530 - max_object_size=mm_config.mm_shm_cache_max_object_size_mb * MiB_bytes,

531 - n_readers=self.world_size,

532 - ring_buffer=ring_buffer,

533 - serde_class=MsgpackSerde,

534 - )

535 - # cache prompt_updates for P0 only

536 - self._p0_cache: dict[str, Sequence[ResolvedPromptUpdate]] = {}

537 -

538 - self._hits = 0

539 - self._total = 0

540 - self._last_info = CacheInfo(hits=0, total=0)

541 -

542 - def _stat(self, *, delta: bool = False) -> CacheInfo:

543 - info = CacheInfo(hits=self._hits, total=self._total)

544 -

545 - if delta:

546 - info_delta = info - self._last_info

547 - self._last_info = info

548 - info = info_delta

549 -

550 - return info

551 -

552 - @override

553 - def is_cached_item(self, mm_hash: str) -> bool:

554 - return self._shm_cache.is_cached(mm_hash)

555 -

556 - @override

557 - def get_and_update_item(

558 - self,

559 - mm_item: MultiModalProcessorCacheInItem,

560 - mm_hash: str,

561 - ) -> MultiModalProcessorCacheOutItem:

562 - if self._shm_cache.is_cached(mm_hash):

563 - self._hits += 1

564 - self._total += 1

565 -

566 - address, monotonic_id = self._shm_cache.get_cached(mm_hash)

567 - prompt_updates = self._p0_cache[mm_hash]

568 - return self.address_as_item(address, monotonic_id), prompt_updates

569 -

570 - assert mm_item is not None, f"Expected a cached item for {mm_hash=}"

571 - item, prompt_updates = mm_item

572 -

573 - self._total += 1

574 -

575 - try:

576 - address, monotonic_id = self._shm_cache.put(mm_hash, item)

577 - # Try to remove dangling items if p0 cache is too large.

578 - if len(self._p0_cache) >= 2 * len(self._shm_cache.key_index):

579 - self.remove_dangling_items()

580 -

581 - self._p0_cache[mm_hash] = prompt_updates

582 - return self.address_as_item(address, monotonic_id), prompt_updates

583 - except ValueError as e:

584 - # `put` raises ValueError either for an oversize item or for a

585 - # duplicate key (concurrent insert); the latter is benign so we

586 - # only warn on the oversize case. Subsequent UUID-only requests

587 - # for an oversize item will fail with a cache miss.

588 - if "already exists" not in str(e):

589 - logger.warning_once(

590 - "mm_input %s too large to cache; "

591 - "raise --mm-shm-cache-max-object-size-mb. (%s)",

592 - mm_hash,

593 - str(e),

594 - )

595 - return mm_item

596 - except MemoryError as e:

597 - # Cache full and protected items prevent eviction.

598 - logger.debug(

599 - "mm_input %s not cached; shm cache full, "

600 - "consider raising --mm-processor-cache-gb. (%s)",

601 - mm_hash,

602 - str(e),

603 - )

604 - return mm_item

605 -

606 - @override

607 - def touch_sender_cache_item(self, mm_hash: str) -> None:

608 - """Touch the item in shared memory cache to prevent eviction.

609 - Increments writer_flag on sender side."""

610 - self._shm_cache.touch(mm_hash)

611 -

612 - @override

613 - def clear_cache(self) -> None:

614 - self._shm_cache.clear()

615 - self._p0_cache.clear()

616 -

617 - self._hits = 0

618 - self._total = 0

619 - self._last_info = CacheInfo(hits=0, total=0)

620 -

621 - @override

622 - def make_stats(self, *, delta: bool = False) -> CacheInfo:

623 - return self._stat(delta=delta)

624 -

625 - @override

626 - def close(self) -> None:

627 - self._shm_cache.close()

628 -

629 - def remove_dangling_items(self) -> None:

630 - """Remove items that are no longer in the shared memory cache."""

631 - cached_hashes = self._shm_cache.key_index.keys()

632 - dangling_hashes = set(self._p0_cache.keys()) - cached_hashes

633 - for mm_hash in dangling_hashes:

634 - del self._p0_cache[mm_hash]

635 -

636 - def address_as_item(

637 - self,

638 - address: int,

639 - monotonic_id: int,

640 - ) -> MultiModalKwargsItem:

641 - addr_elem = MultiModalFieldElem(

642 - data=address,

643 - field=MultiModalBatchedField(),

644 - )

645 - id_elem = MultiModalFieldElem(

646 - data=monotonic_id,

647 - field=MultiModalBatchedField(),

648 - )

649 -

650 - return MultiModalKwargsItem({"address": addr_elem, "monotonic_id": id_elem})

651 -

652 -

434 class BaseMultiModalReceiverCache(

435 BaseMultiModalCache[MultiModalKwargsItem | None, MultiModalKwargsItem]

436 ):

@@ -703,139 +484,3 @@ class BaseMultiModalReceiverCache(

484

485 """

486 raise NotImplementedError

706 -

707 -

708 -class MultiModalReceiverCache(BaseMultiModalReceiverCache):

709 - """The cache which is used on P1 when IPC caching is enabled.

710 -

711 - How to update each item:

712 -

713 - - If the caller sent tensor data, store it (replacing any cached item

714 - under the same key) and return that data. P0 can miss after independent

715 - LRU eviction and resend a different item for the same identity.

716 - - If the caller sent no data and the item is cached, return the cached item.

717 - - If the caller sent no data and the item is not cached, raise

718 - ``MultiModalCacheMissError``.

719 - """

720 -

721 - def __init__(self, model_config: "ModelConfig") -> None:

722 - super().__init__()

723 -

724 - mm_config = model_config.get_multimodal_config()

725 -

726 - self._cache = MultiModalCache.get_lru_cache(

727 - mm_config.mm_processor_cache_gb,

728 - MultiModalKwargsItem,

729 - )

730 -

731 - @override

732 - def get_and_update_item(

733 - self,

734 - mm_item: MultiModalKwargsItem | None,

735 - mm_hash: str,

736 - ) -> MultiModalKwargsItem:

737 - if mm_item is not None:

738 - # P0 sent a payload. Never keep a stale cached tensor under this

739 - # identity: a later P0 hit would pair new placeholders with the old

740 - # item and crash the engine. Drop first so a too-large replacement

741 - # cannot leave the previous entry in place.

742 - self._cache.pop(mm_hash, None)

743 - self.cache_if_fits(self._cache, mm_hash, mm_item)

744 - return mm_item

745 -

746 - if (cached_item := self._cache.get(mm_hash)) is not None:

747 - return cached_item

748 -

749 - # No data and not cached here: P0 sent data=None trusting its shadow, but

750 - # the P0/P1 caches have drifted. Raise a retryable error (not assert) so the

751 - # engine can have P0 drop the stale entry and the client resend the data.

752 - raise MultiModalCacheMissError([mm_hash])

753 -

754 - @override

755 - def touch_receiver_cache_item(

756 - self,

757 - mm_hash: str,

758 - mm_item: MultiModalKwargsItem | None = None,

759 - ) -> None:

760 - self._cache.touch(mm_hash)

761 -

762 - @override

763 - def clear_cache(self) -> None:

764 - self._cache.clear()

765 -

766 -

767 -class ShmObjectStoreReceiverCache(BaseMultiModalReceiverCache):

768 - """The cache which is used on P1 Worker Process when IPC caching is enabled.

769 -

770 - How to update each item:

771 -

772 - - If the item has an address, replace the input with the cached item.

773 - - If not, return the input.

774 - """

775 -

776 - def __init__(

777 - self,

778 - vllm_config: "VllmConfig",

779 - shared_worker_lock: LockType,

780 - ) -> None:

781 - super().__init__()

782 -

783 - self.world_size = vllm_config.parallel_config.world_size

784 - mm_config = vllm_config.model_config.get_multimodal_config()

785 -

786 - ring_buffer = SingleWriterShmRingBuffer(

787 - data_buffer_size=int(mm_config.mm_processor_cache_gb * GiB_bytes),

788 - name=envs.VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME,

789 - create=False, # Server is a reader

790 - )

791 - self._shm_cache = SingleWriterShmObjectStorage(

792 - max_object_size=mm_config.mm_shm_cache_max_object_size_mb * MiB_bytes,

793 - n_readers=self.world_size,

794 - ring_buffer=ring_buffer,

795 - serde_class=MsgpackSerde,

796 - reader_lock=shared_worker_lock,

797 - )

798 -

799 - @override

800 - def get_and_update_features(

801 - self, mm_features: list["MultiModalFeatureSpec"]

802 - ) -> list["MultiModalFeatureSpec"]:

803 - # strip_covered_mm_data preserves address items, so None here represents

804 - # a stripped uncached payload and has no SHM reference to acknowledge.

805 - features_with_data = [

806 - feature for feature in mm_features if feature.data is not None

807 - ]

808 - super().get_and_update_features(features_with_data)

809 - return mm_features

810 -

811 - @override

812 - def get_and_update_item(

813 - self,

814 - mm_item: MultiModalKwargsItem | None,

815 - mm_hash: str,

816 - ) -> MultiModalKwargsItem:

817 - assert mm_item is not None, f"Expected an address item for {mm_hash=}"

818 - if "address" in mm_item:

819 - address = cast(int, mm_item["address"].data)

820 - monotonic_id = cast(int, mm_item["monotonic_id"].data)

821 - return self._shm_cache.get(address, monotonic_id)

822 -

823 - return mm_item

824 -

825 - @override

826 - def touch_receiver_cache_item(

827 - self,

828 - mm_hash: str,

829 - mm_item: MultiModalKwargsItem | None = None,

830 - ) -> None:

831 - """Touch the item in shared memory cache to prevent eviction.

832 - Increments reader_count on receiver side."""

833 - assert mm_item is not None

834 - if "address" in mm_item:

835 - address = cast(int, mm_item["address"].data)

836 - monotonic_id = cast(int, mm_item["monotonic_id"].data)

837 - self._shm_cache.touch(mm_hash, address=address, monotonic_id=monotonic_id)

838 -

839 - @override

840 - def clear_cache(self) -> None:

841 - self._shm_cache.clear()

vllm/multimodal/cache/factories.py new

+107

@@ -0,0 +1,107 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +from multiprocessing.synchronize import Lock as LockType

4 +from typing import Literal

5 +

6 +from vllm.config import VllmConfig

7 +

8 +from .base import (

9 + BaseMultiModalProcessorCache,

10 + BaseMultiModalReceiverCache,

11 + MultiModalProcessorOnlyCache,

12 +)

13 +from .lru import (

14 + LruKeyReplicatedReceiverCache,

15 + LruKeyReplicatedSenderCache,

16 +)

17 +from .shm import (

18 + ShmObjectStoreReceiverCache,

19 + ShmObjectStoreSenderCache,

20 +)

21 +

22 +

23 +def _get_cache_type(

24 + vllm_config: VllmConfig,

25 +) -> Literal[None, "processor_only", "lru", "shm"]:

26 + model_config = vllm_config.model_config

27 + if not model_config.supports_multimodal_inputs:

28 + return None

29 +

30 + # Check if the cache is disabled.

31 + mm_config = model_config.get_multimodal_config()

32 + if mm_config.mm_processor_cache_gb <= 0:

33 + return None

34 +

35 + # Check if IPC caching is supported.

36 + parallel_config = vllm_config.parallel_config

37 + is_ipc_supported = parallel_config._api_process_count == 1 and (

38 + parallel_config.data_parallel_size == 1

39 + or parallel_config.data_parallel_external_lb

40 + )

41 +

42 + if not is_ipc_supported:

43 + return "processor_only"

44 +

45 + mm_config = model_config.get_multimodal_config()

46 + return mm_config.mm_processor_cache_type

47 +

48 +

49 +def processor_cache_from_config(

50 + vllm_config: VllmConfig,

51 +) -> BaseMultiModalProcessorCache | None:

52 + """Return a `BaseMultiModalProcessorCache`, if enabled."""

53 + cache_type = _get_cache_type(vllm_config)

54 + if cache_type is None:

55 + return None

56 + elif cache_type == "processor_only":

57 + return MultiModalProcessorOnlyCache(vllm_config.model_config)

58 + elif cache_type == "lru":

59 + return LruKeyReplicatedSenderCache(vllm_config.model_config)

60 + elif cache_type == "shm":

61 + return ShmObjectStoreSenderCache(vllm_config)

62 + else:

63 + raise ValueError(f"Unknown cache type: {cache_type!r}")

64 +

65 +

66 +def processor_only_cache_from_config(

67 + vllm_config: VllmConfig,

68 +) -> MultiModalProcessorOnlyCache | None:

69 + """Return a `MultiModalProcessorOnlyCache`, if enabled."""

70 + cache_type = _get_cache_type(vllm_config)

71 + if cache_type is None:

72 + return None

73 +

74 + return MultiModalProcessorOnlyCache(vllm_config.model_config)

75 +

76 +

77 +def engine_receiver_cache_from_config(

78 + vllm_config: VllmConfig,

79 +) -> BaseMultiModalReceiverCache | None:

80 + """Return a `BaseMultiModalReceiverCache` for the engine process."""

81 + cache_type = _get_cache_type(vllm_config)

82 + if cache_type in (None, "processor_only", "shm"):

83 + return None

84 + elif cache_type == "lru":

85 + return LruKeyReplicatedReceiverCache(vllm_config.model_config)

86 + else:

87 + raise ValueError(f"Unknown cache type: {cache_type!r}")

88 +

89 +

90 +def worker_receiver_cache_from_config(

91 + vllm_config: VllmConfig,

92 + shared_worker_lock: LockType | None,

93 +) -> BaseMultiModalReceiverCache | None:

94 + """Return a `BaseMultiModalReceiverCache` for the worker process."""

95 + cache_type = _get_cache_type(vllm_config)

96 + if cache_type in (None, "processor_only", "lru"):

97 + return None

98 + elif cache_type == "shm":

99 + if shared_worker_lock is None:

100 + raise ValueError(

101 + "Missing `shared_worker_lock` argument from executor. "

102 + "This argument is needed for mm_processor_cache_type='shm'."

103 + )

104 +

105 + return ShmObjectStoreReceiverCache(vllm_config, shared_worker_lock)

106 + else:

107 + raise ValueError(f"Unknown cache type: {cache_type!r}")

vllm/multimodal/cache/lru.py new

+142

@@ -0,0 +1,142 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +"""Implementation of Key-Replicated Cache (see docs/configuration/optimization.md)."""

4 +

5 +from typing_extensions import override

6 +

7 +from vllm.config import ModelConfig

8 +from vllm.utils.cache import CacheInfo

9 +

10 +from ..inputs import MultiModalKwargsItem

11 +from .base import (

12 + BaseMultiModalProcessorCache,

13 + BaseMultiModalReceiverCache,

14 + MultiModalCache,

15 + MultiModalCacheMissError,

16 + MultiModalProcessorCacheInItem,

17 + MultiModalProcessorCacheItemMetadata,

18 + MultiModalProcessorCacheOutItem,

19 +)

20 +

21 +

22 +class LruKeyReplicatedSenderCache(BaseMultiModalProcessorCache):

23 + """The cache which is used on P0 when LRU caching is enabled.

24 +

25 + How to update each item:

26 +

27 + - If the item is already in the cache, clear the input to avoid

28 + unnecessary IPC.

29 +

30 + - If the item is not in the cache, store the metadata of that item so

31 + that the eviction policy remains the same as the cache on P1,

32 + and return the input.

33 + By only storing the metadata, we avoid keeping the data itself in

34 + memory inside P0.

35 + """

36 +

37 + def __init__(self, model_config: ModelConfig) -> None:

38 + super().__init__()

39 +

40 + mm_config = model_config.get_multimodal_config()

41 +

42 + self._cache = MultiModalCache.get_lru_cache(

43 + mm_config.mm_processor_cache_gb,

44 + MultiModalProcessorCacheItemMetadata,

45 + )

46 +

47 + @override

48 + def is_cached_item(self, mm_hash: str) -> bool:

49 + return mm_hash in self._cache

50 +

51 + @override

52 + def get_and_update_item(

53 + self,

54 + mm_item: MultiModalProcessorCacheInItem,

55 + mm_hash: str,

56 + ) -> MultiModalProcessorCacheOutItem:

57 + if (cached_item := self._cache.get(mm_hash)) is not None:

58 + return None, cached_item.prompt_updates

59 +

60 + assert mm_item is not None, f"Expected a cached item for {mm_hash=}"

61 +

62 + self.cache_if_fits(

63 + self._cache, mm_hash, MultiModalProcessorCacheItemMetadata(*mm_item)

64 + )

65 + return mm_item

66 +

67 + @override

68 + def touch_sender_cache_item(self, mm_hash: str) -> None:

69 + self._cache.touch(mm_hash)

70 +

71 + @override

72 + def clear_cache(self) -> None:

73 + self._cache.clear()

74 +

75 + @override

76 + def make_stats(self, *, delta: bool = False) -> CacheInfo:

77 + return self._cache.stat(delta=delta)

78 +

79 + @override

80 + def invalidate(self, mm_hash: str) -> None:

81 + # Drop our stale shadow entry so the next request for this hash re-sends the

82 + # data and repopulates P1 (see MultiModalCacheMissError).

83 + self._cache.pop(mm_hash, None)

84 +

85 +

86 +class LruKeyReplicatedReceiverCache(BaseMultiModalReceiverCache):

87 + """The cache which is used on P1 when LRU caching is enabled.

88 +

89 + How to update each item:

90 +

91 + - If the caller sent tensor data, store it (replacing any cached item

92 + under the same key) and return that data. P0 can miss after independent

93 + LRU eviction and resend a different item for the same identity.

94 + - If the caller sent no data and the item is cached, return the cached item.

95 + - If the caller sent no data and the item is not cached, raise

96 + `MultiModalCacheMissError`.

97 + """

98 +

99 + def __init__(self, model_config: ModelConfig) -> None:

100 + super().__init__()

101 +

102 + mm_config = model_config.get_multimodal_config()

103 +

104 + self._cache = MultiModalCache.get_lru_cache(

105 + mm_config.mm_processor_cache_gb,

106 + MultiModalKwargsItem,

107 + )

108 +

109 + @override

110 + def get_and_update_item(

111 + self,

112 + mm_item: MultiModalKwargsItem | None,

113 + mm_hash: str,

114 + ) -> MultiModalKwargsItem:

115 + if mm_item is not None:

116 + # P0 sent a payload. Never keep a stale cached tensor under this

117 + # identity: a later P0 hit would pair new placeholders with the old

118 + # item and crash the engine. Drop first so a too-large replacement

119 + # cannot leave the previous entry in place.

120 + self._cache.pop(mm_hash, None)

121 + self.cache_if_fits(self._cache, mm_hash, mm_item)

122 + return mm_item

123 +

124 + if (cached_item := self._cache.get(mm_hash)) is not None:

125 + return cached_item

126 +

127 + # No data and not cached here: P0 sent data=None trusting its shadow, but

128 + # the P0/P1 caches have drifted. Raise a retryable error (not assert) so the

129 + # engine can have P0 drop the stale entry and the client resend the data.

130 + raise MultiModalCacheMissError([mm_hash])

131 +

132 + @override

133 + def touch_receiver_cache_item(

134 + self,

135 + mm_hash: str,

136 + mm_item: MultiModalKwargsItem | None = None,

137 + ) -> None:

138 + self._cache.touch(mm_hash)

139 +

140 + @override

141 + def clear_cache(self) -> None:

142 + self._cache.clear()

vllm/multimodal/cache/shm.py new

+261

@@ -0,0 +1,261 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +"""Implementation of Shared Memory Cache (see docs/configuration/optimization.md)."""

4 +

5 +from collections.abc import Sequence

6 +from multiprocessing.synchronize import Lock as LockType

7 +from typing import TYPE_CHECKING, cast

8 +

9 +from typing_extensions import override

10 +

11 +import vllm.envs as envs

12 +from vllm.config import VllmConfig

13 +from vllm.distributed.device_communicators.shm_object_storage import (

14 + MsgpackSerde,

15 + SingleWriterShmObjectStorage,

16 + SingleWriterShmRingBuffer,

17 +)

18 +from vllm.logger import init_logger

19 +from vllm.utils.cache import CacheInfo

20 +from vllm.utils.mem_constants import GiB_bytes, MiB_bytes

21 +

22 +from ..inputs import (

23 + MultiModalBatchedField,

24 + MultiModalFeatureSpec,

25 + MultiModalFieldElem,

26 + MultiModalKwargsItem,

27 +)

28 +from .base import (

29 + BaseMultiModalProcessorCache,

30 + BaseMultiModalReceiverCache,

31 + MultiModalProcessorCacheInItem,

32 + MultiModalProcessorCacheOutItem,

33 +)

34 +

35 +if TYPE_CHECKING:

36 + from ..processing.processor import ResolvedPromptUpdate

37 +

38 +logger = init_logger(__name__)

39 +

40 +

41 +class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):

42 + """The cache which is used on P0 when SHM caching is enabled.

43 +

44 + How to update each item:

45 +

46 + - If the item is already in the cache, clear the input to avoid

47 + unnecessary IPC.

48 +

49 + - If the item is not in the cache, store the data in shared memory.

50 + """

51 +

52 + def __init__(self, vllm_config: VllmConfig) -> None:

53 + super().__init__()

54 +

55 + self.world_size = vllm_config.parallel_config.world_size

56 + mm_config = vllm_config.model_config.get_multimodal_config()

57 +

58 + ring_buffer = SingleWriterShmRingBuffer(

59 + data_buffer_size=int(mm_config.mm_processor_cache_gb * GiB_bytes),

60 + name=envs.VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME,

61 + create=True, # sender is the writer

62 + )

63 + self._shm_cache = SingleWriterShmObjectStorage(

64 + max_object_size=mm_config.mm_shm_cache_max_object_size_mb * MiB_bytes,

65 + n_readers=self.world_size,

66 + ring_buffer=ring_buffer,

67 + serde_class=MsgpackSerde,

68 + )

69 + # cache prompt_updates for P0 only

70 + self._p0_cache: dict[str, Sequence[ResolvedPromptUpdate]] = {}

71 +

72 + self._hits = 0

73 + self._total = 0

74 + self._last_info = CacheInfo(hits=0, total=0)

75 +

76 + def _stat(self, *, delta: bool = False) -> CacheInfo:

77 + info = CacheInfo(hits=self._hits, total=self._total)

78 +

79 + if delta:

80 + info_delta = info - self._last_info

81 + self._last_info = info

82 + info = info_delta

83 +

84 + return info

85 +

86 + @override

87 + def is_cached_item(self, mm_hash: str) -> bool:

88 + return self._shm_cache.is_cached(mm_hash)

89 +

90 + @override

91 + def get_and_update_item(

92 + self,

93 + mm_item: MultiModalProcessorCacheInItem,

94 + mm_hash: str,

95 + ) -> MultiModalProcessorCacheOutItem:

96 + if self._shm_cache.is_cached(mm_hash):

97 + self._hits += 1

98 + self._total += 1

99 +

100 + address, monotonic_id = self._shm_cache.get_cached(mm_hash)

101 + prompt_updates = self._p0_cache[mm_hash]

102 + return self.address_as_item(address, monotonic_id), prompt_updates

103 +

104 + assert mm_item is not None, f"Expected a cached item for {mm_hash=}"

105 + item, prompt_updates = mm_item

106 +

107 + self._total += 1

108 +

109 + try:

110 + address, monotonic_id = self._shm_cache.put(mm_hash, item)

111 + # Try to remove dangling items if p0 cache is too large.

112 + if len(self._p0_cache) >= 2 * len(self._shm_cache.key_index):

113 + self.remove_dangling_items()

114 +

115 + self._p0_cache[mm_hash] = prompt_updates

116 + return self.address_as_item(address, monotonic_id), prompt_updates

117 + except ValueError as e:

118 + # `put` raises ValueError either for an oversize item or for a

119 + # duplicate key (concurrent insert); the latter is benign so we

120 + # only warn on the oversize case. Subsequent UUID-only requests

121 + # for an oversize item will fail with a cache miss.

122 + if "already exists" not in str(e):

123 + logger.warning_once(

124 + "mm_input %s too large to cache; "

125 + "raise --mm-shm-cache-max-object-size-mb. (%s)",

126 + mm_hash,

127 + str(e),

128 + )

129 + return mm_item

130 + except MemoryError as e:

131 + # Cache full and protected items prevent eviction.

132 + logger.debug(

133 + "mm_input %s not cached; shm cache full, "

134 + "consider raising --mm-processor-cache-gb. (%s)",

135 + mm_hash,

136 + str(e),

137 + )

138 + return mm_item

139 +

140 + @override

141 + def touch_sender_cache_item(self, mm_hash: str) -> None:

142 + """Touch the item in shared memory cache to prevent eviction.

143 + Increments writer_flag on sender side."""

144 + self._shm_cache.touch(mm_hash)

145 +

146 + @override

147 + def clear_cache(self) -> None:

148 + self._shm_cache.clear()

149 + self._p0_cache.clear()

150 +

151 + self._hits = 0

152 + self._total = 0

153 + self._last_info = CacheInfo(hits=0, total=0)

154 +

155 + @override

156 + def make_stats(self, *, delta: bool = False) -> CacheInfo:

157 + return self._stat(delta=delta)

158 +

159 + @override

160 + def close(self) -> None:

161 + self._shm_cache.close()

162 +

163 + def remove_dangling_items(self) -> None:

164 + """Remove items that are no longer in the shared memory cache."""

165 + cached_hashes = self._shm_cache.key_index.keys()

166 + dangling_hashes = set(self._p0_cache.keys()) - cached_hashes

167 + for mm_hash in dangling_hashes:

168 + del self._p0_cache[mm_hash]

169 +

170 + def address_as_item(

171 + self,

172 + address: int,

173 + monotonic_id: int,

174 + ) -> MultiModalKwargsItem:

175 + addr_elem = MultiModalFieldElem(

176 + data=address,

177 + field=MultiModalBatchedField(),

178 + )

179 + id_elem = MultiModalFieldElem(

180 + data=monotonic_id,

181 + field=MultiModalBatchedField(),

182 + )

183 +

184 + return MultiModalKwargsItem({"address": addr_elem, "monotonic_id": id_elem})

185 +

186 +

187 +class ShmObjectStoreReceiverCache(BaseMultiModalReceiverCache):

188 + """The cache which is used on P1 Worker Process when SHM caching is enabled.

189 +

190 + How to update each item:

191 +

192 + - If the item has an address, replace the input with the cached item.

193 + - If not, return the input.

194 + """

195 +

196 + def __init__(

197 + self,

198 + vllm_config: VllmConfig,

199 + shared_worker_lock: LockType,

200 + ) -> None:

201 + super().__init__()

202 +

203 + self.world_size = vllm_config.parallel_config.world_size

204 + mm_config = vllm_config.model_config.get_multimodal_config()

205 +

206 + ring_buffer = SingleWriterShmRingBuffer(

207 + data_buffer_size=int(mm_config.mm_processor_cache_gb * GiB_bytes),

208 + name=envs.VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME,

209 + create=False, # Server is a reader

210 + )

211 + self._shm_cache = SingleWriterShmObjectStorage(

212 + max_object_size=mm_config.mm_shm_cache_max_object_size_mb * MiB_bytes,

213 + n_readers=self.world_size,

214 + ring_buffer=ring_buffer,

215 + serde_class=MsgpackSerde,

216 + reader_lock=shared_worker_lock,

217 + )

218 +

219 + @override

220 + def get_and_update_features(

221 + self, mm_features: list["MultiModalFeatureSpec"]

222 + ) -> list["MultiModalFeatureSpec"]:

223 + # strip_covered_mm_data preserves address items, so None here represents

224 + # a stripped uncached payload and has no SHM reference to acknowledge.

225 + features_with_data = [

226 + feature for feature in mm_features if feature.data is not None

227 + ]

228 + super().get_and_update_features(features_with_data)

229 + return mm_features

230 +

231 + @override

232 + def get_and_update_item(

233 + self,

234 + mm_item: MultiModalKwargsItem | None,

235 + mm_hash: str,

236 + ) -> MultiModalKwargsItem:

237 + assert mm_item is not None, f"Expected an address item for {mm_hash=}"

238 + if "address" in mm_item:

239 + address = cast(int, mm_item["address"].data)

240 + monotonic_id = cast(int, mm_item["monotonic_id"].data)

241 + return self._shm_cache.get(address, monotonic_id)

242 +

243 + return mm_item

244 +

245 + @override

246 + def touch_receiver_cache_item(

247 + self,

248 + mm_hash: str,

249 + mm_item: MultiModalKwargsItem | None = None,

250 + ) -> None:

251 + """Touch the item in shared memory cache to prevent eviction.

252 + Increments reader_count on receiver side."""

253 + assert mm_item is not None

254 + if "address" in mm_item:

255 + address = cast(int, mm_item["address"].data)

256 + monotonic_id = cast(int, mm_item["monotonic_id"].data)

257 + self._shm_cache.touch(mm_hash, address=address, monotonic_id=monotonic_id)

258 +

259 + @override

260 + def clear_cache(self) -> None:

261 + self._shm_cache.clear()

vllm/multimodal/encoder_budget.py

+5 -4

@@ -4,7 +4,10 @@ from collections.abc import Mapping

5 from vllm.config import VllmConfig

6 from vllm.logger import init_logger

7 -from vllm.multimodal.cache import BaseMultiModalProcessorCache

7 +from vllm.multimodal.cache import (

8 + BaseMultiModalProcessorCache,

9 + processor_only_cache_from_config,

10 +)

11 from vllm.multimodal.inputs import MultiModalKwargsItem

12 from vllm.multimodal.processing import BaseMultiModalProcessor

13 from vllm.multimodal.registry import MultiModalRegistry

@@ -57,9 +60,7 @@ class MultiModalBudget:

61 with set_default_torch_num_threads(): # Avoid hang during startup

62 cache = (

60 - mm_registry.processor_only_cache_from_config(vllm_config)

61 - if enable_cache

62 - else None

63 + processor_only_cache_from_config(vllm_config) if enable_cache else None

64 )

65 processor = mm_registry.create_processor(model_config)

vllm/multimodal/registry.py

+2 -138

@@ -3,21 +3,11 @@

3 import threading

4 from collections import defaultdict

5 from dataclasses import dataclass

6 -from multiprocessing.synchronize import Lock as LockType

7 -from typing import TYPE_CHECKING, Generic, Literal, Protocol, TypeVar, cast

6 +from typing import TYPE_CHECKING, Generic, Protocol, TypeVar, cast

8 from vllm.logger import init_logger

9 from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config

12 -from .cache import (

13 - BaseMultiModalProcessorCache,

14 - BaseMultiModalReceiverCache,

15 - MultiModalProcessorOnlyCache,

16 - MultiModalProcessorSenderCache,

17 - MultiModalReceiverCache,

18 - ShmObjectStoreReceiverCache,

19 - ShmObjectStoreSenderCache,

20 -)

11 from .processing import (

12 BaseDummyInputsBuilder,

13 BaseMultiModalProcessor,

@@ -27,7 +17,7 @@ from .processing import (

17 )

19 if TYPE_CHECKING:

30 - from vllm.config import ModelConfig, ObservabilityConfig, VllmConfig

20 + from vllm.config import ModelConfig, ObservabilityConfig

21 from vllm.model_executor.models.interfaces import SupportsMultiModal

23 logger = init_logger(__name__)

@@ -89,51 +79,6 @@ class _ProcessorFactories(Generic[_I]):

79 class MultiModalRegistry:

80 """A registry that dispatches data processing according to the model."""

92 - def supports_multimodal_inputs(self, model_config: "ModelConfig") -> bool:

93 - """Checks if the model supports multimodal inputs.

94 - Returns True if the model is multimodal with any non-zero supported

95 - modalities, otherwise returns False, effectively running in

96 - text-only mode.

97 - """

98 - if not model_config.is_multimodal_model:

99 - return False

100 -

101 - mm_config = model_config.get_multimodal_config()

102 - try:

103 - info = self._create_processing_info(model_config, tokenizer=None)

104 - except ValueError:

105 - # Speculative drafters for multimodal targets (e.g. Qwen3_5MTP,

106 - # Exaone4_5_MTP, MiMoV2OmniMTP) declare `SupportsMultiModal` so

107 - # that they can consume the embeddings merged by the target model,

108 - # but they never run a multi-modal processor of their own. Running

109 - # in text-only mode is the expected outcome for them, not a

110 - # misconfiguration worth warning about.

111 - if model_config.runner_type != "draft":

112 - logger.warning_once(

113 - "Model %s is treated as multimodal but has no registered "

114 - "multimodal processor; running in text-only mode.",

115 - model_config.model,

116 - )

117 - return False

118 -

119 - # Check if all supported modalities have limit == 0

120 - if all(

121 - mm_config.get_limit_per_prompt(modality) == 0

122 - for modality in info.supported_mm_limits

123 - ):

124 - # If enable_mm_embeds is True, we still need MM infrastructure

125 - # to process pre-computed embeddings even though encoder won't run

126 - if mm_config.enable_mm_embeds:

127 - return True

128 -

129 - logger.info_once(

130 - "All limits of multimodal modalities supported by the model "

131 - "are set to 0, running in text-only mode."

132 - )

133 - return False

134 -

135 - return True

136 -

82 def register_processor(

83 self,

84 processor: MultiModalProcessorFactory[_I],

@@ -220,87 +165,6 @@ class MultiModalRegistry:

165

166 return factories.build_processor(ctx)

167

223 - def _get_cache_type(

224 - self,

225 - vllm_config: "VllmConfig",

226 - ) -> Literal[None, "processor_only", "lru", "shm"]:

227 - model_config = vllm_config.model_config

228 - if not self.supports_multimodal_inputs(model_config):

229 - return None

230 -

231 - # Check if the cache is disabled.

232 - mm_config = model_config.get_multimodal_config()

233 - if mm_config.mm_processor_cache_gb <= 0:

234 - return None

235 -

236 - # Check if IPC caching is supported.

237 - parallel_config = vllm_config.parallel_config

238 - is_ipc_supported = parallel_config._api_process_count == 1 and (

239 - parallel_config.data_parallel_size == 1

240 - or parallel_config.data_parallel_external_lb

241 - )

242 -

243 - if not is_ipc_supported:

244 - return "processor_only"

245 -

246 - mm_config = model_config.get_multimodal_config()

247 - return mm_config.mm_processor_cache_type

248 -

249 - def processor_cache_from_config(

250 - self,

251 - vllm_config: "VllmConfig",

252 - ) -> BaseMultiModalProcessorCache | None:

253 - """Return a `BaseMultiModalProcessorCache`, if enabled."""

254 - cache_type = self._get_cache_type(vllm_config)

255 - if cache_type is None:

256 - return None

257 - elif cache_type == "processor_only":

258 - return MultiModalProcessorOnlyCache(vllm_config.model_config)

259 - elif cache_type == "lru":

260 - return MultiModalProcessorSenderCache(vllm_config.model_config)

261 - elif cache_type == "shm":

262 - return ShmObjectStoreSenderCache(vllm_config)

263 - else:

264 - raise ValueError(f"Unknown cache type: {cache_type!r}")

265 -

266 - def processor_only_cache_from_config(

267 - self,

268 - vllm_config: "VllmConfig",

269 - ) -> MultiModalProcessorOnlyCache | None:

270 - """Return a `MultiModalProcessorOnlyCache`, if enabled."""

271 - cache_type = self._get_cache_type(vllm_config)

272 - if cache_type is None:

273 - return None

274 -

275 - return MultiModalProcessorOnlyCache(vllm_config.model_config)

276 -

277 - def engine_receiver_cache_from_config(

278 - self,

279 - vllm_config: "VllmConfig",

280 - ) -> BaseMultiModalReceiverCache | None:

281 - """Return a `BaseMultiModalReceiverCache` for the engine process."""

282 - cache_type = self._get_cache_type(vllm_config)

283 - if cache_type in (None, "processor_only", "shm"):

284 - return None

285 - elif cache_type == "lru":

286 - return MultiModalReceiverCache(vllm_config.model_config)

287 - else:

288 - raise ValueError(f"Unknown cache type: {cache_type!r}")

289 -

290 - def worker_receiver_cache_from_config(

291 - self,

292 - vllm_config: "VllmConfig",

293 - shared_worker_lock: LockType,

294 - ) -> BaseMultiModalReceiverCache | None:

295 - """Return a `BaseMultiModalReceiverCache` for the worker process."""

296 - cache_type = self._get_cache_type(vllm_config)

297 - if cache_type in (None, "processor_only", "lru"):

298 - return None

299 - elif cache_type == "shm":

300 - return ShmObjectStoreReceiverCache(vllm_config, shared_worker_lock)

301 - else:

302 - raise ValueError(f"Unknown cache type: {cache_type!r}")

303 -

168

169 class MultiModalTimingRegistry:

170 def __init__(self, observability_config: "ObservabilityConfig | None") -> None:

vllm/renderers/base.py

+8 -6

@@ -32,7 +32,11 @@ from vllm.inputs import (

32 )

33 from vllm.logger import init_logger

34 from vllm.multimodal import MULTIMODAL_REGISTRY as mm_registry

35 -from vllm.multimodal.cache import BaseMultiModalProcessorCache

35 +from vllm.multimodal.cache import (

36 + BaseMultiModalProcessorCache,

37 + processor_cache_from_config,

38 + processor_only_cache_from_config,

39 +)

40 from vllm.multimodal.gpu_ipc_memory import maybe_init_mm_gpu_ipc_pool

41 from vllm.multimodal.parse import (

42 MultiModalDataItems,

@@ -149,7 +153,7 @@ class BaseRenderer(ABC, Generic[_T]):

153

154 self._mm_cache_stats: MultiModalCacheStats | None = None

155

152 - if mm_registry.supports_multimodal_inputs(config.model_config):

156 + if config.model_config.supports_multimodal_inputs:

157 # Install the process-global GPU memory pool used to gate

158 # frontend GPU-side multimodal decoding (no-op when the budget

159 # is 0). Lives in the API-server process only.

@@ -166,10 +170,8 @@ class BaseRenderer(ABC, Generic[_T]):

170 tokenizer=self.tokenizer,

171 )

172

169 - self._mm_processor_cache = mm_registry.processor_cache_from_config(config)

170 - self._mm_processor_only_cache = (

171 - mm_registry.processor_only_cache_from_config(config)

172 - )

173 + self._mm_processor_cache = processor_cache_from_config(config)

174 + self._mm_processor_only_cache = processor_only_cache_from_config(config)

175

176 # This is used to generate internal request ID for MM processing

177 # It has no relation to the request ID for engine core

vllm/v1/core/sched/scheduler.py

+1 -3

@@ -234,9 +234,7 @@ class Scheduler(SchedulerInterface):

234

235 # Encoder-related.

236 # Calculate encoder cache size if applicable

237 - supports_mm_inputs = mm_registry.supports_multimodal_inputs(

238 - vllm_config.model_config

239 - )

237 + supports_mm_inputs = vllm_config.model_config.supports_multimodal_inputs

238 mm_budget = (

239 MultiModalBudget(vllm_config, mm_registry) if supports_mm_inputs else None

240 )

vllm/v1/engine/core.py

+5 -6

@@ -31,8 +31,10 @@ from vllm.envs import enable_envs_cache

31 from vllm.logger import init_logger

32 from vllm.logging_utils.dump_input import dump_engine_exception

33 from vllm.lora.request import LoRARequest

34 -from vllm.multimodal import MULTIMODAL_REGISTRY

35 -from vllm.multimodal.cache import MultiModalCacheMissError

34 +from vllm.multimodal.cache import (

35 + MultiModalCacheMissError,

36 + engine_receiver_cache_from_config,

37 +)

38 from vllm.tasks import POOLING_TASKS, SupportedTask

39 from vllm.tracing import instrument, maybe_init_worker_tracer

40 from vllm.transformers_utils.config import maybe_register_config_serialize_by_value

@@ -182,10 +184,7 @@ class EngineCore:

184 if self.scheduler.ec_connector is not None: # type: ignore

185 self.model_executor.init_ec_output_aggregator()

186

185 - mm_registry = MULTIMODAL_REGISTRY

186 - self.mm_receiver_cache = mm_registry.engine_receiver_cache_from_config(

187 - vllm_config

188 - )

187 + self.mm_receiver_cache = engine_receiver_cache_from_config(vllm_config)

188

189 # If a KV connector is initialized for scheduler, we want to collect

190 # handshake metadata from all workers so the connector in the scheduler

vllm/v1/engine/input_processor.py

+1 -1

@@ -62,7 +62,7 @@ class InputProcessor:

63 self.renderer = renderer or renderer_from_config(vllm_config)

65 - self.supports_mm_inputs = mm_registry.supports_multimodal_inputs(model_config)

65 + self.supports_mm_inputs = model_config.supports_multimodal_inputs

66 self.mm_encoder_cache_size = 0

67 self.skip_prompt_length_check = False

68 if self.supports_mm_inputs:

vllm/v1/spec_decode/llm_base_proposer.py

+1 -5

@@ -35,7 +35,6 @@ from vllm.model_executor.models.laguna_dflash import DFlashLagunaForCausalLM

35 from vllm.model_executor.models.llama_eagle3 import Eagle3LlamaForCausalLM

36 from vllm.model_executor.models.qwen3_dflash import DFlashQwen3ForCausalLM

37 from vllm.model_executor.models.qwen3_eagle3 import Eagle3Qwen3ForCausalLM

38 -from vllm.multimodal import MULTIMODAL_REGISTRY

38 from vllm.platforms import current_platform

39 from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d

40 from vllm.v1.attention.backend import CommonAttentionMetadata

@@ -165,10 +164,7 @@ class SpecDecodeBaseProposer:

164 self.max_positions = self.max_num_tokens

165

166 # Multi-modal data support

168 - self.mm_registry = MULTIMODAL_REGISTRY

169 - self.supports_mm_inputs = self.mm_registry.supports_multimodal_inputs(

170 - vllm_config.model_config

171 - )

167 + self.supports_mm_inputs = vllm_config.model_config.supports_multimodal_inputs

168

169 self.draft_attn_groups: list[AttentionGroup] = []

170 self.kv_cache_gid: int = -1

vllm/v1/worker/gpu/model_runner.py

+1 -3

@@ -257,9 +257,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):

257

258 # Multimodal

259 self.mm_registry = MULTIMODAL_REGISTRY

260 - self.supports_mm_inputs = self.mm_registry.supports_multimodal_inputs(

261 - self.model_config

262 - )

260 + self.supports_mm_inputs = self.model_config.supports_multimodal_inputs

261 self.uses_inputs_embeds = (

262 self.supports_mm_inputs or self.model_config.enable_prompt_embeds

263 )

vllm/v1/worker/gpu/spec_decode/speculator.py

+1 -4

@@ -15,7 +15,6 @@ from vllm.distributed.eplb.eplb_state import EplbState

15 from vllm.logger import init_logger

16 from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase

17 from vllm.model_executor.models import supports_multimodal_embeddings

18 -from vllm.multimodal import MULTIMODAL_REGISTRY

18 from vllm.v1.kv_cache_interface import KVCacheConfig

19 from vllm.v1.watermarking import create_watermarker

20 from vllm.v1.watermarking.spec_decode import (

@@ -226,9 +225,7 @@ class DraftModelSpeculator(BaseSpeculator):

225 )

226 self.draft_attn_layer_names = all_attn_layers - target_attn_layer_names

227

229 - target_supports_mm = MULTIMODAL_REGISTRY.supports_multimodal_inputs(

230 - self.vllm_config.model_config

231 - )

228 + target_supports_mm = self.vllm_config.model_config.supports_multimodal_inputs

229 draft_supports_mm = supports_multimodal_embeddings(self.model)

230 self.supports_mm_inputs = target_supports_mm and draft_supports_mm

231 if target_supports_mm and not draft_supports_mm:

vllm/v1/worker/gpu_model_runner.py

+1 -3

@@ -550,9 +550,7 @@ class GPUModelRunner(

550 self.mm_registry = MULTIMODAL_REGISTRY

551 self.uses_mrope = model_config.uses_mrope

552 self.mrope_num_dims = model_config.mrope_num_dims

553 - self.supports_mm_inputs = self.mm_registry.supports_multimodal_inputs(

554 - model_config

555 - )

553 + self.supports_mm_inputs = model_config.supports_multimodal_inputs

554

555 if self.model_config.is_encoder_decoder:

556 # Maximum length of the encoder input, only for encoder-decoder

vllm/v1/worker/worker_base.py

+5 -21

@@ -12,7 +12,7 @@ from vllm.config import VllmConfig, set_current_vllm_config

12 from vllm.distributed.kv_transfer.kv_connector.utils import get_current_attn_backends

13 from vllm.logger import init_logger

14 from vllm.lora.request import LoRARequest

15 -from vllm.multimodal import MULTIMODAL_REGISTRY

15 +from vllm.multimodal.cache import worker_receiver_cache_from_config

16 from vllm.tracing import instrument

17 from vllm.utils.import_utils import resolve_obj_by_qualname

18 from vllm.utils.system_utils import update_environment_variables

@@ -315,26 +315,10 @@ class WorkerWrapperBase:

315 )

316

317 shared_worker_lock = kwargs.pop("shared_worker_lock", None)

318 - if shared_worker_lock is None:

319 - msg = (

320 - "Missing `shared_worker_lock` argument from executor. "

321 - "This argument is needed for mm_processor_cache_type='shm'."

322 - )

323 -

324 - mm_config = vllm_config.model_config.multimodal_config

325 - if mm_config and mm_config.mm_processor_cache_type == "shm":

326 - raise ValueError(msg)

327 - else:

328 - logger.warning_once(msg)

329 -

330 - self.mm_receiver_cache = None

331 - else:

332 - self.mm_receiver_cache = (

333 - MULTIMODAL_REGISTRY.worker_receiver_cache_from_config(

334 - vllm_config,

335 - shared_worker_lock,

336 - )

337 - )

318 + self.mm_receiver_cache = worker_receiver_cache_from_config(

319 + vllm_config,

320 + shared_worker_lock,

321 + )

322

323 with set_current_vllm_config(self.vllm_config):

324 # To make vLLM config available during worker initialization