@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 69244e67e6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Core] Use key-only cache for `BaseMultiModalProcessor` (#23018)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Aug 27, 2025 at 14:19 UTC 69244e67e6822f1c15816f887659e1ccc18c2632
29 files changed +954 -394
docs/configuration/conserving_memory.md
+1 -1
@@ -86,7 +86,7 @@ llm = LLM(model="meta-llama/Llama-3.1-8B-Instruct",
86
87 If you run out of CPU RAM, try the following options:
88
89 -- (Multi-modal models only) you can set the size of multi-modal processor cache by setting `mm_processor_cache_gb` engine argument (default 4 GiB per API process + 4 GiB per engine core process)
89 +- (Multi-modal models only) you can set the size of multi-modal cache by setting `mm_processor_cache_gb` engine argument (default 4 GiB).
90 - (CPU backend only) you can set the size of KV cache using `VLLM_CPU_KVCACHE_SPACE` environment variable (default 4 GiB).
91
92 ## Multi-modal input limits
docs/configuration/optimization.md
+35 -9
@@ -204,20 +204,33 @@ vllm serve Qwen/Qwen2.5-VL-3B-Instruct --api-server-count 4 -dp 2
204 to avoid CPU resource exhaustion.
205
206 !!! note
207 - [Multi-modal processor cache](#processor-cache) is disabled when API server scale-out is enabled
208 - because it requires a one-to-one correspondence between API and engine core processes.
207 + API server scale-out disables [multi-modal IPC caching](#ipc-caching)
208 + because it requires a one-to-one correspondance between API and engine core processes.
209
210 -## Multi-Modal Caching
210 + This does not impact [multi-modal processor caching](#processor-caching).
211
212 -### Processor Cache
212 +## Multi-Modal Caching
213
214 -By default, the multi-modal processor cache is enabled to avoid repeatedly processing
215 -the same multi-modal inputs via Hugging Face `AutoProcessor`,
214 +Multi-modal caching avoids repeated transfer or processing of the same multi-modal data,
215 which commonly occurs in multi-turn conversations.
216
218 -You can adjust the size of the cache by setting the value of `mm_processor_cache_gb`
219 -(default 4 GiB per API process + 4 GiB per engine core process).
220 -If you do not benefit much from the cache, you can disable it completely via `mm_processor_cache_gb=0`.
217 +### Processor Caching
218 +
219 +Multi-modal processor caching is automatically enabled
220 +to avoid repeatedly processing the same multi-modal inputs in `BaseMultiModalProcessor`.
221 +
222 +### IPC Caching
223 +
224 +Multi-modal IPC caching is automatically enabled when
225 +there is a one-to-one correspondance between API (`P0`) and engine core (`P1`) processes,
226 +to avoid repeatedly transferring the same multi-modal inputs between them.
227 +
228 +### Configuration
229 +
230 +You can adjust the size of the cache by setting the value of `mm_processor_cache_gb` (default 4 GiB).
231 +
232 +If you do not benefit much from the cache, you can disable both IPC
233 +and processor caching completely via `mm_processor_cache_gb=0`.
234
235 Examples:
236
@@ -230,3 +243,16 @@ llm = LLM(model="Qwen/Qwen2.5-VL-3B-Instruct",
243 llm = LLM(model="Qwen/Qwen2.5-VL-3B-Instruct",
244 mm_processor_cache_gb=0)
245 ```
246 +
247 +### Cache Placement
248 +
249 +Based on the configuration, the content of the multi-modal caches on `P0` and `P1` are as follows:
250 +
251 +| Processor Caching | IPC Caching | `P0` Cache | `P1` Cache | Max. Memory |
252 +|-------------------|-------------|------------|------------|-------------|
253 +| ✅ | ✅ | K | K + V | `mm_processor_cache_gb * data_parallel_size` |
254 +| ✅ | ❌ | K + V | N/A | `mm_processor_cache_gb * api_server_count` |
255 +| ❌ | ❌ | N/A | N/A | `0` |
256 +
257 +K: Stores the hashes of multi-modal items
258 +V: Stores the processed tensor data of multi-modal items
tests/models/multimodal/processing/test_common.py
+5 -3
@@ -14,8 +14,9 @@ from PIL import Image
14 from vllm.config import ModelConfig
15 from vllm.inputs import InputProcessingContext
16 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict
17 +from vllm.multimodal.cache import MultiModalProcessorOnlyCache
18 from vllm.multimodal.inputs import MultiModalInputs
18 -from vllm.multimodal.processing import BaseMultiModalProcessor, ProcessingCache
19 +from vllm.multimodal.processing import BaseMultiModalProcessor
20 from vllm.transformers_utils.tokenizer import (AnyTokenizer, MistralTokenizer,
21 cached_tokenizer_from_config,
22 encode_tokens)
@@ -63,6 +64,8 @@ def _test_processing_correctness(
64 revision=model_info.revision,
65 trust_remote_code=model_info.trust_remote_code,
66 hf_overrides=model_info.hf_overrides,
67 + # Ensure that the cache can fit all of the data
68 + mm_processor_cache_gb=2048,
69 )
70
71 model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
@@ -71,8 +74,7 @@ def _test_processing_correctness(
74 model_config,
75 tokenizer=cached_tokenizer_from_config(model_config),
76 )
74 - # Ensure that it can fit all of the data
75 - cache = ProcessingCache(capacity_gb=2048)
77 + cache = MultiModalProcessorOnlyCache(model_config)
78
79 processing_info = factories.info(ctx)
80 supported_mm_limits = processing_info.get_supported_mm_limits()
tests/multimodal/test_cache.py
+174 -8
@@ -1,32 +1,64 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +from typing import Optional
4 +
5 +import numpy as np
6 import pytest
7 import torch
8
6 -from vllm.multimodal.cache import MultiModalCache, MultiModalCacheItemMetadata
9 +from vllm.config import ModelConfig, ParallelConfig, VllmConfig
10 +from vllm.multimodal.cache import (MultiModalCache,
11 + MultiModalProcessorCacheItem,
12 + MultiModalProcessorCacheItemMetadata,
13 + processor_cache_from_config,
14 + receiver_cache_from_config)
15 +from vllm.multimodal.hasher import MultiModalHasher
16 from vllm.multimodal.inputs import (MultiModalFieldElem, MultiModalKwargsItem,
17 MultiModalKwargsItems,
18 MultiModalSharedField)
19 +from vllm.multimodal.processing import PromptInsertion
20 +from vllm.multimodal.registry import MultiModalRegistry
21 +
22
23 +def _dummy_elem(
24 + modality: str,
25 + key: str,
26 + size: int,
27 + *,
28 + rng: Optional[np.random.RandomState] = None,
29 +):
30 + if rng is None:
31 + data = torch.empty((size, ), dtype=torch.int8)
32 + else:
33 + data = torch.from_numpy(rng.randint(4, size=(size, ), dtype=np.int8))
34
12 -def _dummy_elem(modality: str, key: str, size: int):
35 return MultiModalFieldElem(
36 modality=modality,
37 key=key,
16 - data=torch.empty((size, ), dtype=torch.int8),
38 + data=data,
39 field=MultiModalSharedField(1),
40 )
41
42
21 -def _dummy_item(modality: str, size_by_key: dict[str, int]):
43 +def _dummy_item(
44 + modality: str,
45 + size_by_key: dict[str, int],
46 + *,
47 + rng: Optional[np.random.RandomState] = None,
48 +):
49 return MultiModalKwargsItem.from_elems([
23 - _dummy_elem(modality, key, size) for key, size in size_by_key.items()
50 + _dummy_elem(modality, key, size, rng=rng)
51 + for key, size in size_by_key.items()
52 ])
53
54
27 -def _dummy_items(size_by_key_modality: dict[str, dict[str, int]]):
55 +def _dummy_items(
56 + size_by_key_modality: dict[str, dict[str, int]],
57 + *,
58 + rng: Optional[np.random.RandomState] = None,
59 +):
60 return MultiModalKwargsItems.from_seq([
29 - _dummy_item(modality, size_by_key)
61 + _dummy_item(modality, size_by_key, rng=rng)
62 for modality, size_by_key in size_by_key_modality.items()
63 ])
64
@@ -48,5 +80,139 @@ def test_cache_item_size(item, expected_size):
80 cache[""] = item
81 assert cache.currsize == expected_size
82
51 - cache[""] = MultiModalCacheItemMetadata.wraps(item)
83 + prompt_update = PromptInsertion("dummy", "target", "insertion") \
84 + .resolve(0)
85 +
86 + cache[""] = MultiModalProcessorCacheItem(item, [prompt_update])
87 + assert cache.currsize == expected_size
88 +
89 + cache[""] = MultiModalProcessorCacheItemMetadata(item, [prompt_update])
90 assert cache.currsize == expected_size
91 +
92 +
93 +def _create_vllm_config(
94 + *,
95 + mm_processor_cache_gb: float,
96 + enable_ipc: bool,
97 +):
98 + return VllmConfig(
99 + model_config=ModelConfig(mm_processor_cache_gb=mm_processor_cache_gb),
100 + parallel_config=ParallelConfig(
101 + data_parallel_size=1 if enable_ipc else 2),
102 + )
103 +
104 +
105 +def _compare_caches(
106 + config_0: VllmConfig,
107 + config_1: VllmConfig,
108 + *,
109 + item_capacity: int = 8,
110 + hit_rate: float = 0.5,
111 + max_items_per_iter: int = 3,
112 + is_cached_calls_per_iter: int,
113 + n_iter: int = 100,
114 + seed: int = 0,
115 +):
116 + mm_registry = MultiModalRegistry()
117 + cache_0_p0 = processor_cache_from_config(config_0, mm_registry)
118 + cache_0_p1 = receiver_cache_from_config(config_0, mm_registry)
119 + cache_1_p0 = processor_cache_from_config(config_1, mm_registry)
120 + cache_1_p1 = receiver_cache_from_config(config_1, mm_registry)
121 +
122 + cache_size_gb = max(
123 + config_0.model_config.mm_processor_cache_gb,
124 + config_1.model_config.mm_processor_cache_gb,
125 + )
126 + item_size_gb = int(cache_size_gb / item_capacity)
127 +
128 + rng = np.random.RandomState(seed)
129 + all_items = [
130 + _dummy_item("item", {"key": item_size_gb}, rng=rng)
131 + for _ in range(int(item_capacity / hit_rate))
132 + ]
133 + all_hashes = [
134 + MultiModalHasher.hash_kwargs(item=item.get_data())
135 + for item in all_items
136 + ]
137 +
138 + # Should not be used since there is nothing to convert to text
139 + prompt_update = PromptInsertion("dummy", "target", "insertion")
140 +
141 + for it in range(n_iter):
142 + num_items_to_select = rng.randint(0, max_items_per_iter)
143 + item_idxs_to_select = rng.choice(len(all_items), num_items_to_select)
144 +
145 + selected_items = [all_items[idx] for idx in item_idxs_to_select]
146 + selected_hashes = [all_hashes[idx] for idx in item_idxs_to_select]
147 +
148 + if cache_0_p0 is None:
149 + cache_0_p0_out = selected_items
150 + else:
151 + for _ in range(is_cached_calls_per_iter):
152 + cache_0_p0.is_cached(selected_hashes)
153 + cache_0_p0_out = [
154 + item for item, _ in cache_0_p0.get_and_update(
155 + [(item, prompt_update.content) for item in selected_items],
156 + selected_hashes,
157 + )
158 + ]
159 +
160 + if cache_1_p0 is None:
161 + cache_1_p0_out = selected_items
162 + else:
163 + for _ in range(is_cached_calls_per_iter):
164 + cache_1_p0.is_cached(selected_hashes)
165 + cache_1_p0_out = [
166 + item for item, _ in cache_1_p0.get_and_update(
167 + [(item, prompt_update.content) for item in selected_items],
168 + selected_hashes,
169 + )
170 + ]
171 +
172 + if cache_0_p1 is None:
173 + cache_0_p1_out = cache_0_p0_out
174 + else:
175 + cache_0_p1_out = cache_0_p1.get_and_update(cache_0_p0_out,
176 + selected_hashes)
177 +
178 + if cache_1_p1 is None:
179 + cache_1_p1_out = cache_1_p0_out
180 + else:
181 + cache_1_p1_out = cache_1_p1.get_and_update(cache_1_p0_out,
182 + selected_hashes)
183 +
184 + assert cache_0_p1_out == cache_1_p1_out, f"Failed at {it=}"
185 +
186 +
187 +@pytest.mark.parametrize("is_cached_calls_per_iter", [1, 2, 3])
188 +def test_ipc_enable_disable_consistency(is_cached_calls_per_iter):
189 + cache_size_gb = 1 / (1 << 20)
190 +
191 + vllm_config_ipc_enabled = _create_vllm_config(
192 + mm_processor_cache_gb=cache_size_gb,
193 + enable_ipc=True,
194 + )
195 + vllm_config_ipc_disabled = _create_vllm_config(
196 + mm_processor_cache_gb=0,
197 + enable_ipc=False,
198 + )
199 + vllm_config_cache_disabled = _create_vllm_config(
200 + mm_processor_cache_gb=cache_size_gb,
201 + enable_ipc=True,
202 + )
203 +
204 + _compare_caches(
205 + vllm_config_ipc_enabled,
206 + vllm_config_ipc_disabled,
207 + is_cached_calls_per_iter=is_cached_calls_per_iter,
208 + )
209 + _compare_caches(
210 + vllm_config_ipc_disabled,
211 + vllm_config_cache_disabled,
212 + is_cached_calls_per_iter=is_cached_calls_per_iter,
213 + )
214 + _compare_caches(
215 + vllm_config_cache_disabled,
216 + vllm_config_ipc_enabled,
217 + is_cached_calls_per_iter=is_cached_calls_per_iter,
218 + )
vllm/config/__init__.py
+2 -24
@@ -437,7 +437,7 @@ class ModelConfig:
437 from `AutoProcessor.from_pretrained`. The available overrides depend on the
438 model that is being run. For example, for Phi-3-Vision: `{"num_crops": 4}`.
439 """
440 - mm_processor_cache_gb: int = 4
440 + mm_processor_cache_gb: float = 4
441 """The size (in GiB) of the multi-modal processor cache, which is used to
442 avoid re-processing past multi-modal inputs.
443
@@ -884,12 +884,6 @@ class ModelConfig:
884
885 return None
886
887 - def set_mm_processor_cache_gb(self, value: int) -> None:
888 - mm_config = self.get_multimodal_config()
889 -
890 - self.mm_processor_cache_gb = value
891 - mm_config.mm_processor_cache_gb = value
892 -
887 def _get_encoder_config(self):
888 return get_sentence_transformer_tokenizer_config(
889 self.model, self.revision)
@@ -1697,22 +1691,6 @@ class ModelConfig:
1691 def is_multimodal_model(self) -> bool:
1692 return self.multimodal_config is not None
1693
1700 - @property
1701 - def enable_mm_processor_cache(self) -> bool:
1702 - """Whether the multi-modal processor cache should be enabled."""
1703 - mm_config = self.multimodal_config
1704 - if mm_config is None:
1705 - return False
1706 -
1707 - return mm_config.mm_processor_cache_gb > 0
1708 -
1709 - def get_mm_input_cache_gb(self) -> int:
1710 - mm_config = self.multimodal_config
1711 - if mm_config is None:
1712 - return 0
1713 -
1714 - return envs.VLLM_MM_INPUT_CACHE_GIB
1715 -
1694 @property
1695 def is_cross_encoder(self) -> bool:
1696 return (self._model_info.supports_cross_encoding
@@ -2561,7 +2539,7 @@ class MultiModalConfig:
2539 `{"num_crops": 4}`.
2540 """
2541
2564 - mm_processor_cache_gb: int = 4
2542 + mm_processor_cache_gb: float = 4
2543 """
2544 The size (in GiB) of the multi-modal processor cache, which is used to
2545
vllm/engine/arg_utils.py
+1 -13
@@ -351,7 +351,7 @@ class EngineArgs:
351 mm_processor_kwargs: Optional[Dict[str, Any]] = \
352 MultiModalConfig.mm_processor_kwargs
353 disable_mm_preprocessor_cache: bool = False # DEPRECATED
354 - mm_processor_cache_gb: int = MultiModalConfig.mm_processor_cache_gb
354 + mm_processor_cache_gb: float = MultiModalConfig.mm_processor_cache_gb
355 mm_encoder_tp_mode: MMEncoderTPMode = MultiModalConfig.mm_encoder_tp_mode
356 skip_mm_profiling: bool = MultiModalConfig.skip_mm_profiling
357 # LoRA fields
@@ -1293,18 +1293,6 @@ class EngineArgs:
1293 worker_extension_cls=self.worker_extension_cls,
1294 )
1295
1296 - if model_config.is_multimodal_model:
1297 - dp_supports_mm_processor_cache = (self.data_parallel_size == 1
1298 - or data_parallel_external_lb)
1299 - if (not dp_supports_mm_processor_cache
1300 - and model_config.mm_processor_cache_gb > 0):
1301 - logger.warning(
1302 - "Multi-modal processor cache is disabled because "
1303 - "it is not compatible with data parallelism when "
1304 - "there does not exist a one-to-one correspondance "
1305 - "between API and engine core processes.")
1306 - model_config.set_mm_processor_cache_gb(0)
1307 -
1296 speculative_config = self.create_speculative_config(
1297 target_model_config=model_config,
1298 target_parallel_config=parallel_config,
vllm/engine/llm_engine.py
+10 -5
@@ -36,6 +36,7 @@ from vllm.logits_process import get_bad_words_logits_processors
36 from vllm.lora.request import LoRARequest
37 from vllm.model_executor.layers.sampler import SamplerOutput
38 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
39 +from vllm.multimodal.cache import processor_only_cache_from_config
40 from vllm.multimodal.processing import EncDecMultiModalProcessor
41 from vllm.outputs import (PoolingRequestOutput, RequestOutput,
42 RequestOutputFactory)
@@ -250,9 +251,13 @@ class LLMEngine:
251 self.generation_config_fields = (
252 self.model_config.try_get_generation_config())
253
253 - self.input_preprocessor = InputPreprocessor(self.model_config,
254 - self.tokenizer,
255 - mm_registry)
254 + self.input_preprocessor = InputPreprocessor(
255 + self.model_config,
256 + self.tokenizer,
257 + mm_registry,
258 + mm_processor_cache=processor_only_cache_from_config(
259 + self.model_config, mm_registry),
260 + )
261
262 self.model_executor = executor_class(vllm_config=vllm_config)
263
@@ -840,8 +845,8 @@ class LLMEngine:
845
846 def reset_mm_cache(self) -> bool:
847 """Reset the multi-modal cache."""
843 - return self.input_preprocessor.mm_registry.reset_processor_cache(
844 - self.model_config)
848 + self.input_preprocessor.clear_cache()
849 + return True
850
851 def reset_prefix_cache(self, device: Optional[Device] = None) -> bool:
852 """Reset prefix cache for all devices."""
vllm/inputs/preprocess.py
+18 -4
@@ -11,6 +11,7 @@ from vllm.config import ModelConfig
11 from vllm.logger import init_logger
12 from vllm.lora.request import LoRARequest
13 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
14 +from vllm.multimodal.cache import BaseMultiModalProcessorCache
15 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalEncDecInputs,
16 MultiModalInputs)
17 from vllm.transformers_utils.tokenizer import AnyTokenizer
@@ -32,12 +33,14 @@ class InputPreprocessor:
33 model_config: ModelConfig,
34 tokenizer: Optional[TokenizerGroup],
35 mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
36 + mm_processor_cache: Optional[BaseMultiModalProcessorCache] = None,
37 ) -> None:
38 super().__init__()
39
40 self.model_config = model_config
41 self.tokenizer = tokenizer
42 self.mm_registry = mm_registry
43 + self.mm_processor_cache = mm_processor_cache
44
45 def get_tokenizer_group(self) -> TokenizerGroup:
46 if self.tokenizer is None:
@@ -261,8 +264,11 @@ class InputPreprocessor:
264 """
265 tokenizer = self._get_mm_tokenizer(lora_request)
266
264 - mm_processor = self.mm_registry.create_processor(self.model_config,
265 - tokenizer=tokenizer)
267 + mm_processor = self.mm_registry.create_processor(
268 + self.model_config,
269 + tokenizer=tokenizer,
270 + cache=self.mm_processor_cache,
271 + )
272
273 if mm_processor_kwargs is None:
274 mm_processor_kwargs = {}
@@ -286,8 +292,12 @@ class InputPreprocessor:
292 """
293 tokenizer = await self._get_mm_tokenizer_async(lora_request)
294
289 - mm_processor = self.mm_registry.create_processor(self.model_config,
290 - tokenizer=tokenizer)
295 + mm_processor = self.mm_registry.create_processor(
296 + self.model_config,
297 + tokenizer=tokenizer,
298 + cache=self.mm_processor_cache,
299 + )
300 +
301 if mm_processor_kwargs is None:
302 mm_processor_kwargs = {}
303
@@ -860,3 +870,7 @@ class InputPreprocessor:
870 tokenization_kwargs=tokenization_kwargs,
871 lora_request=lora_request,
872 )
873 +
874 + def clear_cache(self) -> None:
875 + if self.mm_processor_cache is not None:
876 + self.mm_processor_cache.clear_cache()
vllm/inputs/registry.py
+9 -3
@@ -223,20 +223,26 @@ class InputRegistry:
223 The model is identified by ``model_config``.
224 """
225 # Avoid circular import
226 + from vllm.multimodal.cache import processor_only_cache_from_config
227 from vllm.sequence import SequenceData
228
229 if not model_config.is_multimodal_model:
230 seq_data = SequenceData.from_prompt_token_counts((0, seq_len))
231 return DummyData(seq_data=seq_data)
232
233 + cache = processor_only_cache_from_config(model_config, mm_registry)
234 +
235 # Encoder dummy data does not contain multi-modal data
236 if is_encoder_data:
234 - enc_data = mm_registry.get_encoder_dummy_data(
235 - model_config, seq_len)
237 + enc_data = mm_registry.get_encoder_dummy_data(model_config,
238 + seq_len,
239 + cache=cache)
240 seq_data = SequenceData.from_seqs(enc_data.prompt_token_ids)
241 return DummyData(seq_data=seq_data)
242
239 - dec_data = mm_registry.get_decoder_dummy_data(model_config, seq_len)
243 + dec_data = mm_registry.get_decoder_dummy_data(model_config,
244 + seq_len,
245 + cache=cache)
246
247 return DummyData(
248 seq_data=SequenceData.from_seqs(dec_data.prompt_token_ids),
vllm/model_executor/models/hyperclovax_vision.py
+4 -3
@@ -33,12 +33,13 @@ from vllm.inputs import InputProcessingContext
33 from vllm.model_executor.layers.quantization import QuantizationConfig
34 from vllm.model_executor.sampling_metadata import SamplingMetadata
35 from vllm.multimodal import MULTIMODAL_REGISTRY
36 +from vllm.multimodal.cache import BaseMultiModalProcessorCache
37 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
38 MultiModalKwargsItems)
39 from vllm.multimodal.parse import ImageSize, MultiModalDataItems
40 from vllm.multimodal.processing import (BaseMultiModalProcessor,
40 - BaseProcessingInfo, ProcessingCache,
41 - PromptReplacement, PromptUpdate)
41 + BaseProcessingInfo, PromptReplacement,
42 + PromptUpdate)
43 from vllm.multimodal.profiling import BaseDummyInputsBuilder
44 from vllm.sequence import IntermediateTensors
45
@@ -367,7 +368,7 @@ def _build_hcxvision_hf_processor(
368 info: HCXVisionProcessingInfo,
369 dummy_inputs: BaseDummyInputsBuilder[HCXVisionProcessingInfo],
370 *,
370 - cache: Optional[ProcessingCache] = None,
371 + cache: Optional[BaseMultiModalProcessorCache] = None,
372 ) -> BaseMultiModalProcessor:
373 if isinstance(info, HCXVisionProcessingInfo):
374 return HCXVisionMultiModalProcessor(
vllm/model_executor/models/llava.py
+4 -4
@@ -22,14 +22,14 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
22 from vllm.model_executor.layers.quantization import QuantizationConfig
23 from vllm.model_executor.sampling_metadata import SamplingMetadata
24 from vllm.multimodal import MULTIMODAL_REGISTRY
25 +from vllm.multimodal.cache import BaseMultiModalProcessorCache
26 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
27 MultiModalInputs, MultiModalKwargsItems)
28 from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
29 ImageSize, MultiModalDataItems)
30 from vllm.multimodal.processing import (BaseMultiModalProcessor,
30 - BaseProcessingInfo, ProcessingCache,
31 - PromptReplacement, PromptUpdate,
32 - PromptUpdateDetails)
31 + BaseProcessingInfo, PromptReplacement,
32 + PromptUpdate, PromptUpdateDetails)
33 from vllm.multimodal.profiling import BaseDummyInputsBuilder
34 from vllm.sequence import IntermediateTensors
35 from vllm.utils.jsontree import json_map_leaves
@@ -394,7 +394,7 @@ def _build_llava_or_pixtral_hf_processor(
394 info: _I,
395 dummy_inputs: BaseDummyInputsBuilder[_I],
396 *,
397 - cache: Optional[ProcessingCache] = None,
397 + cache: Optional[BaseMultiModalProcessorCache] = None,
398 ) -> BaseMultiModalProcessor:
399 if isinstance(info, PixtralHFProcessingInfo):
400 return PixtralHFMultiModalProcessor(
vllm/model_executor/models/minicpmv.py
+39 -1
@@ -58,7 +58,8 @@ from vllm.multimodal.parse import (DictEmbeddingItems, ImageItem,
58 VideoItem, VideoProcessorItems)
59 from vllm.multimodal.processing import (BaseMultiModalProcessor,
60 BaseProcessingInfo, PromptReplacement,
61 - PromptUpdate, PromptUpdateDetails)
61 + PromptUpdate, PromptUpdateDetails,
62 + ResolvedPromptUpdate, _seq2text)
63 from vllm.multimodal.profiling import BaseDummyInputsBuilder
64 from vllm.platforms import current_platform
65 from vllm.sequence import IntermediateTensors
@@ -744,6 +745,43 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
745 for modality, pattern in placeholders
746 ]
747
748 + def _recompute_cached_prompt_update(
749 + self,
750 + cached_update: ResolvedPromptUpdate,
751 + new_item_idx: int,
752 + ) -> ResolvedPromptUpdate:
753 + new_update = super()._recompute_cached_prompt_update(
754 + cached_update,
755 + new_item_idx,
756 + )
757 +
758 + if cached_update.modality == "image":
759 + tokenizer = self.info.get_tokenizer()
760 + image_processor = self.info.get_image_processor()
761 + version = self.info.get_model_version()
762 +
763 + text = _seq2text(tokenizer, cached_update.content.full)
764 + prev_item_idx = cached_update.item_idx
765 +
766 + if version == (2, 0) or version == (2, 5):
767 + im_start = image_processor.im_start_token
768 + im_end = image_processor.im_end_token
769 + else:
770 + im_start = image_processor.im_id_start
771 + im_end = image_processor.im_id_end
772 +
773 + new_update = new_update.with_content(
774 + PromptUpdateDetails.select_text(
775 + text.replace(
776 + f"{im_start}{prev_item_idx}{im_end}",
777 + f"{im_start}{new_item_idx}{im_end}",
778 + 1,
779 + ),
780 + "<unk>",
781 + ))
782 +
783 + return new_update
784 +
785 def _get_mm_fields_config(
786 self,
787 hf_inputs: BatchFeature,
vllm/model_executor/models/mistral3.py
+4 -4
@@ -22,14 +22,14 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
22 from vllm.model_executor.models.module_mapping import MultiModelKeys
23 from vllm.model_executor.sampling_metadata import SamplingMetadata
24 from vllm.multimodal import MULTIMODAL_REGISTRY
25 +from vllm.multimodal.cache import BaseMultiModalProcessorCache
26 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
27 MultiModalKwargsItems)
28 from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
29 MultiModalDataItems)
30 from vllm.multimodal.processing import (BaseMultiModalProcessor,
30 - BaseProcessingInfo, ProcessingCache,
31 - PromptReplacement, PromptUpdate,
32 - PromptUpdateDetails)
31 + BaseProcessingInfo, PromptReplacement,
32 + PromptUpdate, PromptUpdateDetails)
33 from vllm.multimodal.profiling import BaseDummyInputsBuilder
34 from vllm.sequence import IntermediateTensors
35 from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -322,7 +322,7 @@ def _build_mistral3_processor(
322 info: _I,
323 dummy_inputs: BaseDummyInputsBuilder[_I],
324 *,
325 - cache: Optional[ProcessingCache] = None,
325 + cache: Optional[BaseMultiModalProcessorCache] = None,
326 ) -> BaseMultiModalProcessor:
327 assert isinstance(info, Mistral3ProcessingInfo)
328 return Mistral3MultiModalProcessor(
vllm/model_executor/models/phi3v.py
+19 -1
@@ -41,7 +41,8 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
41 BaseProcessingInfo,
42 MultiModalPromptUpdates,
43 PlaceholderFeaturesInfo,
44 - PromptReplacement, PromptUpdate)
44 + PromptReplacement, PromptUpdate,
45 + ResolvedPromptUpdate)
46 # yapf: enable
47 from vllm.multimodal.profiling import BaseDummyInputsBuilder
48 from vllm.sequence import IntermediateTensors
@@ -440,6 +441,23 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
441 )
442 ]
443
444 + def _recompute_cached_prompt_update(
445 + self,
446 + cached_update: ResolvedPromptUpdate,
447 + new_item_idx: int,
448 + ) -> ResolvedPromptUpdate:
449 + new_update = super()._recompute_cached_prompt_update(
450 + cached_update,
451 + new_item_idx,
452 + )
453 +
454 + if cached_update.modality == "image":
455 + hf_processor = self.info.get_hf_processor()
456 + image_tokens: list[str] = hf_processor.img_tokens # type: ignore
457 + new_update = new_update.with_target(image_tokens[new_item_idx])
458 +
459 + return new_update
460 +
461 def _apply_prompt_updates(
462 self,
463 token_ids: list[int],
vllm/model_executor/models/phi4mm.py
+20 -1
@@ -27,7 +27,7 @@ from vllm.multimodal.parse import (AudioProcessorItems, ImageEmbeddingItems,
27 MultiModalDataItems, MultiModalDataParser)
28 from vllm.multimodal.processing import (BaseMultiModalProcessor,
29 BaseProcessingInfo, PromptReplacement,
30 - PromptUpdate)
30 + PromptUpdate, ResolvedPromptUpdate)
31 from vllm.multimodal.profiling import BaseDummyInputsBuilder
32 from vllm.sequence import IntermediateTensors
33 from vllm.utils import is_list_of
@@ -850,6 +850,25 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
850 ),
851 ]
852
853 + def _recompute_cached_prompt_update(
854 + self,
855 + cached_update: ResolvedPromptUpdate,
856 + new_item_idx: int,
857 + ) -> ResolvedPromptUpdate:
858 + new_update = super()._recompute_cached_prompt_update(
859 + cached_update,
860 + new_item_idx,
861 + )
862 +
863 + if cached_update.modality == "image":
864 + image_tokens: list[str] = self.info.image_tokens # type: ignore
865 + new_update = new_update.with_target(image_tokens[new_item_idx])
866 + elif cached_update.modality == "audio":
867 + audio_tokens: list[str] = self.info.audio_tokens # type: ignore
868 + new_update = new_update.with_target(audio_tokens[new_item_idx])
869 +
870 + return new_update
871 +
872
873 @MULTIMODAL_REGISTRY.register_processor(
874 Phi4MMMultiModalProcessor,
vllm/model_executor/models/tarsier.py
+4 -3
@@ -25,12 +25,13 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
25 from vllm.model_executor.models.llava import LlavaDummyInputsBuilder
26 from vllm.model_executor.sampling_metadata import SamplingMetadata
27 from vllm.multimodal import MULTIMODAL_REGISTRY
28 +from vllm.multimodal.cache import BaseMultiModalProcessorCache
29 from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
30 from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
31 ImageSize, MultiModalDataItems)
32 from vllm.multimodal.processing import (BaseMultiModalProcessor,
32 - BaseProcessingInfo, ProcessingCache,
33 - PromptReplacement, PromptUpdate)
33 + BaseProcessingInfo, PromptReplacement,
34 + PromptUpdate)
35 from vllm.multimodal.profiling import BaseDummyInputsBuilder
36 from vllm.sequence import IntermediateTensors
37 from vllm.utils.jsontree import json_map_leaves
@@ -332,7 +333,7 @@ def _build_tarsier_hf_processor(
333 info: _I_Tarsier,
334 dummy_inputs: BaseDummyInputsBuilder[_I_Tarsier],
335 *,
335 - cache: Optional[ProcessingCache] = None,
336 + cache: Optional[BaseMultiModalProcessorCache] = None,
337 ) -> BaseMultiModalProcessor:
338 if isinstance(info, TarsierProcessingInfo):
339 return TarsierMultiModalProcessor(
vllm/multimodal/cache.py
+390 -15
@@ -1,11 +1,12 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 import sys
4 -from collections.abc import Mapping
5 -from dataclasses import dataclass
6 -from typing import TypeVar, Union
4 +from abc import ABC, abstractmethod
5 +from collections.abc import Mapping, Sequence
6 +from typing import TYPE_CHECKING, Generic, Optional, TypeVar, Union
7
8 import torch
9 +from typing_extensions import TypeAlias, override
10
11 from vllm.logger import init_logger
12 from vllm.utils import GiB_bytes, LRUCache
@@ -15,24 +16,67 @@ from .inputs import (MultiModalFieldElem, MultiModalKwargs,
16 MultiModalKwargsItem, MultiModalKwargsItems,
17 NestedTensors)
18
19 +if TYPE_CHECKING:
20 + from vllm.config import ModelConfig, VllmConfig
21 +
22 + from .processing import ResolvedPromptUpdate
23 + from .registry import MultiModalRegistry
24 +
25 logger = init_logger(__name__)
26
27
21 -@dataclass
22 -class MultiModalCacheItemMetadata:
23 - size: int
28 +class MultiModalProcessorCacheItem:
29 + """
30 + The data to store inside `MultiModalProcessorOnlyCache`.
31
25 - @classmethod
26 - def wraps(cls, value: "MultiModalCacheValue"):
27 - return cls(size=MultiModalCache.get_item_size(value))
32 + Args:
33 + item: The processed tensor data corresponding to a multi-modal item.
34 + prompt_updates: The prompt updates corresponding to `item`.
35 + """
36 +
37 + def __init__(
38 + self,
39 + item: MultiModalKwargsItem,
40 + prompt_updates: Sequence["ResolvedPromptUpdate"],
41 + ) -> None:
42 + super().__init__()
43 +
44 + self.item = item
45 + self.prompt_updates = prompt_updates
46 +
47 +
48 +class MultiModalProcessorCacheItemMetadata:
49 + """
50 + The metadata to store inside `MultiModalProcessorSenderCache`.
51 +
52 + Args:
53 + item: The processed tensor data corresponding to a multi-modal item.
54 + Since P1 already stores the tensor data, we only store its size
55 + metadata in P0 to reduce memory usage. The size metadata is still
56 + needed to keep the same cache eviction policy as P0.
57 + prompt_updates: The prompt updates corresponding to `item`.
58 + This needs to stay on P0 because for some models, they are
59 + dependent on the processed tensor data (cached on P1).
60 + """
61 +
62 + def __init__(
63 + self,
64 + item: MultiModalKwargsItem,
65 + prompt_updates: Sequence["ResolvedPromptUpdate"],
66 + ) -> None:
67 + super().__init__()
68 +
69 + self.item_size = MultiModalCache.get_item_size(item)
70 + self.prompt_updates = prompt_updates
71
72
73 MultiModalCacheValue = Union[
74 + MultiModalProcessorCacheItem,
75 + MultiModalProcessorCacheItemMetadata,
76 MultiModalKwargsItems,
77 MultiModalKwargsItem,
78 MultiModalKwargs,
79 Mapping[str, NestedTensors],
35 - MultiModalCacheItemMetadata,
80 ]
81
82 _V = TypeVar("_V", bound=MultiModalCacheValue)
@@ -47,8 +91,10 @@ class MultiModalCache:
91 *,
92 debug: bool = False,
93 ) -> int:
50 - if isinstance(leaf, MultiModalFieldElem):
51 - return cls.get_item_size(leaf.data) # type: ignore
94 + if isinstance(leaf, MultiModalProcessorCacheItem):
95 + return cls.get_leaf_size(leaf.item)
96 + if isinstance(leaf, MultiModalProcessorCacheItemMetadata):
97 + return leaf.item_size
98
99 # These are not subclasses of dict
100 if isinstance(leaf, MultiModalKwargsItems):
@@ -58,13 +104,13 @@ class MultiModalCache:
104 if isinstance(leaf, MultiModalKwargs):
105 return cls.get_item_size(leaf.data) # type: ignore
106
107 + if isinstance(leaf, MultiModalFieldElem):
108 + return cls.get_item_size(leaf.data) # type: ignore
109 +
110 # sys.getsizeof doesn't work for tensors
111 if isinstance(leaf, torch.Tensor):
112 return leaf.nbytes
113
65 - if isinstance(leaf, MultiModalCacheItemMetadata):
66 - return leaf.size
67 -
114 return sys.getsizeof(leaf)
115
116 @classmethod
@@ -98,3 +144,332 @@ class MultiModalCache:
144 GiB_bytes * capacity_gb,
145 getsizeof=lambda x: cls.get_item_size(x, debug=debug),
146 )
147 +
148 +
149 +_I = TypeVar("_I", contravariant=True)
150 +_O = TypeVar("_O", covariant=True)
151 +
152 +
153 +class BaseMultiModalCache(ABC, Generic[_I, _O]):
154 + """
155 + Abstract base class to read/write multi-modal items from cache.
156 +
157 + The idea of multi-modal caching is based on having a client and server
158 + where the client executes in the frontend process (=P0) and
159 + the server in the core process (=P1). The data flow is as follows:
160 +
161 + ```
162 + is_cached() x N get_and_update()
163 + P0: From API -----------------> -----------------> To P1
164 +
165 + get_and_update()
166 + P1: From P0 -----------------> To model
167 + ```
168 +
169 + `is_cached()` can be called any number of times in P0. However,
170 + `get_and_update()` must be called in P0 and P1 one after another
171 + so that their cache eviction order remains the same.
172 +
173 + This ensures that the keys in P0 and P1 caches are mirrored,
174 + allowing us to determine whether a key is cached in P1 by looking
175 + up the P0 cache, without having to communicate with P1.
176 + """
177 +
178 + @abstractmethod
179 + def get_and_update_item(
180 + self,
181 + mm_item: _I,
182 + mm_hash: str,
183 + ) -> _O:
184 + """
185 + Possibly update a multi-modal item based on whether it is
186 + in the underlying cache.
187 +
188 + This update is done out-of-place and updates the cache eviction order.
189 +
190 + Args:
191 + mm_item: The multi-modal item to update.
192 + mm_hash: The hash of `mm_item`.
193 +
194 + Returns:
195 + The update multi-modal item.
196 + """
197 + raise NotImplementedError
198 +
199 + def get_and_update(
200 + self,
201 + mm_items: Sequence[_I],
202 + mm_hashes: list[str],
203 + ) -> list[_O]:
204 + """
205 + Possibly update a sequence of multi-modal items based on whether they
206 + are in the underlying cache.
207 +
208 + This update is done out-of-place and updates the cache eviction order.
209 +
210 + Args:
211 + mm_items: The multi-modal items to update.
212 + mm_hashes: The hash of each item in `mm_items`.
213 +
214 + Returns:
215 + A new list of updated multi-modal items.
216 + """
217 + assert len(mm_items) == len(mm_hashes)
218 +
219 + return [
220 + self.get_and_update_item(mm_item, mm_hash)
221 + for mm_item, mm_hash in zip(mm_items, mm_hashes)
222 + ]
223 +
224 + @abstractmethod
225 + def clear_cache(self) -> None:
226 + """Clear the underlying cache."""
227 + raise NotImplementedError
228 +
229 +
230 +MultiModalProcessorCacheInItem: TypeAlias = \
231 + Optional[tuple[MultiModalKwargsItem, Sequence["ResolvedPromptUpdate"]]]
232 +
233 +
234 +MultiModalProcessorCacheOutItem: TypeAlias = \
235 + tuple[Optional[MultiModalKwargsItem], Sequence["ResolvedPromptUpdate"]]
236 +
237 +
238 +class BaseMultiModalProcessorCache(
239 + BaseMultiModalCache[MultiModalProcessorCacheInItem,
240 + MultiModalProcessorCacheOutItem]):
241 + """The required interface for caches on P0."""
242 +
243 + @abstractmethod
244 + def is_cached_item(self, mm_hash: str) -> bool:
245 + """
246 + Check whether a multi-modal item is
247 + in the underlying cache.
248 +
249 + This **DOES NOT** update the cache eviction order.
250 +
251 + Args:
252 + mm_hash: The hash of the item to check.
253 +
254 + Returns:
255 + `True` if the item is cached, otherwise `False`.
256 + """
257 + raise NotImplementedError
258 +
259 + def is_cached(self, mm_hashes: list[str]) -> list[bool]:
260 + """
261 + Check whether a sequence of multi-modal items are
262 + in the underlying cache.
263 +
264 + This **DOES NOT** update the cache eviction order.
265 +
266 + Args:
267 + mm_hashes: The hash of each item to check.
268 +
269 + Returns:
270 + For each item, `True` if the item is cached, otherwise `False`.
271 + """
272 + return [self.is_cached_item(mm_hash) for mm_hash in mm_hashes]
273 +
274 +
275 +class MultiModalProcessorOnlyCache(BaseMultiModalProcessorCache):
276 + """
277 + The cache which is used on P0 when IPC caching is disabled.
278 +
279 + How to update each item:
280 +
281 + - If the item is in the cache, replace the input with the cached item.
282 + - If the item is not in the cache, store that item (which includes
283 + tensor data and metadata) into the cache, and return the input.
284 + """
285 +
286 + def __init__(self, model_config: "ModelConfig") -> None:
287 + super().__init__()
288 +
289 + mm_config = model_config.get_multimodal_config()
290 +
291 + self._cache = MultiModalCache.get_lru_cache(
292 + mm_config.mm_processor_cache_gb,
293 + MultiModalProcessorCacheItem,
294 + )
295 +
296 + @override
297 + def is_cached_item(self, mm_hash: str) -> bool:
298 + return mm_hash in self._cache
299 +
300 + @override
301 + def get_and_update_item(
302 + self,
303 + mm_item: MultiModalProcessorCacheInItem,
304 + mm_hash: str,
305 + ) -> MultiModalProcessorCacheOutItem:
306 + if (cached_item := self._cache.get(mm_hash)) is not None:
307 + return cached_item.item, cached_item.prompt_updates
308 +
309 + assert mm_item is not None, f"Expected a cached item for {mm_hash=}"
310 +
311 + self._cache[mm_hash] = MultiModalProcessorCacheItem(*mm_item)
312 +
313 + return mm_item
314 +
315 + @override
316 + def clear_cache(self) -> None:
317 + self._cache.clear()
318 +
319 +
320 +class MultiModalProcessorSenderCache(BaseMultiModalProcessorCache):
321 + """
322 + The cache which is used on P0 when IPC caching is enabled.
323 +
324 + How to update each item:
325 +
326 + - If the item is already in the cache, clear the input to avoid
327 + unnecessary IPC.
328 +
329 + - If the item is not in the cache, store the metadata of that item so
330 + that the eviction policy remains the same as the cache on P1,
331 + and return the input.
332 + By only storing the metadata, we avoid keeping the data itself in
333 + memory inside P0.
334 + """
335 +
336 + def __init__(self, model_config: "ModelConfig") -> None:
337 + super().__init__()
338 +
339 + mm_config = model_config.get_multimodal_config()
340 +
341 + self._cache = MultiModalCache.get_lru_cache(
342 + mm_config.mm_processor_cache_gb,
343 + MultiModalProcessorCacheItemMetadata,
344 + )
345 +
346 + @override
347 + def is_cached_item(self, mm_hash: str) -> bool:
348 + return mm_hash in self._cache
349 +
350 + @override
351 + def get_and_update_item(
352 + self,
353 + mm_item: MultiModalProcessorCacheInItem,
354 + mm_hash: str,
355 + ) -> MultiModalProcessorCacheOutItem:
356 + if (cached_item := self._cache.get(mm_hash)) is not None:
357 + return None, cached_item.prompt_updates
358 +
359 + assert mm_item is not None, f"Expected a cached item for {mm_hash=}"
360 +
361 + self._cache[mm_hash] = MultiModalProcessorCacheItemMetadata(*mm_item)
362 +
363 + return mm_item
364 +
365 + @override
366 + def clear_cache(self) -> None:
367 + self._cache.clear()
368 +
369 +
370 +def _enable_processor_cache(
371 + model_config: "ModelConfig",
372 + mm_registry: "MultiModalRegistry",
373 +) -> bool:
374 + if not mm_registry.supports_multimodal_inputs(model_config):
375 + return False
376 +
377 + mm_config = model_config.get_multimodal_config()
378 + return mm_config.mm_processor_cache_gb > 0
379 +
380 +
381 +def _enable_ipc_cache(vllm_config: "VllmConfig") -> bool:
382 + parallel_config = vllm_config.parallel_config
383 + supports_ipc_cache = (parallel_config.data_parallel_size == 1
384 + or parallel_config.data_parallel_external_lb)
385 +
386 + return supports_ipc_cache
387 +
388 +
389 +def processor_cache_from_config(
390 + vllm_config: "VllmConfig",
391 + mm_registry: "MultiModalRegistry",
392 +) -> Optional[BaseMultiModalProcessorCache]:
393 + """Return a `BaseMultiModalProcessorCache`, if enabled."""
394 + model_config = vllm_config.model_config
395 +
396 + if not _enable_processor_cache(model_config, mm_registry):
397 + return None
398 +
399 + if not _enable_ipc_cache(vllm_config):
400 + return MultiModalProcessorOnlyCache(model_config)
401 +
402 + return MultiModalProcessorSenderCache(model_config)
403 +
404 +
405 +def processor_only_cache_from_config(
406 + model_config: "ModelConfig",
407 + mm_registry: "MultiModalRegistry",
408 +):
409 + """Return a `MultiModalProcessorOnlyCache`, if enabled."""
410 + if not _enable_processor_cache(model_config, mm_registry):
411 + return None
412 +
413 + return MultiModalProcessorOnlyCache(model_config)
414 +
415 +
416 +class BaseMultiModalReceiverCache(
417 + BaseMultiModalCache[Optional[MultiModalKwargsItem],
418 + MultiModalKwargsItem]):
419 + """The required interface for caches on P1."""
420 +
421 +
422 +class MultiModalReceiverCache(BaseMultiModalReceiverCache):
423 + """
424 + The cache which is used on P1 when IPC caching is enabled.
425 +
426 + How to update each item:
427 +
428 + - If the item is in the cache, replace the input with the cached item.
429 + - If the item is not in the cache, store that item (which includes tensor
430 + data) into the cache, and return the input.
431 + """
432 +
433 + def __init__(self, model_config: "ModelConfig") -> None:
434 + super().__init__()
435 +
436 + mm_config = model_config.get_multimodal_config()
437 +
438 + self._cache = MultiModalCache.get_lru_cache(
439 + mm_config.mm_processor_cache_gb,
440 + MultiModalKwargsItem,
441 + )
442 +
443 + @override
444 + def get_and_update_item(
445 + self,
446 + mm_item: Optional[MultiModalKwargsItem],
447 + mm_hash: str,
448 + ) -> MultiModalKwargsItem:
449 + if (cached_item := self._cache.get(mm_hash)) is not None:
450 + return cached_item
451 +
452 + assert mm_item is not None, f"Expected a cached item for {mm_hash=}"
453 +
454 + self._cache[mm_hash] = mm_item
455 + return mm_item
456 +
457 + @override
458 + def clear_cache(self) -> None:
459 + self._cache.clear()
460 +
461 +
462 +def receiver_cache_from_config(
463 + vllm_config: "VllmConfig",
464 + mm_registry: "MultiModalRegistry",
465 +) -> Optional[BaseMultiModalReceiverCache]:
466 + """Return a `BaseMultiModalReceiverCache`, if enabled."""
467 + model_config = vllm_config.model_config
468 +
469 + if not _enable_processor_cache(model_config, mm_registry):
470 + return None
471 +
472 + if not _enable_ipc_cache(vllm_config):
473 + return None
474 +
475 + return MultiModalReceiverCache(model_config)
vllm/multimodal/inputs.py
+28 -10
@@ -7,11 +7,11 @@ from collections.abc import Mapping, Sequence
7 from dataclasses import dataclass
8 from functools import partial
9 from itertools import accumulate
10 -from typing import (TYPE_CHECKING, Any, Literal, Optional, TypedDict, TypeVar,
11 - Union, cast, final)
10 +from typing import (TYPE_CHECKING, Any, Literal, Optional, TypedDict, Union,
11 + cast, final)
12
13 import numpy as np
14 -from typing_extensions import NotRequired, TypeAlias, deprecated
14 +from typing_extensions import NotRequired, TypeAlias, TypeVar, deprecated
15
16 from vllm.utils import LazyLoader, full_groupby, is_list_of
17 from vllm.utils.jsontree import JSONTree, json_map_leaves
@@ -668,7 +668,15 @@ class MultiModalKwargsItem(UserDict[str, MultiModalFieldElem]):
668 return {key: elem.data for key, elem in self.items()}
669
670
671 -class MultiModalKwargsItems(UserDict[str, Sequence[MultiModalKwargsItem]]):
671 +_I = TypeVar(
672 + "_I",
673 + MultiModalKwargsItem,
674 + Optional[MultiModalKwargsItem],
675 + default=MultiModalKwargsItem,
676 +)
677 +
678 +
679 +class MultiModalKwargsItems(UserDict[str, Sequence[_I]]):
680 """
681 A dictionary of
682 [`MultiModalKwargsItem`][vllm.multimodal.inputs.MultiModalKwargsItem]s
@@ -714,27 +722,37 @@ class MultiModalKwargsItems(UserDict[str, Sequence[MultiModalKwargsItem]]):
722 items_by_modality = full_groupby(items, key=lambda x: x.modality)
723 return MultiModalKwargsItems(items_by_modality)
724
717 - def __getitem__(self, modality: str):
725 + def __getitem__(self, modality: str) -> Sequence[_I]:
726 if modality not in self:
727 raise KeyError(f"Modality {modality!r} not found. "
728 f"Available modalities: {set(self.keys())}")
729
722 - return super().__getitem__(modality)
730 + return super().__getitem__(modality) # type: ignore[return-value]
731
732 def get_data(self, *, pin_memory: bool = False) -> "MultiModalKwargs":
733 elems_by_key = defaultdict[str, list[MultiModalFieldElem]](list)
726 - for items in self.values():
727 - for item in items:
734 + for modality, items in self.items():
735 + for i, item in enumerate(items):
736 + if item is None:
737 + raise RuntimeError("Cannot build data from empty "
738 + f"mm_items[{modality}][{i}]")
739 +
740 for key, elem in item.items():
741 elems_by_key[key].append(elem)
742
743 return MultiModalKwargs({
744 key:
745 elems[0].field.reduce_data(elems, pin_memory=pin_memory)
734 - for key, elems in elems_by_key.items() if len(elems) > 0
746 + for key, elems in elems_by_key.items()
747 })
748
749
750 +MultiModalKwargsOptionalItems: TypeAlias = Union[
751 + MultiModalKwargsItems[MultiModalKwargsItem],
752 + MultiModalKwargsItems[Optional[MultiModalKwargsItem]],
753 +]
754 +
755 +
756 class MultiModalKwargs(UserDict[str, NestedTensors]):
757 """
758 A dictionary that represents the keyword arguments to
@@ -898,7 +916,7 @@ class MultiModalInputs(TypedDict):
916 token_type_ids: NotRequired[list[int]]
917 """The token type IDs of the prompt."""
918
901 - mm_kwargs: MultiModalKwargsItems
919 + mm_kwargs: MultiModalKwargsOptionalItems
920 """Keyword arguments to be directly passed to the model after batching."""
921
922 mm_hashes: "MultiModalHashDict"
vllm/multimodal/processing.py
+110 -77
@@ -4,7 +4,7 @@ from abc import ABC, abstractmethod
4 from collections import defaultdict
5 from collections.abc import (Callable, Generator, ItemsView, Iterable, Mapping,
6 Sequence)
7 -from dataclasses import dataclass, field
7 +from dataclasses import dataclass, field, replace
8 from enum import Enum
9 from functools import lru_cache
10 from typing import (TYPE_CHECKING, Generic, NamedTuple, Optional, Protocol,
@@ -20,12 +20,11 @@ from vllm.transformers_utils.tokenizer import (AnyTokenizer, decode_tokens,
20 encode_tokens)
21 from vllm.utils import flatten_2d_lists, full_groupby
22
23 -from .cache import MultiModalCache
23 from .hasher import MultiModalHasher
24 from .inputs import (MultiModalDataDict, MultiModalEncDecInputs,
25 MultiModalFieldConfig, MultiModalInputs,
26 MultiModalKwargsItem, MultiModalKwargsItems,
28 - PlaceholderRange)
27 + MultiModalKwargsOptionalItems, PlaceholderRange)
28 from .parse import (DictEmbeddingItems, EmbeddingItems, MultiModalDataItems,
29 MultiModalDataParser)
30
@@ -34,6 +33,7 @@ if TYPE_CHECKING:
33 from transformers.feature_extraction_utils import BatchFeature
34 from transformers.processing_utils import ProcessorMixin
35
36 + from .cache import BaseMultiModalProcessorCache
37 from .profiling import BaseDummyInputsBuilder
38
39 logger = init_logger(__name__)
@@ -557,6 +557,15 @@ class ResolvedPromptUpdate:
557
558 return self.iter_token_matches(prompt, tokenizer, start_idx=start_idx)
559
560 + def with_target(self, target: UpdateTarget):
561 + return replace(self, target=target)
562 +
563 + def with_content(self, content: PromptUpdateInfo):
564 + if not isinstance(content, PromptUpdateDetails):
565 + content = PromptUpdateDetails.from_seq(content)
566 +
567 + return replace(self, content=content)
568 +
569
570 class _TokenMatch(NamedTuple):
571 start_idx: int
@@ -865,21 +874,6 @@ def find_mm_placeholders(
874 return dict(full_groupby_modality(it))
875
876
868 -class ProcessingCache(MultiModalCache):
869 -
870 - def __init__(self, capacity_gb: float) -> None:
871 - super().__init__()
872 -
873 - self._cache = self.get_lru_cache(capacity_gb, MultiModalKwargsItem)
874 -
875 - self.get = self._cache.get
876 - self.put = self._cache.put
877 - self.reset = self._cache.clear
878 -
879 -
880 -_CacheItemOrHash = Union[MultiModalKwargsItem, str]
881 -
882 -
877 class BaseProcessingInfo:
878 """Base class to provide the information necessary for data processing."""
879
@@ -982,7 +976,7 @@ For an item `MultiModalPromptUpdates[k][i]`,
976
977
978 class MultiModalProcessingInfo(NamedTuple):
985 - kwargs: MultiModalKwargsItems
979 + kwargs: MultiModalKwargsOptionalItems
980 hashes: MultiModalHashes
981 prompt_updates: MultiModalPromptUpdates
982
@@ -994,11 +988,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
988 Not to be confused with `transformers.ProcessorMixin`.
989 """
990
997 - def __init__(self,
998 - info: _I,
999 - dummy_inputs: "BaseDummyInputsBuilder[_I]",
1000 - *,
1001 - cache: Optional[ProcessingCache] = None) -> None:
991 + def __init__(
992 + self,
993 + info: _I,
994 + dummy_inputs: "BaseDummyInputsBuilder[_I]",
995 + *,
996 + cache: Optional["BaseMultiModalProcessorCache"] = None,
997 + ) -> None:
998 super().__init__()
999
1000 self.info = info
@@ -1355,32 +1351,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1351
1352 return prompt_ids, mm_processed_data, False
1353
1358 - def _get_cache_missing_items(
1359 - self,
1360 - cache: ProcessingCache,
1361 - mm_data_items: MultiModalDataItems,
1362 - mm_hashes: MultiModalHashes,
1363 - ) -> tuple[dict[str, list[_CacheItemOrHash]], MultiModalDataItems]:
1364 - mm_cache_items_or_hashes: dict[str, list[_CacheItemOrHash]] = {
1365 - modality: [(h if (v := cache.get(h)) is None else v)
1366 - for h in hashes]
1367 - for modality, hashes in mm_hashes.items()
1368 - }
1369 -
1370 - mm_missing_idxs = {
1371 - modality: [
1372 - idx for idx, item_or_hash in enumerate(items_or_hashes)
1373 - if isinstance(item_or_hash, str)
1374 - ]
1375 - for modality, items_or_hashes in mm_cache_items_or_hashes.items()
1376 - }
1377 - mm_missing_data = {
1378 - modality: [mm_data_items[modality][idx] for idx in idxs]
1379 - for modality, idxs in mm_missing_idxs.items()
1380 - }
1381 -
1382 - return mm_cache_items_or_hashes, self._to_mm_items(mm_missing_data)
1383 -
1354 def _hash_mm_items(
1355 self,
1356 mm_items: MultiModalDataItems,
@@ -1401,28 +1371,92 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1371 for modality, items in mm_items.items()
1372 }
1373
1374 + def _get_cache_missing_items(
1375 + self,
1376 + cache: "BaseMultiModalProcessorCache",
1377 + mm_data_items: MultiModalDataItems,
1378 + mm_hashes: MultiModalHashes,
1379 + ) -> MultiModalDataItems:
1380 + mm_is_cached = {
1381 + modality: cache.is_cached(hashes)
1382 + for modality, hashes in mm_hashes.items()
1383 + }
1384 +
1385 + mm_missing_idxs = {
1386 + modality: [
1387 + idx for idx, item_is_cached in enumerate(items_is_cached)
1388 + if not item_is_cached
1389 + ]
1390 + for modality, items_is_cached in mm_is_cached.items()
1391 + }
1392 + mm_missing_data = {
1393 + modality: [mm_data_items[modality][idx] for idx in idxs]
1394 + for modality, idxs in mm_missing_idxs.items()
1395 + }
1396 +
1397 + return self._to_mm_items(mm_missing_data)
1398 +
1399 + def _recompute_cached_prompt_update(
1400 + self,
1401 + cached_update: ResolvedPromptUpdate,
1402 + new_item_idx: int,
1403 + ) -> ResolvedPromptUpdate:
1404 + """
1405 + Override this if other attributes of `ResolvedPromptUpdate`
1406 + also need to be recomputed after retrieving from the cache.
1407 + """
1408 + return replace(cached_update, item_idx=new_item_idx)
1409 +
1410 def _merge_mm_kwargs(
1411 self,
1406 - cache: ProcessingCache,
1407 - mm_cache_items_or_hashes: dict[str, list[_CacheItemOrHash]],
1412 + cache: "BaseMultiModalProcessorCache",
1413 + mm_hashes: MultiModalHashes,
1414 mm_missing_kwargs: MultiModalKwargsItems,
1409 - ) -> MultiModalKwargsItems:
1415 + mm_missing_prompt_updates: MultiModalPromptUpdates,
1416 + ) -> tuple[MultiModalKwargsOptionalItems, MultiModalPromptUpdates]:
1417 + # Need to calculate this at the beginning to avoid skipping cache logic
1418 + # for subsequently repeated items in the same modality
1419 + mm_is_cached = {
1420 + modality: cache.is_cached(hashes)
1421 + for modality, hashes in mm_hashes.items()
1422 + }
1423 +
1424 mm_missing_next_idx = defaultdict[str, int](lambda: 0)
1425
1412 - merged_items = defaultdict[str, list[MultiModalKwargsItem]](list)
1413 - for modality, items_or_hashes in mm_cache_items_or_hashes.items():
1414 - for item_or_hash in items_or_hashes:
1415 - if isinstance(item_or_hash, str):
1416 - kw_item = mm_missing_kwargs[modality][
1417 - mm_missing_next_idx[modality]]
1418 - cache.put(item_or_hash, kw_item)
1426 + merged_kwargs = defaultdict[str,
1427 + list[Optional[MultiModalKwargsItem]]](list)
1428 + merged_prompt_updates = defaultdict[
1429 + str, list[Sequence[ResolvedPromptUpdate]]](list)
1430 + for modality, hashes in mm_hashes.items():
1431 + missing_kwargs = mm_missing_kwargs.get(modality, [])
1432 + missing_prompt_updates = mm_missing_prompt_updates.get(
1433 + modality, [])
1434 +
1435 + for item_idx, item_hash in enumerate(hashes):
1436 + kwargs: Optional[MultiModalKwargsItem]
1437 + if not mm_is_cached[modality][item_idx]:
1438 + missing_next_idx = mm_missing_next_idx[modality]
1439 + kwargs = missing_kwargs[missing_next_idx]
1440 + updates = missing_prompt_updates[missing_next_idx]
1441 +
1442 mm_missing_next_idx[modality] += 1
1443 +
1444 + item = kwargs, updates
1445 else:
1421 - kw_item = item_or_hash
1446 + item = None
1447 +
1448 + kwargs, updates = cache.get_and_update_item(item, item_hash)
1449 +
1450 + merged_kwargs[modality].append(kwargs)
1451 + merged_prompt_updates[modality].append([
1452 + self._recompute_cached_prompt_update(update, item_idx)
1453 + for update in updates
1454 + ])
1455
1423 - merged_items[modality].append(kw_item)
1456 + mm_kwargs = MultiModalKwargsItems(merged_kwargs)
1457 + mm_prompt_updates = dict(merged_prompt_updates)
1458
1425 - return MultiModalKwargsItems(merged_items)
1459 + return mm_kwargs, mm_prompt_updates
1460
1461 def _apply_hf_processor(
1462 self,
@@ -1490,10 +1524,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1524
1525 mm_hashes = self._hash_mm_items(mm_data_items, hf_processor_mm_kwargs,
1526 tokenization_kwargs)
1493 - (
1494 - mm_cache_items_or_hashes,
1495 - mm_missing_data_items,
1496 - ) = self._get_cache_missing_items(
1527 +
1528 + mm_missing_data_items = self._get_cache_missing_items(
1529 cache=cache,
1530 mm_data_items=mm_data_items,
1531 mm_hashes=mm_hashes,
@@ -1520,16 +1552,17 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1552 hf_processor_mm_kwargs),
1553 )
1554
1523 - mm_kwargs = self._merge_mm_kwargs(
1524 - cache,
1525 - mm_cache_items_or_hashes=mm_cache_items_or_hashes,
1526 - mm_missing_kwargs=mm_missing_kwargs,
1555 + mm_missing_prompt_updates = self._get_mm_prompt_updates(
1556 + mm_missing_data_items,
1557 + hf_processor_mm_kwargs,
1558 + mm_missing_kwargs,
1559 )
1560
1529 - mm_prompt_updates = self._get_mm_prompt_updates(
1530 - mm_data_items,
1531 - hf_processor_mm_kwargs,
1532 - mm_kwargs,
1561 + mm_kwargs, mm_prompt_updates = self._merge_mm_kwargs(
1562 + cache,
1563 + mm_hashes=mm_hashes,
1564 + mm_missing_kwargs=mm_missing_kwargs,
1565 + mm_missing_prompt_updates=mm_missing_prompt_updates,
1566 )
1567
1568 mm_info = MultiModalProcessingInfo(
@@ -1614,7 +1647,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1647
1648 def _validate_mm_kwargs(
1649 self,
1617 - mm_kwargs: MultiModalKwargsItems,
1650 + mm_kwargs: MultiModalKwargsOptionalItems,
1651 mm_item_counts: Mapping[str, int],
1652 ) -> None:
1653 for modality, item_count in mm_item_counts.items():
@@ -1655,7 +1688,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1688 self,
1689 mm_items: MultiModalDataItems,
1690 prompt_ids: list[int],
1658 - mm_kwargs: MultiModalKwargsItems,
1691 + mm_kwargs: MultiModalKwargsOptionalItems,
1692 mm_prompt_updates: MultiModalPromptUpdates,
1693 is_update_applied: bool,
1694 ) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
vllm/multimodal/profiling.py
+2 -2
@@ -13,7 +13,7 @@ import vllm.envs as envs
13 from vllm.logger import init_logger
14
15 from .inputs import (MultiModalDataDict, MultiModalEncDecInputs,
16 - MultiModalInputs, MultiModalKwargsItems,
16 + MultiModalInputs, MultiModalKwargsOptionalItems,
17 MultiModalPlaceholderDict)
18 from .processing import (BaseMultiModalProcessor, BaseProcessingInfo,
19 EncDecMultiModalProcessor)
@@ -43,7 +43,7 @@ class DummyDecoderData(NamedTuple):
43 """Dummy data used for profiling."""
44
45 prompt_token_ids: list[int]
46 - multi_modal_data: MultiModalKwargsItems
46 + multi_modal_data: MultiModalKwargsOptionalItems
47 multi_modal_placeholders: MultiModalPlaceholderDict
48
49
vllm/multimodal/registry.py
+36 -54
@@ -2,7 +2,6 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 from collections.abc import Mapping
4 from dataclasses import dataclass
5 -from functools import lru_cache
5 from typing import TYPE_CHECKING, Generic, Optional, Protocol, TypeVar
6
7 import torch.nn as nn
@@ -13,8 +12,9 @@ from vllm.transformers_utils.tokenizer import (AnyTokenizer,
12 cached_tokenizer_from_config)
13 from vllm.utils import ClassRegistry
14
16 -from .processing import (BaseMultiModalProcessor, BaseProcessingInfo,
17 - ProcessingCache)
15 +from .cache import (BaseMultiModalProcessorCache,
16 + processor_only_cache_from_config)
17 +from .processing import BaseMultiModalProcessor, BaseProcessingInfo
18 from .profiling import (BaseDummyInputsBuilder, DummyDecoderData,
19 DummyEncoderData, MultiModalProfiler)
20
@@ -65,7 +65,7 @@ class MultiModalProcessorFactory(Protocol[_I]):
65 info: _I,
66 dummy_inputs: BaseDummyInputsBuilder[_I],
67 *,
68 - cache: Optional[ProcessingCache] = None,
68 + cache: Optional[BaseMultiModalProcessorCache] = None,
69 ) -> BaseMultiModalProcessor[_I]:
70 ...
71
@@ -80,20 +80,13 @@ class _ProcessorFactories(Generic[_I]):
80 self,
81 ctx: InputProcessingContext,
82 *,
83 - cache: Optional[ProcessingCache] = None,
83 + cache: Optional[BaseMultiModalProcessorCache] = None,
84 ):
85 info = self.info(ctx)
86 dummy_inputs_builder = self.dummy_inputs(info)
87 return self.processor(info, dummy_inputs_builder, cache=cache)
88
89
90 -# Make sure a different cache is used for each model config
91 -# NOTE: ModelConfig is not hashable so it cannot be passed directly
92 -@lru_cache(maxsize=1)
93 -def _get_processor_cache(model_id: str, capacity_gb: int):
94 - return ProcessingCache(capacity_gb) if capacity_gb > 0 else None
95 -
96 -
90 class MultiModalRegistry:
91 """
92 A registry that dispatches data processing according to the model.
@@ -103,31 +96,6 @@ class MultiModalRegistry:
96 self._processor_factories = ClassRegistry[nn.Module,
97 _ProcessorFactories]()
98
106 - def _get_processor_cache(self, model_config: "ModelConfig"):
107 - model_id = model_config.model
108 - capacity_gb = model_config.mm_processor_cache_gb
109 - return _get_processor_cache(model_id, capacity_gb)
110 -
111 - def reset_processor_cache(self, model_config: "ModelConfig") -> bool:
112 - """Reset the multi-modal processing cache."""
113 - if processor_cache := self._get_processor_cache(model_config):
114 - processor_cache.reset()
115 -
116 - return True # Success
117 -
118 - def enable_mm_input_cache(self, model_config: "ModelConfig") -> bool:
119 - """Whether the multi-modal input cache should be enabled.
120 - NOTE: This is put under MultiModalRegistry on purpose to respect
121 - text-only mode for multimodal models.
122 - """
123 -
124 - if not self.supports_multimodal_inputs(model_config):
125 - return False
126 -
127 - mm_config = model_config.get_multimodal_config()
128 -
129 - return mm_config.mm_processor_cache_gb > 0
130 -
99 def supports_multimodal_inputs(self, model_config: "ModelConfig") -> bool:
100 """
101 Checks if the model supports multimodal inputs.
@@ -157,6 +125,8 @@ class MultiModalRegistry:
125 def get_max_tokens_per_item_by_modality(
126 self,
127 model_config: "ModelConfig",
128 + *,
129 + cache: Optional[BaseMultiModalProcessorCache] = None,
130 ) -> Mapping[str, int]:
131 """
132 Get the maximum number of tokens per data item from each modality based
@@ -165,11 +135,11 @@ class MultiModalRegistry:
135 if not model_config.is_multimodal_model:
136 return {}
137
168 - processor = self.create_processor(model_config, disable_cache=False)
138 + processor = self.create_processor(model_config, cache=cache)
139 profiler = MultiModalProfiler(processor)
140
141 seq_len = model_config.max_model_len
172 - mm_limits = self.get_mm_limits_per_prompt(model_config)
142 + mm_limits = self.get_mm_limits_per_prompt(model_config, cache=cache)
143
144 return profiler.get_mm_max_contiguous_tokens(
145 seq_len,
@@ -182,6 +152,8 @@ class MultiModalRegistry:
152 def get_max_tokens_per_item_by_nonzero_modality(
153 self,
154 model_config: "ModelConfig",
155 + *,
156 + cache: Optional[BaseMultiModalProcessorCache] = None,
157 ) -> Mapping[str, int]:
158 """
159 Get the maximum number of tokens per data item from each modality based
@@ -192,15 +164,19 @@ class MultiModalRegistry:
164 This is currently directly used only in V1 for profiling the memory
165 usage of a model.
166 """
195 - mm_limits = self.get_mm_limits_per_prompt(model_config)
167 + mm_limits = self.get_mm_limits_per_prompt(model_config, cache=cache)
168 + max_tokens_per_item = self.get_max_tokens_per_item_by_modality(
169 + model_config,
170 + cache=cache,
171 + )
172
173 return {
174 key: max_tokens_per_mm_item
199 - for key, max_tokens_per_mm_item in
200 - self.get_max_tokens_per_item_by_modality(model_config).items()
175 + for key, max_tokens_per_mm_item in max_tokens_per_item.items()
176 if mm_limits[key] > 0
177 }
178
179 + # TODO: Remove once V0 is gone
180 def get_max_tokens_by_modality(
181 self,
182 model_config: "ModelConfig",
@@ -209,14 +185,19 @@ class MultiModalRegistry:
185 Get the maximum number of tokens from each modality
186 for profiling the memory usage of a model.
187 """
212 - mm_limits = self.get_mm_limits_per_prompt(model_config)
188 + cache = processor_only_cache_from_config(model_config, self)
189 + mm_limits = self.get_mm_limits_per_prompt(model_config, cache=cache)
190 + max_tokens_per_item = self.get_max_tokens_per_item_by_modality(
191 + model_config,
192 + cache=cache,
193 + )
194
195 return {
196 key: mm_limits[key] * max_tokens_per_mm_item
216 - for key, max_tokens_per_mm_item in
217 - self.get_max_tokens_per_item_by_modality(model_config).items()
197 + for key, max_tokens_per_mm_item in max_tokens_per_item.items()
198 }
199
200 + # TODO: Remove once V0 is gone
201 def get_max_multimodal_tokens(self, model_config: "ModelConfig") -> int:
202 """
203 Get the maximum number of multi-modal tokens
@@ -227,6 +208,8 @@ class MultiModalRegistry:
208 def get_mm_limits_per_prompt(
209 self,
210 model_config: "ModelConfig",
211 + *,
212 + cache: Optional[BaseMultiModalProcessorCache] = None,
213 ) -> Mapping[str, int]:
214 """
215 Get the maximum number of multi-modal input instances for each modality
@@ -235,7 +218,7 @@ class MultiModalRegistry:
218 if not model_config.is_multimodal_model:
219 return {}
220
238 - processor = self.create_processor(model_config, disable_cache=False)
221 + processor = self.create_processor(model_config, cache=cache)
222 profiler = MultiModalProfiler(processor)
223 return profiler.get_mm_limits()
224
@@ -303,7 +286,7 @@ class MultiModalRegistry:
286 model_config: "ModelConfig",
287 *,
288 tokenizer: Optional[AnyTokenizer] = None,
306 - disable_cache: Optional[bool] = None,
289 + cache: Optional[BaseMultiModalProcessorCache] = None,
290 ) -> BaseMultiModalProcessor[BaseProcessingInfo]:
291 """
292 Create a multi-modal processor for a specific model and tokenizer.
@@ -311,15 +294,10 @@ class MultiModalRegistry:
294 if not model_config.is_multimodal_model:
295 raise ValueError(f"{model_config.model} is not a multimodal model")
296
314 - if disable_cache is None:
315 - disable_cache = not model_config.enable_mm_processor_cache
316 -
297 model_cls = self._get_model_cls(model_config)
298 factories = self._processor_factories[model_cls]
299
300 ctx = self._create_processing_ctx(model_config, tokenizer)
321 - cache = None if disable_cache else self._get_processor_cache(
322 - model_config)
301
302 return factories.build_processor(ctx, cache=cache)
303
@@ -328,13 +306,15 @@ class MultiModalRegistry:
306 model_config: "ModelConfig",
307 seq_len: int,
308 mm_counts: Optional[Mapping[str, int]] = None,
309 + *,
310 + cache: Optional[BaseMultiModalProcessorCache] = None,
311 ) -> DummyDecoderData:
312 """
313 Create dummy data for profiling the memory usage of a model.
314
315 The model is identified by ``model_config``.
316 """
337 - processor = self.create_processor(model_config, disable_cache=False)
317 + processor = self.create_processor(model_config, cache=cache)
318 profiler = MultiModalProfiler(processor)
319 dummy_data = profiler.get_decoder_dummy_data(seq_len, mm_counts)
320
@@ -352,13 +332,15 @@ class MultiModalRegistry:
332 model_config: "ModelConfig",
333 seq_len: int,
334 mm_counts: Optional[Mapping[str, int]] = None,
335 + *,
336 + cache: Optional[BaseMultiModalProcessorCache] = None,
337 ) -> DummyEncoderData:
338 """
339 Create dummy data for profiling the memory usage of a model.
340
341 The model is identified by ``model_config``.
342 """
361 - processor = self.create_processor(model_config, disable_cache=False)
343 + processor = self.create_processor(model_config, cache=cache)
344 profiler = MultiModalProfiler(processor)
345 dummy_data = profiler.get_encoder_dummy_data(seq_len, mm_counts)
346
vllm/v1/engine/async_llm.py
+1 -2
@@ -597,8 +597,7 @@ class AsyncLLM(EngineClient):
597 await asyncio.gather(*coros)
598
599 async def reset_mm_cache(self) -> None:
600 - self.processor.mm_registry.reset_processor_cache(self.model_config)
601 - self.processor.mm_input_cache_client.reset()
600 + self.processor.clear_cache()
601 await self.engine_core.reset_mm_cache_async()
602
603 async def reset_prefix_cache(self,
vllm/v1/engine/core.py
+10 -7
@@ -22,6 +22,7 @@ from vllm.logger import init_logger
22 from vllm.logging_utils.dump_input import dump_engine_exception
23 from vllm.lora.request import LoRARequest
24 from vllm.multimodal import MULTIMODAL_REGISTRY
25 +from vllm.multimodal.cache import receiver_cache_from_config
26 from vllm.tasks import POOLING_TASKS, SupportedTask
27 from vllm.transformers_utils.config import (
28 maybe_register_config_serialize_by_value)
@@ -38,7 +39,6 @@ from vllm.v1.engine import (EngineCoreOutputs, EngineCoreRequest,
39 EngineCoreRequestType,
40 ReconfigureDistributedRequest, ReconfigureRankType,
41 UtilityOutput, UtilityResult)
41 -from vllm.v1.engine.mm_input_cache import MultiModalInputCacheServer
42 from vllm.v1.engine.utils import EngineHandshakeMetadata, EngineZmqAddresses
43 from vllm.v1.executor.abstract import Executor
44 from vllm.v1.kv_cache_interface import KVCacheConfig
@@ -128,8 +128,9 @@ class EngineCore:
128 )
129 self.use_spec_decode = vllm_config.speculative_config is not None
130
131 - self.mm_input_cache_server = MultiModalInputCacheServer(
132 - vllm_config.model_config, MULTIMODAL_REGISTRY)
131 + self.mm_registry = mm_registry = MULTIMODAL_REGISTRY
132 + self.mm_receiver_cache = receiver_cache_from_config(
133 + vllm_config, mm_registry)
134
135 # Setup batch queue for pipeline parallelism.
136 # Batch queue for scheduled batches. This enables us to asynchronously
@@ -370,7 +371,8 @@ class EngineCore:
371 logger.warning("Resetting the multi-modal cache when requests are "
372 "in progress may lead to desynced internal caches.")
373
373 - self.mm_input_cache_server.reset()
374 + if self.mm_receiver_cache is not None:
375 + self.mm_receiver_cache.clear_cache()
376
377 def reset_prefix_cache(self):
378 self.scheduler.reset_prefix_cache()
@@ -435,10 +437,11 @@ class EngineCore:
437 assert request.mm_kwargs is not None
438
439 # Note on thread safety: no race condition.
438 - # `mm_input_cache_server` is reset at the end of LLMEngine init,
440 + # `mm_receiver_cache` is reset at the end of LLMEngine init,
441 # and will only accessed in the input processing thread afterwards.
440 - request.mm_kwargs = self.mm_input_cache_server.get_and_update(
441 - request.mm_kwargs, request.mm_hashes)
442 + if self.mm_receiver_cache is not None:
443 + request.mm_kwargs = self.mm_receiver_cache.get_and_update(
444 + request.mm_kwargs, request.mm_hashes)
445
446 req = Request.from_engine_core_request(request,
447 self.request_block_hasher)
vllm/v1/engine/llm_engine.py
+1 -2
@@ -271,8 +271,7 @@ class LLMEngine:
271 self.engine_core.profile(False)
272
273 def reset_mm_cache(self):
274 - self.processor.mm_registry.reset_processor_cache(self.model_config)
275 - self.processor.mm_input_cache_client.reset()
274 + self.processor.clear_cache()
275 self.engine_core.reset_mm_cache()
276
277 def reset_prefix_cache(self, device: Optional[Device] = None):
vllm/v1/engine/mm_input_cache.py deleted
-121
@@ -1,121 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -from collections.abc import Sequence
4 -from typing import TYPE_CHECKING, Optional
5 -
6 -from vllm.multimodal import MultiModalRegistry
7 -from vllm.multimodal.cache import MultiModalCache, MultiModalCacheItemMetadata
8 -from vllm.multimodal.inputs import MultiModalKwargsItem
9 -from vllm.utils import is_list_of
10 -
11 -if TYPE_CHECKING:
12 - from vllm.config import ModelConfig
13 -
14 -# The idea of multimodal input caching is based on having a client and
15 -# a server, where the client executes in the frontend process (=P0) and the
16 -# server in the core process (=P1).
17 -#
18 -# -- P0:
19 -# - BaseMultiModalProcessor calls MultiModalHasher to get the `mm_hash` of
20 -# each input multi-modal item (e.g. image),
21 -# - BaseMultiModalProcessor processes the input items into `mm_kwargs`,
22 -# which are MultiModalKwargsItem instances that each correspond to an
23 -# input multi-modal item.
24 -# - MultiModalInputCacheClient accepts the `mm_kwargs` and corresponding
25 -# `mm_hash` for each item. It stores the `mm_hash` as keys and the size
26 -# of `mm_kwargs`, but not the `mm_kwargs` themselves, to avoid taking
27 -# up additional memory in P0.
28 -# - The `mm_hash` is always sent to P1.
29 -# - The corresponding `mm_kwargs` are only sent to P1 if they are not cached
30 -# in MultiModalInputCacheServer.
31 -#
32 -# -- P1:
33 -# - If the `mm_hash` is cached (i.e. `mm_kwargs` are not sent from P0),
34 -# MultiModalInputCacheServer retrieves the corresponding `mm_kwargs`.
35 -# - If the `mm_hash` is not cached (i.e. `mm_kwargs` are sent from P0),
36 -# MultiModalInputCacheServer stores `mm_kwargs` under the key `mm_hash`.
37 -# - Either way, the `mm_hash` and corresponding `mm_kwargs` are sent to
38 -# the engine for model execution.
39 -#
40 -# Both Client and Server must perform cache update and eviction based on the
41 -# same item size. This ensures that the keys of MultiModalInputCacheClient
42 -# and MultiModalInputCacheServer are mirrored, allowing us to determine in P0
43 -# whether a key is cached in MultiModalInputCacheServer by querying
44 -# MultiModalInputCacheClient without having to communicate with P1.
45 -
46 -
47 -class MultiModalInputCacheClient:
48 - """Used by P0 to check whether multi-modal kwargs are cached in P1."""
49 -
50 - def __init__(self, model_config: "ModelConfig",
51 - mm_registry: MultiModalRegistry) -> None:
52 - super().__init__()
53 -
54 - self.enabled = mm_registry.enable_mm_input_cache(model_config)
55 - self.mm_cache = MultiModalCache.get_lru_cache(
56 - model_config.get_mm_input_cache_gb(),
57 - MultiModalCacheItemMetadata,
58 - )
59 -
60 - def get_and_update(
61 - self,
62 - mm_kwargs: Sequence[MultiModalKwargsItem],
63 - mm_hashes: list[str],
64 - ) -> list[Optional[MultiModalKwargsItem]]:
65 - if not self.enabled:
66 - return list(mm_kwargs)
67 -
68 - assert len(mm_kwargs) == len(mm_hashes)
69 -
70 - out_mm_items = list[Optional[MultiModalKwargsItem]]()
71 - for mm_item, mm_hash in zip(mm_kwargs, mm_hashes):
72 - if self.mm_cache.get(mm_hash) is not None:
73 - out_mm_items.append(None)
74 - else:
75 - self.mm_cache[mm_hash] = \
76 - MultiModalCacheItemMetadata.wraps(mm_item)
77 - out_mm_items.append(mm_item)
78 -
79 - return out_mm_items
80 -
81 - def reset(self) -> None:
82 - self.mm_cache.clear()
83 -
84 -
85 -class MultiModalInputCacheServer:
86 - """Used by P1 to avoid requiring past multi-modal kwargs from P0."""
87 -
88 - def __init__(self, model_config: "ModelConfig",
89 - mm_registry: MultiModalRegistry) -> None:
90 - super().__init__()
91 -
92 - self.enabled = mm_registry.enable_mm_input_cache(model_config)
93 - self.mm_cache = MultiModalCache.get_lru_cache(
94 - model_config.get_mm_input_cache_gb(),
95 - MultiModalKwargsItem,
96 - )
97 -
98 - def get_and_update(
99 - self,
100 - mm_kwargs: Sequence[Optional[MultiModalKwargsItem]],
101 - mm_hashes: list[str],
102 - ) -> list[MultiModalKwargsItem]:
103 - if not self.enabled:
104 - mm_kwargs_lst = list(mm_kwargs)
105 - assert is_list_of(mm_kwargs_lst, MultiModalKwargsItem)
106 - return mm_kwargs_lst
107 -
108 - assert len(mm_kwargs) == len(mm_hashes)
109 -
110 - out_mm_items = list[MultiModalKwargsItem]()
111 - for mm_item, mm_hash in zip(mm_kwargs, mm_hashes):
112 - if mm_item is None:
113 - out_mm_items.append(self.mm_cache[mm_hash])
114 - else:
115 - self.mm_cache[mm_hash] = mm_item
116 - out_mm_items.append(mm_item)
117 -
118 - return out_mm_items
119 -
120 - def reset(self) -> None:
121 - self.mm_cache.clear()
vllm/v1/engine/processor.py
+14 -15
@@ -11,6 +11,7 @@ from vllm.inputs.parse import split_enc_dec_inputs
11 from vllm.inputs.preprocess import InputPreprocessor
12 from vllm.lora.request import LoRARequest
13 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
14 +from vllm.multimodal.cache import processor_cache_from_config
15 from vllm.multimodal.inputs import MultiModalKwargsItem, PlaceholderRange
16 from vllm.multimodal.processing import EncDecMultiModalProcessor
17 from vllm.multimodal.utils import argsort_mm_positions
@@ -18,7 +19,6 @@ from vllm.pooling_params import PoolingParams
19 from vllm.sampling_params import SamplingParams
20 from vllm.transformers_utils.tokenizer_group import TokenizerGroup
21 from vllm.v1.engine import EngineCoreRequest
21 -from vllm.v1.engine.mm_input_cache import MultiModalInputCacheClient
22 from vllm.v1.structured_output.backend_guidance import (
23 validate_guidance_grammar)
24 from vllm.v1.structured_output.backend_lm_format_enforcer import (
@@ -47,16 +47,17 @@ class Processor:
47
48 self.generation_config_fields = (
49 self.model_config.try_get_generation_config())
50 - self.input_preprocessor = InputPreprocessor(self.model_config,
51 - self.tokenizer,
52 - mm_registry)
50
54 - self.mm_input_cache_client = MultiModalInputCacheClient(
55 - self.model_config, mm_registry)
51 + self.mm_registry = mm_registry
52 + self.mm_processor_cache = processor_cache_from_config(
53 + vllm_config, mm_registry)
54
57 - @property
58 - def mm_registry(self):
59 - return self.input_preprocessor.mm_registry
55 + self.input_preprocessor = InputPreprocessor(
56 + self.model_config,
57 + self.tokenizer,
58 + mm_registry,
59 + mm_processor_cache=self.mm_processor_cache,
60 + )
61
62 def _validate_logprobs(
63 self,
@@ -310,7 +311,7 @@ class Processor:
311 # in the input sequence.
312 sorted_mm_idxs = argsort_mm_positions(decoder_mm_positions)
313
313 - orig_sorted_mm_inputs = [
314 + sorted_mm_inputs = [
315 decoder_mm_inputs[modality][idx]
316 for modality, idx in sorted_mm_idxs
317 ]
@@ -323,11 +324,6 @@ class Processor:
324 for modality, idx in sorted_mm_idxs
325 ]
326
326 - sorted_mm_inputs = self.mm_input_cache_client.get_and_update(
327 - orig_sorted_mm_inputs,
328 - sorted_mm_hashes,
329 - )
330 -
327 return decoder_inputs.get("prompt"), EngineCoreRequest(
328 request_id=request_id,
329 prompt_token_ids=decoder_inputs["prompt_token_ids"],
@@ -415,3 +411,6 @@ class Processor:
411 # TODO: Find out how many placeholder tokens are there so we can
412 # check that chunked prefill does not truncate them
413 # max_batch_len = self.scheduler_config.max_num_batched_tokens
414 +
415 + def clear_cache(self) -> None:
416 + self.input_preprocessor.clear_cache()
vllm/v1/worker/gpu_model_runner.py
+3
@@ -2186,10 +2186,13 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
2186 max_items_per_batch: int,
2187 ) -> BatchedTensorInputs:
2188 """Dummy data for profiling and precompiling multimodal models."""
2189 + assert self.mm_budget is not None
2190 +
2191 dummy_decoder_data = self.mm_registry.get_decoder_dummy_data(
2192 model_config=self.model_config,
2193 seq_len=self.max_num_tokens,
2194 mm_counts={modality: 1},
2195 + cache=self.mm_budget.cache,
2196 )
2197 dummy_mm_data = dummy_decoder_data.multi_modal_data
2198
vllm/v1/worker/tpu_model_runner.py
+3
@@ -1813,10 +1813,13 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
1813 max_items_per_batch: int,
1814 ) -> BatchedTensorInputs:
1815 """Dummy data for profiling and precompiling multimodal models."""
1816 + assert self.mm_budget is not None
1817 +
1818 dummy_decoder_data = self.mm_registry.get_decoder_dummy_data(
1819 model_config=self.model_config,
1820 seq_len=self.max_num_tokens,
1821 mm_counts={modality: 1},
1822 + cache=self.mm_budget.cache,
1823 )
1824 dummy_mm_data = dummy_decoder_data.multi_modal_data
1825
vllm/v1/worker/utils.py
+7 -2
@@ -10,6 +10,7 @@ from vllm.attention.backends.abstract import AttentionBackend
10 from vllm.config import ModelConfig, SchedulerConfig
11 from vllm.model_executor.models.interfaces import MultiModalEmbeddings
12 from vllm.model_executor.models.utils import extract_layer_index
13 +from vllm.multimodal.cache import processor_only_cache_from_config
14 from vllm.multimodal.registry import MultiModalRegistry
15 from vllm.v1.attention.backends.utils import AttentionMetadataBuilder
16 from vllm.v1.core.encoder_cache_manager import compute_mm_encoder_budget
@@ -33,14 +34,18 @@ class MultiModalBudget:
34 self.model_config = model_config
35 self.scheduler_config = scheduler_config
36 self.mm_registry = mm_registry
37 + self.cache = cache = processor_only_cache_from_config(
38 + model_config, mm_registry)
39
40 self.max_model_len = model_config.max_model_len
41 self.max_num_reqs = scheduler_config.max_num_seqs
42
40 - self.mm_limits = mm_registry.get_mm_limits_per_prompt(model_config)
43 + self.mm_limits = mm_registry.get_mm_limits_per_prompt(model_config,
44 + cache=cache)
45
46 max_tokens_by_modality = mm_registry \
43 - .get_max_tokens_per_item_by_nonzero_modality(model_config)
47 + .get_max_tokens_per_item_by_nonzero_modality(model_config,
48 + cache=cache)
49
50 encoder_compute_budget, encoder_cache_size = compute_mm_encoder_budget(
51 scheduler_config,