A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 01:00 UTC
[feat] Enable mm caching for transformers backend (#21358)
Signed-off-by: raushan <raushan@huggingface.co>
Raushan Turganbay committed
Jul 22, 2025 at 17:18 UTC
f38ee34a0a48b9acd67ea635be80e1d5ba992694
4 files changed
+7
-18
docs/models/supported_models.md
+1
-1
@@ -18,7 +18,7 @@ These models are what we list in [supported-text-models][supported-text-models]
18
19
### Transformers
20
21
-vLLM also supports model implementations that are available in Transformers. This does not currently work for all models, but most decoder language models and common vision language models are supported! Vision-language models currently accept only image inputs, and require setting `--disable_mm_preprocessor_cache` when running. Support for video inputs and caching of multi-modal preprocessors will be added in future releases.
21
+vLLM also supports model implementations that are available in Transformers. This does not currently work for all models, but most decoder language models and common vision language models are supported! Vision-language models currently accept only image inputs. Support for video inputs will be added in future releases.
22
23
To check if the modeling backend is Transformers, you can simply do this:
24
tests/models/multimodal/generation/test_common.py
-8
@@ -186,8 +186,6 @@ VLM_TEST_SETTINGS = {
186
image_size_factors=[(0.25, 0.5, 1.0)],
187
vllm_runner_kwargs={
188
"model_impl": "transformers",
189
- "disable_mm_preprocessor_cache": True,
190
- "enable_prefix_caching": False,
189
},
190
marks=[pytest.mark.core_model],
191
),
@@ -205,8 +203,6 @@ VLM_TEST_SETTINGS = {
203
# image_size_factors=[(0.25, 0.5, 1.0)],
204
# vllm_runner_kwargs={
205
# "model_impl": "transformers",
208
- # "disable_mm_preprocessor_cache": True,
209
- # "enable_prefix_caching": False,
206
# },
207
# marks=[pytest.mark.core_model],
208
# ),
@@ -223,8 +219,6 @@ VLM_TEST_SETTINGS = {
219
image_size_factors=[(0.25, 0.2, 0.15)],
220
vllm_runner_kwargs={
221
"model_impl": "transformers",
226
- "disable_mm_preprocessor_cache": True,
227
- "enable_prefix_caching": False,
222
},
223
marks=[large_gpu_mark(min_gb=32)],
224
),
@@ -239,8 +233,6 @@ VLM_TEST_SETTINGS = {
233
image_size_factors=[(0.25, 0.5, 1.0)],
234
vllm_runner_kwargs={
235
"model_impl": "auto",
242
- "disable_mm_preprocessor_cache": True,
243
- "enable_prefix_caching": False,
236
},
237
auto_cls=AutoModelForImageTextToText,
238
marks=[pytest.mark.core_model],
vllm/model_executor/models/transformers.py
+3
-6
@@ -315,11 +315,6 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
315
Apply HF Processor on prompt text and multi-modal data together,
316
outputting token IDs and processed tensors.
317
"""
318
- if return_mm_hashes:
319
- raise ValueError(
320
- "TransformersForMultimodalLM doesn't support mm hashing yet! "
321
- "Probably you didn't set `disable_mm_preprocessor_cache=True`")
322
-
318
if tokenization_kwargs is None:
319
tokenization_kwargs = {}
320
@@ -375,12 +370,14 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
370
num_image_patches),
371
)
372
373
+ mm_hashes = self._hash_mm_items(mm_items, hf_processor_mm_kwargs,
374
+ tokenization_kwargs)
375
return MultiModalInputs(
376
type="multimodal",
377
prompt=prompt,
378
prompt_token_ids=prompt_ids,
379
mm_kwargs=mm_kwargs,
383
- mm_hashes=None,
380
+ mm_hashes=mm_hashes,
381
mm_placeholders=mm_placeholders,
382
)
383
vllm/v1/core/kv_cache_utils.py
+3
-3
@@ -406,9 +406,9 @@ def need_extra_keys(request: Request) -> bool:
406
# Multimodal requests need to include the MM hash.
407
# LoRA requests need to include the LoRA ID.
408
# Request with provided cache salt need to include the salt.
409
- return bool(request.mm_positions) or (request.lora_request
410
- is not None) or (request.cache_salt
411
- is not None)
409
+ return bool(request.mm_hashes) or (request.lora_request
410
+ is not None) or (request.cache_salt
411
+ is not None)
412
413
414
def _gen_mm_extra_hash_keys(request: Request, start_token_idx: int,