A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[CI/Build] Fix multimodal tests (#22491)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Aug 8, 2025 at 15:31 UTC
1712543df6d0ebdc2cc9649e246ae983c92dabd3
4 files changed
+17
-15
vllm/engine/llm_engine.py
+2
-1
@@ -845,7 +845,8 @@ class LLMEngine:
845
846
def reset_mm_cache(self) -> bool:
847
"""Reset the multi-modal cache."""
848
- return self.input_preprocessor.mm_registry.reset_processor_cache()
848
+ return self.input_preprocessor.mm_registry.reset_processor_cache(
849
+ self.model_config)
850
851
def reset_prefix_cache(self, device: Optional[Device] = None) -> bool:
852
"""Reset prefix cache for all devices."""
vllm/multimodal/registry.py
+13
-12
@@ -2,6 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from collections.abc import Mapping
4
from dataclasses import dataclass
5
+from functools import lru_cache
6
from typing import TYPE_CHECKING, Generic, Optional, Protocol, TypeVar
7
8
import torch.nn as nn
@@ -86,6 +87,13 @@ class _ProcessorFactories(Generic[_I]):
87
return self.processor(info, dummy_inputs_builder, cache=cache)
88
89
90
+# Make sure a different cache is used for each model config
91
+# NOTE: ModelConfig is not hashable so it cannot be passed directly
92
+@lru_cache(maxsize=1)
93
+def _get_processor_cache(model_id: str, capacity_gb: int):
94
+ return ProcessingCache(capacity_gb) if capacity_gb > 0 else None
95
+
96
+
97
class MultiModalRegistry:
98
"""
99
A registry that dispatches data processing according to the model.
@@ -95,22 +103,15 @@ class MultiModalRegistry:
103
self._processor_factories = ClassRegistry[nn.Module,
104
_ProcessorFactories]()
105
98
- self._processor_cache: Optional[ProcessingCache] = None
99
-
106
def _get_processor_cache(self, model_config: "ModelConfig"):
107
+ model_id = model_config.model
108
capacity_gb = model_config.mm_processor_cache_gb
102
- if capacity_gb is None:
103
- return None # Overrides `disable_cache` argument
104
-
105
- if self._processor_cache is None:
106
- self._processor_cache = ProcessingCache(capacity_gb)
107
-
108
- return self._processor_cache
109
+ return _get_processor_cache(model_id, capacity_gb)
110
110
- def reset_processor_cache(self) -> bool:
111
+ def reset_processor_cache(self, model_config: "ModelConfig") -> bool:
112
"""Reset the multi-modal processing cache."""
112
- if self._processor_cache:
113
- self._processor_cache.reset()
113
+ if processor_cache := self._get_processor_cache(model_config):
114
+ processor_cache.reset()
115
116
return True # Success
117
vllm/v1/engine/async_llm.py
+1
-1
@@ -566,7 +566,7 @@ class AsyncLLM(EngineClient):
566
await self.engine_core.profile_async(False)
567
568
async def reset_mm_cache(self) -> None:
569
- self.processor.mm_registry.reset_processor_cache()
569
+ self.processor.mm_registry.reset_processor_cache(self.model_config)
570
self.processor.mm_input_cache_client.reset()
571
await self.engine_core.reset_mm_cache_async()
572
vllm/v1/engine/llm_engine.py
+1
-1
@@ -271,7 +271,7 @@ class LLMEngine:
271
self.engine_core.profile(False)
272
273
def reset_mm_cache(self):
274
- self.processor.mm_registry.reset_processor_cache()
274
+ self.processor.mm_registry.reset_processor_cache(self.model_config)
275
self.processor.mm_input_cache_client.reset()
276
self.engine_core.reset_mm_cache()
277