[Bugfix] Skip MM processor cache inserts larger than capacity (#53016)
Signed-off-by: Prudhvivuda <prudhvi12042001@gmail.com>
Prudhvi Vuda committed
Aug 20, 2026 at 09:42 UTC
de216b6e6487b8b452805b582b242616ee616682
6 files changed
+108
-5
docs/configuration/optimization.md
+3
@@ -376,6 +376,9 @@ processes.
376
### Configuration
377
378
You can adjust the size of the cache by setting the value of `mm_processor_cache_gb` (default 4 GiB).
379
+A processed item larger than this budget is served uncached (with a warning)
380
+instead of failing engine startup; raise `mm_processor_cache_gb` if you want
381
+those items cached.
382
383
If you do not benefit much from the cache, you can disable both IPC
384
and processor caching completely via `mm_processor_cache_gb=0`.
tests/multimodal/test_cache.py
+29
@@ -17,6 +17,7 @@ from vllm.multimodal.cache import (
17
MultiModalProcessorCacheInItem,
18
MultiModalProcessorCacheItem,
19
MultiModalProcessorCacheItemMetadata,
20
+ MultiModalProcessorOnlyCache,
21
MultiModalProcessorSenderCache,
22
MultiModalReceiverCache,
23
ShmObjectStoreReceiverCache,
@@ -247,6 +248,34 @@ class _StubModelConfig:
248
return self._mm_config
249
250
251
+@pytest.mark.skip_global_cleanup
252
+def test_oversized_item_is_served_uncached():
253
+ """Items larger than the processor cache must not crash insert.
254
+
255
+ cachetools.LRUCache raises ValueError("value too large") when a single
256
+ item exceeds maxsize. Engine profiling uses a max-size dummy item, so this
257
+ used to abort EngineCore startup (vllm-project/vllm#52835). Skip the
258
+ insert and serve the item uncached instead.
259
+ """
260
+ # 1 KiB cache; a 4 KiB item cannot fit even if the cache is empty.
261
+ model_config = _StubModelConfig(mm_processor_cache_gb=1024 / GiB_bytes)
262
+ item = MultiModalKwargsItem.dummy(nbytes=4096)
263
+ small = MultiModalKwargsItem.dummy(nbytes=64)
264
+
265
+ p0_only = MultiModalProcessorOnlyCache(model_config) # type: ignore[arg-type]
266
+ assert p0_only.get_and_update_item((item, []), "big")[0] is item
267
+ assert not p0_only.is_cached_item("big")
268
+ assert p0_only.get_and_update_item((small, []), "small")[0] is small
269
+ assert p0_only.is_cached_item("small")
270
+
271
+ p0 = MultiModalProcessorSenderCache(model_config) # type: ignore[arg-type]
272
+ p1 = MultiModalReceiverCache(model_config) # type: ignore[arg-type]
273
+ assert p0.get_and_update_item((item, []), "big")[0] is item
274
+ assert not p0.is_cached_item("big")
275
+ assert p1.get_and_update_item(item, "big") is item
276
+ assert "big" not in p1._cache
277
+
278
+
279
def test_mm_cache_miss_raises_and_recovers():
280
"""A P0/P1 multimodal cache drift must be recoverable, not a hard crash.
281
tests/utils_/test_cache.py
+15
@@ -1,5 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import pytest
4
+
5
from vllm.utils.cache import CacheInfo, LRUCache
6
7
@@ -123,3 +125,16 @@ def test_lru_cache():
125
assert 2 in cache
126
assert 4 in cache
127
assert 6 in cache
128
+
129
+
130
+def test_lru_cache_put_if_fits():
131
+ cache = LRUCache(10, getsizeof=lambda x: x)
132
+
133
+ assert cache.put_if_fits("ok", 4) is True
134
+ assert cache["ok"] == 4
135
+
136
+ assert cache.put_if_fits("big", 11) is False
137
+ assert "big" not in cache
138
+
139
+ with pytest.raises(ValueError, match="value too large"):
140
+ cache.put("big", 11)
vllm/config/multimodal.py
+3
@@ -157,6 +157,9 @@ class MultiModalConfig:
157
resulting in a total memory usage of
158
`mm_processor_cache_gb * (api_server_count + data_parallel_size)`.
159
160
+ A single processed item larger than this budget is served uncached
161
+ (with a warning) instead of failing. Raise this value to cache such items.
162
+
163
Set to `0` to disable this cache completely (not recommended)."""
164
mm_processor_cache_type: MMCacheType = "lru"
165
"""Type of cache to use for the multi-modal preprocessor/mapper. If `shm`,
vllm/multimodal/cache.py
+41
-5
@@ -266,6 +266,42 @@ class BaseMultiModalCache(ABC, Generic[_I, _O]):
266
for mm_item, mm_hash in zip(mm_items, mm_hashes)
267
]
268
269
+ def cache_if_fits(
270
+ self,
271
+ cache: LRUCache[str, _V],
272
+ key: str,
273
+ value: _V,
274
+ ) -> bool:
275
+ """Insert `value` if it fits in `cache`.
276
+
277
+ `cachetools.Cache` raises `ValueError("value too large")` when a
278
+ single item exceeds `maxsize`. An item bigger than the whole
279
+ processor cache can never be a hit, so skip the insert and serve it
280
+ uncached instead of aborting engine startup.
281
+
282
+ LRU P0/P1 caches call this so they stay mirrored. SHM subclasses do
283
+ not use it; they already skip oversize items in `put()`.
284
+
285
+ Args:
286
+ cache: The LRU cache to update.
287
+ key: Cache key (typically the multi-modal item hash).
288
+ value: Value to insert.
289
+
290
+ Returns:
291
+ `True` if the item was cached, otherwise `False`.
292
+ """
293
+ if cache.put_if_fits(key, value):
294
+ return True
295
+ logger.warning_once(
296
+ "Skipping multi-modal processor cache insert for an item of "
297
+ "%s GiB because it exceeds --mm-processor-cache-gb=%s. "
298
+ "The item will be processed uncached; increase "
299
+ "--mm-processor-cache-gb to cache items of this size.",
300
+ format_gib(int(cache.getsizeof(value))),
301
+ format_gib(int(cache.maxsize)),
302
+ )
303
+ return False
304
+
305
@abstractmethod
306
def clear_cache(self) -> None:
307
"""Clear the underlying cache."""
@@ -389,8 +425,7 @@ class MultiModalProcessorOnlyCache(BaseMultiModalProcessorCache):
425
426
assert mm_item is not None, f"Expected a cached item for {mm_hash=}"
427
392
- self._cache[mm_hash] = MultiModalProcessorCacheItem(*mm_item)
393
-
428
+ self.cache_if_fits(self._cache, mm_hash, MultiModalProcessorCacheItem(*mm_item))
429
return mm_item
430
431
@override
@@ -447,8 +482,9 @@ class MultiModalProcessorSenderCache(BaseMultiModalProcessorCache):
482
483
assert mm_item is not None, f"Expected a cached item for {mm_hash=}"
484
450
- self._cache[mm_hash] = MultiModalProcessorCacheItemMetadata(*mm_item)
451
-
485
+ self.cache_if_fits(
486
+ self._cache, mm_hash, MultiModalProcessorCacheItemMetadata(*mm_item)
487
+ )
488
return mm_item
489
490
@override
@@ -709,7 +745,7 @@ class MultiModalReceiverCache(BaseMultiModalReceiverCache):
745
if mm_item is None:
746
raise MultiModalCacheMissError([mm_hash])
747
712
- self._cache[mm_hash] = mm_item
748
+ self.cache_if_fits(self._cache, mm_hash, mm_item)
749
return mm_item
750
751
@override
vllm/utils/cache.py
+17
@@ -157,6 +157,23 @@ class LRUCache(cachetools.LRUCache[_K, _V]):
157
def put(self, key: _K, value: _V) -> None:
158
self.__setitem__(key, value)
159
160
+ def put_if_fits(self, key: _K, value: _V) -> bool:
161
+ """Insert `value` if it is not larger than the cache capacity.
162
+
163
+ Unlike `put`, this does not raise when a single item exceeds
164
+ `maxsize`. Size is computed once inside the insert path.
165
+
166
+ Returns:
167
+ `True` if the item was cached, otherwise `False`.
168
+ """
169
+ try:
170
+ self[key] = value
171
+ except ValueError as e:
172
+ if str(e) != "value too large":
173
+ raise
174
+ return False
175
+ return True
176
+
177
def pin(self, key: _K) -> None:
178
"""
179
Pins a key in the cache preventing it from being