@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fc6acc88ca
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 12, 06:00 UTC

[Bugfix] Missing cached item in the MultiModalReceiverCache (#28525)

Signed-off-by: knlnguyen1802 <knlnguyen1802@gmail.com> Co-authored-by: Chenguang Zheng <645327136@qq.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>

knlnguyen1802 committed Dec 2, 2025 at 02:18 UTC fc6acc88caac881a92d84bcef5bc734d67035786
5 files changed +436 -21
tests/multimodal/test_cache.py
+300 -4
@@ -1,5 +1,6 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import multiprocessing as mp
4
5 import numpy as np
6 import pytest
@@ -8,9 +9,16 @@ import torch
9 from vllm.config import ModelConfig, ParallelConfig, VllmConfig
10 from vllm.multimodal import MULTIMODAL_REGISTRY
11 from vllm.multimodal.cache import (
12 + BaseMultiModalProcessorCache,
13 + BaseMultiModalReceiverCache,
14 MultiModalCache,
15 + MultiModalProcessorCacheInItem,
16 MultiModalProcessorCacheItem,
17 MultiModalProcessorCacheItemMetadata,
18 + MultiModalProcessorSenderCache,
19 + MultiModalReceiverCache,
20 + ShmObjectStoreReceiverCache,
21 + ShmObjectStoreSenderCache,
22 engine_receiver_cache_from_config,
23 processor_cache_from_config,
24 )
@@ -22,6 +30,7 @@ from vllm.multimodal.inputs import (
30 MultiModalSharedField,
31 )
32 from vllm.multimodal.processing import PromptInsertion
33 +from vllm.utils.mem_constants import GiB_bytes, MiB_bytes
34
35 pytestmark = pytest.mark.cpu_test
36
@@ -144,8 +153,7 @@ def _compare_caches(
153 MultiModalHasher.hash_kwargs(item=item.get_data()) for item in all_items
154 ]
155
147 - # Should not be used since there is nothing to convert to text
148 - prompt_update = PromptInsertion("dummy", "target", "insertion")
156 + prompt_update = PromptInsertion("dummy", "target", "insertion").resolve(0)
157
158 for it in range(n_iter):
159 num_items_to_select = rng.randint(0, max_items_per_iter)
@@ -159,10 +167,11 @@ def _compare_caches(
167 else:
168 for _ in range(is_cached_calls_per_iter):
169 cache_0_p0.is_cached(selected_hashes)
170 +
171 cache_0_p0_out = [
172 item
173 for item, _ in cache_0_p0.get_and_update(
165 - [(item, prompt_update.content) for item in selected_items],
174 + [(item, [prompt_update]) for item in selected_items],
175 selected_hashes,
176 )
177 ]
@@ -172,10 +181,11 @@ def _compare_caches(
181 else:
182 for _ in range(is_cached_calls_per_iter):
183 cache_1_p0.is_cached(selected_hashes)
184 +
185 cache_1_p0_out = [
186 item
187 for item, _ in cache_1_p0.get_and_update(
178 - [(item, prompt_update.content) for item in selected_items],
188 + [(item, [prompt_update]) for item in selected_items],
189 selected_hashes,
190 )
191 ]
@@ -225,3 +235,289 @@ def test_ipc_enable_disable_consistency(is_cached_calls_per_iter):
235 vllm_config_ipc_enabled,
236 is_cached_calls_per_iter=is_cached_calls_per_iter,
237 )
238 +
239 +
240 +def _run_test_cache_eviction_lru(
241 + p0_cache: BaseMultiModalProcessorCache,
242 + p1_cache: BaseMultiModalReceiverCache,
243 + base_item_size: int,
244 +):
245 + request1_hashes = [
246 + "image_A",
247 + "image_B",
248 + "image_C",
249 + ]
250 + request1_items = {
251 + h: MultiModalKwargsItem.dummy(h, nbytes=2 * base_item_size)
252 + for h in request1_hashes
253 + }
254 +
255 + request2_hashes = ["image_D", "image_E", "image_A", "image_C"]
256 + request2_items = {
257 + h: MultiModalKwargsItem.dummy(h, nbytes=1 * base_item_size)
258 + for h in request2_hashes
259 + }
260 +
261 + ##########################
262 + # STEP 1: Request 1 send
263 + ##########################
264 + sender_is_cached_item_req1 = p0_cache.is_cached(request1_hashes)
265 + # Cache is empty
266 + assert sender_is_cached_item_req1 == [False, False, False]
267 +
268 + # Touch all mm hash for P0 Cache before process
269 + for mm_hash in request1_hashes:
270 + p0_cache.touch_sender_cache_item(mm_hash)
271 +
272 + ###########################
273 + # Process request 1 for P0 Cache
274 + ###########################
275 + item_tuple: MultiModalProcessorCacheInItem
276 + for i, h in enumerate(request1_hashes):
277 + # Use precomputed cache state
278 + is_cached = sender_is_cached_item_req1[i]
279 + item_tuple = (request1_items[h], []) if not is_cached else None
280 + print(f"Request 1: key={h} | cached={is_cached}")
281 +
282 + p0_cache.get_and_update_item(item_tuple, h)
283 +
284 + ###########################
285 + # Process request 1 for P1 Cache
286 + ###########################
287 + # Touch all mm hash for P1 Cache before process
288 + for mm_hash in request1_hashes:
289 + p1_cache.touch_receiver_cache_item(mm_hash)
290 +
291 + for h in request1_hashes:
292 + p1_cache.get_and_update_item(request1_items[h], h)
293 +
294 + expected_hashes = ["image_A", "image_B", "image_C"]
295 + assert list(p0_cache._cache.order) == expected_hashes
296 +
297 + ##########################
298 + # STEP 2: Request 2 send
299 + ##########################
300 + sender_is_cached_item_req2 = p0_cache.is_cached(request2_hashes)
301 + assert sender_is_cached_item_req2 == [False, False, True, True]
302 +
303 + # Touch all mm hash for P0 Cache before process
304 + for mm_hash in request2_hashes:
305 + p0_cache.touch_sender_cache_item(mm_hash)
306 +
307 + ###########################
308 + # Process request 2 for P0 Cache
309 + ###########################
310 + for i, h in enumerate(request2_hashes):
311 + # Use precomputed cache state again
312 + is_cached = sender_is_cached_item_req2[i]
313 + item_tuple = (request2_items[h], []) if not is_cached else None
314 + print(f"Request 2: key={h} | cached={is_cached}")
315 +
316 + p0_cache.get_and_update_item(item_tuple, h)
317 +
318 + ###########################
319 + # Process request 2 for P1 Cache
320 + ###########################
321 +
322 + # Touch all mm hash for P1 Cache before process
323 + for mm_hash in request2_hashes:
324 + p1_cache.touch_receiver_cache_item(mm_hash)
325 +
326 + for h in request2_hashes:
327 + p1_cache.get_and_update_item(request2_items[h], h)
328 +
329 + expected_hashes = ["image_D", "image_E", "image_A", "image_C"]
330 + assert list(p0_cache._cache.order) == expected_hashes
331 +
332 +
333 +def test_cache_eviction_lru_cache():
334 + model_config = ModelConfig(
335 + model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
336 + mm_processor_cache_gb=6 / GiB_bytes,
337 + )
338 + sender_cache = MultiModalProcessorSenderCache(model_config)
339 + receiver_cache = MultiModalReceiverCache(model_config)
340 +
341 + _run_test_cache_eviction_lru(sender_cache, receiver_cache, base_item_size=1)
342 +
343 +
344 +# This test verifies shared-memory cache eviction behavior across processor (p0)
345 +# and receiver (p1) caches.
346 +# Flow summary:
347 +# 1. Request 1 adds images A, B, C — completely filling the cache.
348 +# 2. Request 2 tries to add image_G and image_A, but image_G cannot be added because
349 +# cache is full and A is protected from eviction — cache remains unchanged.
350 +# 3. Request 3 adds image_G, image_H, image_I and image_B
351 +# this time, image_A is evicted, freeing 5MB space
352 +# and image_G, image_H successfully fits,
353 +# image_B is protected from eviction then image_i cannot be added.
354 +# This proving normal eviction and reuse behavior.
355 +def _run_test_cache_eviction_shm(
356 + p0_cache: BaseMultiModalProcessorCache,
357 + p1_cache: BaseMultiModalReceiverCache,
358 + base_item_size: int,
359 +):
360 + request1_hashes = ["image_A", "image_B", "image_C"]
361 + request1_items = {
362 + h: MultiModalKwargsItem.dummy(h, nbytes=5 * base_item_size)
363 + for h in request1_hashes
364 + }
365 + request1_items_p0_result = []
366 +
367 + request2_hashes = ["image_G", "image_A"]
368 + request2_items = {
369 + h: MultiModalKwargsItem.dummy(
370 + h, nbytes=(5 if h in request1_hashes else 2) * base_item_size
371 + )
372 + for h in request2_hashes
373 + }
374 + request2_items_p0_result = []
375 +
376 + request3_hashes = ["image_G", "image_H", "image_I", "image_B"]
377 + request3_items = {
378 + h: MultiModalKwargsItem.dummy(
379 + h, nbytes=(5 if h in request1_hashes else 2) * base_item_size
380 + )
381 + for h in request3_hashes
382 + }
383 + request3_items_p0_result = []
384 +
385 + ##########################
386 + # STEP 1: Request 1 send
387 + # This will fill up the cache
388 + ##########################
389 + sender_is_cached_item_req1 = p0_cache.is_cached(request1_hashes)
390 + # Cache is empty
391 + assert sender_is_cached_item_req1 == [False, False, False]
392 +
393 + # Touch all mm hash for P0 Cache before process
394 + for mm_hash in request1_hashes:
395 + p0_cache.touch_sender_cache_item(mm_hash)
396 +
397 + ###########################
398 + # Process request 1 for P0 Cache
399 + ###########################
400 + item_tuple: MultiModalProcessorCacheInItem
401 + for i, h in enumerate(request1_hashes):
402 + # Use precomputed cache state
403 + is_cached = sender_is_cached_item_req1[i]
404 + item_tuple = (request1_items[h], []) if not is_cached else None
405 + print(f"Request 1: key={h} | cached={is_cached}")
406 +
407 + p0_result = p0_cache.get_and_update_item(item_tuple, h)
408 + # Only get mm item, ignore prompt update result
409 + request1_items_p0_result.append(p0_result[0])
410 +
411 + ###########################
412 + # Process request 1 for P1 Cache
413 + ###########################
414 + # Touch all mm hash for P1 Cache before process
415 + for mm_hash, mm_item in zip(request1_hashes, request1_items_p0_result):
416 + p1_cache.touch_receiver_cache_item(mm_hash, mm_item)
417 +
418 + for mm_hash, mm_item in zip(request1_hashes, request1_items_p0_result):
419 + p1_cache.get_and_update_item(mm_item, mm_hash)
420 +
421 + expected_hashes = ["image_A", "image_B", "image_C"]
422 + assert list(p0_cache._shm_cache.key_index.keys()) == expected_hashes
423 +
424 + ##########################
425 + # STEP 2: Request 2 send
426 + # There is no eviction because image_A is protected
427 + # No new item can add to cache
428 + ##########################
429 + sender_is_cached_item_req2 = p0_cache.is_cached(request2_hashes)
430 + assert sender_is_cached_item_req2 == [False, True]
431 +
432 + # Touch all mm hash for P0 Cache before process
433 + for mm_hash in request2_hashes:
434 + p0_cache.touch_sender_cache_item(mm_hash)
435 +
436 + ###########################
437 + # Process request 2 for P0 Cache
438 + ###########################
439 + for i, h in enumerate(request2_hashes):
440 + # Use precomputed cache state again
441 + is_cached = sender_is_cached_item_req2[i]
442 + item_tuple = (request2_items[h], []) if not is_cached else None
443 + print(f"Request 2: key={h} | cached={is_cached}")
444 +
445 + p0_result = p0_cache.get_and_update_item(item_tuple, h)
446 + # Only get mm item, ignore prompt update result
447 + request2_items_p0_result.append(p0_result[0])
448 +
449 + # image_A cannot be evict then
450 + # image_G will fail to allocate anyway and image_A still in cache
451 + assert p0_cache.is_cached(request2_hashes) == [False, True]
452 +
453 + ###########################
454 + # Process request 2 for P1 Cache
455 + ###########################
456 +
457 + # Touch all mm hash for P1 Cache before process
458 + for mm_hash, mm_item in zip(request2_hashes, request2_items_p0_result):
459 + p1_cache.touch_receiver_cache_item(mm_hash, mm_item)
460 +
461 + for mm_hash, mm_item in zip(request2_hashes, request2_items_p0_result):
462 + p1_cache.get_and_update_item(mm_item, mm_hash)
463 +
464 + # Prove that cache state is unchanged
465 + expected_hashes = ["image_A", "image_B", "image_C"]
466 + assert list(p0_cache._shm_cache.key_index.keys()) == expected_hashes
467 +
468 + ##########################
469 + # STEP 3: Request 3 send
470 + ##########################
471 + ##### Prove that cache eviction work normally
472 + sender_is_cached_item_req3 = p0_cache.is_cached(request3_hashes)
473 + assert sender_is_cached_item_req3 == [False, False, False, True]
474 +
475 + # Touch all mm hash for P0 Cache before process
476 + for mm_hash in request3_hashes:
477 + p0_cache.touch_sender_cache_item(mm_hash)
478 +
479 + ###########################
480 + # Process request 3 for P0 Cache
481 + ###########################
482 + for i, h in enumerate(request3_hashes):
483 + # Use precomputed cache state again
484 + is_cached = sender_is_cached_item_req3[i]
485 + item_tuple = (request3_items[h], []) if not is_cached else None
486 + print(f"Request 3: key={h} | cached={is_cached}")
487 + p0_result = p0_cache.get_and_update_item(item_tuple, h)
488 + # Only get mm item, ignore prompt update result
489 + request3_items_p0_result.append(p0_result[0])
490 +
491 + # image_A got evict and image_G add to cache
492 + # image_B is still protected
493 + # image_G, image_H fit but image_I cannot fit
494 + assert p0_cache.is_cached(request3_hashes) == [True, True, False, True]
495 +
496 + ###########################
497 + # Process request 3 for P1 Cache
498 + ###########################
499 +
500 + # Touch all mm hash for P1 Cache before process
501 + for mm_hash, mm_item in zip(request3_hashes, request3_items_p0_result):
502 + p1_cache.touch_receiver_cache_item(mm_hash, mm_item)
503 +
504 + for mm_hash, mm_item in zip(request3_hashes, request3_items_p0_result):
505 + p1_cache.get_and_update_item(mm_item, mm_hash)
506 +
507 + expected_hashes = ["image_B", "image_C", "image_G", "image_H"]
508 + assert list(p0_cache._shm_cache.key_index.keys()) == expected_hashes
509 +
510 +
511 +def test_cache_eviction_shm_cache():
512 + vllm_config = VllmConfig(
513 + model_config=ModelConfig(
514 + model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
515 + mm_processor_cache_type="shm",
516 + mm_shm_cache_max_object_size_mb=6,
517 + mm_processor_cache_gb=15.2 * MiB_bytes / GiB_bytes,
518 + ),
519 + )
520 + sender_cache = ShmObjectStoreSenderCache(vllm_config)
521 + receiver_cache = ShmObjectStoreReceiverCache(vllm_config, mp.Lock())
522 +
523 + _run_test_cache_eviction_shm(sender_cache, receiver_cache, base_item_size=MiB_bytes)
vllm/distributed/device_communicators/shm_object_storage.py
+38 -1
@@ -574,7 +574,6 @@ class SingleWriterShmObjectStorage:
574 value
575 )
576 buffer_size = self.flag_bytes + data_bytes + md_bytes
577 -
577 # Sanity checks
578 if buffer_size > self.max_object_size:
579 raise ValueError(
@@ -626,6 +625,44 @@ class SingleWriterShmObjectStorage:
625
626 return obj
627
628 + def touch(
629 + self,
630 + key: str,
631 + address: int = 0,
632 + monotonic_id: int = 0,
633 + ) -> None:
634 + """
635 + Touch an existing cached item to update its eviction status.
636 +
637 + For writers (ShmObjectStoreSenderCache): Increment writer_flag
638 + For readers (ShmObjectStoreReceiverCache): Increment reader_count
639 +
640 + Args:
641 + key: String key of the object to touch
642 + address: Address of the object (only for readers)
643 + monotonic_id: Monotonic ID of the object (only for readers)
644 +
645 + """
646 + if self._reader_lock is None:
647 + if key not in self.key_index:
648 + return None
649 + address, monotonic_id = self.key_index[key]
650 + # Writer side: increment writer_flag to raise eviction threshold
651 + self.increment_writer_flag(monotonic_id)
652 + else:
653 + with (
654 + self._reader_lock,
655 + self.ring_buffer.access_buf(address) as (data_view, _),
656 + ):
657 + reader_count = self.ring_buffer.byte2int(data_view[: self.flag_bytes])
658 +
659 + # NOTE(Long):
660 + # Avoid increasing flag on newly added item (sync with sender)
661 + # Since when a new item is added
662 + # pre-touch has no effect on writer side
663 + if reader_count >= self.n_readers:
664 + self.increment_reader_flag(data_view[: self.flag_bytes])
665 +
666 def handle(self):
667 """Get handle for sharing across processes."""
668 return ShmObjectStorageHandle(
vllm/multimodal/cache.py
+76 -1
@@ -302,6 +302,19 @@ class BaseMultiModalProcessorCache(
302 """
303 return [self.is_cached_item(mm_hash) for mm_hash in mm_hashes]
304
305 + @abstractmethod
306 + def touch_sender_cache_item(self, mm_hash: str) -> None:
307 + """
308 + Update the cache eviction order for a multi-modal item.
309 +
310 + This is used to touch the item in the cache without changing
311 + its value.
312 +
313 + Args:
314 + mm_hash: The hash of the multi-modal item.
315 + """
316 + raise NotImplementedError
317 +
318 @abstractmethod
319 def make_stats(self, *, delta: bool = False) -> CacheInfo:
320 """
@@ -353,6 +366,10 @@ class MultiModalProcessorOnlyCache(BaseMultiModalProcessorCache):
366
367 return mm_item
368
369 + @override
370 + def touch_sender_cache_item(self, mm_hash: str) -> None:
371 + self._cache.touch(mm_hash)
372 +
373 @override
374 def clear_cache(self) -> None:
375 self._cache.clear()
@@ -407,6 +424,10 @@ class MultiModalProcessorSenderCache(BaseMultiModalProcessorCache):
424
425 return mm_item
426
427 + @override
428 + def touch_sender_cache_item(self, mm_hash: str) -> None:
429 + self._cache.touch(mm_hash)
430 +
431 @override
432 def clear_cache(self) -> None:
433 self._cache.clear()
@@ -501,6 +522,12 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
522 logger.debug("Failed to cache mm_input with hash %s: %s", mm_hash, e)
523 return mm_item
524
525 + @override
526 + def touch_sender_cache_item(self, mm_hash: str) -> None:
527 + """Touch the item in shared memory cache to prevent eviction.
528 + Increments writer_flag on sender side."""
529 + self._shm_cache.touch(mm_hash)
530 +
531 @override
532 def clear_cache(self) -> None:
533 self._shm_cache.clear()
@@ -610,11 +637,37 @@ class BaseMultiModalReceiverCache(
637 self,
638 mm_features: list["MultiModalFeatureSpec"],
639 ) -> list["MultiModalFeatureSpec"]:
613 - """Update multimodal features with cached encoder outputs."""
640 + """
641 + Update multimodal features with cached encoder outputs.
642 + Touch all identifier at first before update to avoid
643 + item in updated list evict during update.
644 + """
645 + for feature in mm_features:
646 + self.touch_receiver_cache_item(feature.identifier, feature.data)
647 +
648 for feature in mm_features:
649 feature.data = self.get_and_update_item(feature.data, feature.identifier)
650 return mm_features
651
652 + @abstractmethod
653 + def touch_receiver_cache_item(
654 + self,
655 + mm_hash: str,
656 + mm_item: MultiModalKwargsItem | None = None,
657 + ) -> None:
658 + """
659 + Update the cache eviction order for a multi-modal item.
660 +
661 + This is used to touch the item in the cache without changing
662 + its value.
663 +
664 + Args:
665 + mm_hash: The hash of the multi-modal item.
666 + mm_item: The multi-modal item itself. This is optional and
667 + may not be needed by some cache implementations.
668 + """
669 + raise NotImplementedError
670 +
671
672 class MultiModalReceiverCache(BaseMultiModalReceiverCache):
673 """
@@ -651,6 +704,14 @@ class MultiModalReceiverCache(BaseMultiModalReceiverCache):
704 self._cache[mm_hash] = mm_item
705 return mm_item
706
707 + @override
708 + def touch_receiver_cache_item(
709 + self,
710 + mm_hash: str,
711 + mm_item: MultiModalKwargsItem | None = None,
712 + ) -> None:
713 + self._cache.touch(mm_hash)
714 +
715 @override
716 def clear_cache(self) -> None:
717 self._cache.clear()
@@ -703,6 +764,20 @@ class ShmObjectStoreReceiverCache(BaseMultiModalReceiverCache):
764
765 return mm_item
766
767 + @override
768 + def touch_receiver_cache_item(
769 + self,
770 + mm_hash: str,
771 + mm_item: MultiModalKwargsItem | None = None,
772 + ) -> None:
773 + """Touch the item in shared memory cache to prevent eviction.
774 + Increments reader_count on receiver side."""
775 + assert mm_item is not None
776 + if "address" in mm_item:
777 + address = cast(int, mm_item["address"].data)
778 + monotonic_id = cast(int, mm_item["monotonic_id"].data)
779 + self._shm_cache.touch(mm_hash, address=address, monotonic_id=monotonic_id)
780 +
781 @override
782 def clear_cache(self) -> None:
783 self._shm_cache.clear()
vllm/multimodal/inputs.py
+2 -2
@@ -721,12 +721,12 @@ class MultiModalKwargsItem(UserDict[str, MultiModalFieldElem]):
721 """
722
723 @staticmethod
724 - def dummy(modality: str):
724 + def dummy(modality: str, nbytes: int = 1):
725 """Convenience class for testing."""
726 mm_elem = MultiModalFieldElem(
727 modality=modality,
728 key="dummy",
729 - data=torch.empty(1),
729 + data=torch.empty(nbytes, dtype=torch.uint8),
730 field=MultiModalSharedField(1),
731 )
732 return MultiModalKwargsItem.from_elems([mm_elem])
vllm/multimodal/processing.py
+20 -13
@@ -1248,7 +1248,13 @@ _I = TypeVar("_I", bound=BaseProcessingInfo)
1248
1249 MultiModalHashes = dict[str, list[str]]
1250 """
1251 -A collection of hashes with a similar structure as
1251 +A collection of the multi-modal hash for each item, with a similar structure as
1252 +[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
1253 +"""
1254 +
1255 +MultiModalIsCached = dict[str, list[bool]]
1256 +"""
1257 +A collection of the `is_cached` flag for each item, with a similar structure as
1258 [`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
1259 """
1260
@@ -1725,7 +1731,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1731 cache: BaseMultiModalProcessorCache,
1732 mm_data_items: MultiModalDataItems,
1733 mm_hashes: MultiModalHashes,
1728 - ) -> MultiModalDataItems:
1734 + ) -> tuple[MultiModalIsCached, MultiModalDataItems]:
1735 mm_is_cached = {
1736 modality: cache.is_cached(hashes) for modality, hashes in mm_hashes.items()
1737 }
@@ -1752,7 +1758,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1758 missing_modality_data.append(data)
1759 mm_missing_data[modality] = missing_modality_data
1760
1755 - return self._to_mm_items(mm_missing_data)
1761 + return mm_is_cached, self._to_mm_items(mm_missing_data)
1762
1763 def _recompute_cached_prompt_update(
1764 self,
@@ -1769,14 +1775,15 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1775 self,
1776 cache: BaseMultiModalProcessorCache,
1777 mm_hashes: MultiModalHashes,
1778 + mm_is_cached: MultiModalIsCached,
1779 mm_missing_kwargs: MultiModalKwargsItems,
1780 mm_missing_prompt_updates: MultiModalPromptUpdates,
1781 ) -> tuple[MultiModalKwargsOptionalItems, MultiModalPromptUpdates]:
1775 - # Need to calculate this at the beginning to avoid skipping cache logic
1776 - # for subsequently repeated items in the same modality
1777 - mm_is_cached = {
1778 - modality: cache.is_cached(hashes) for modality, hashes in mm_hashes.items()
1779 - }
1782 + # Need to touch all mm hashes before update to avoid hash in updated
1783 + # list evict during update
1784 + for hashes in mm_hashes.values():
1785 + for item_hash in hashes:
1786 + cache.touch_sender_cache_item(item_hash)
1787
1788 mm_missing_next_idx = defaultdict[str, int](lambda: 0)
1789
@@ -1789,15 +1796,14 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1796 missing_prompt_updates = mm_missing_prompt_updates.get(modality, [])
1797
1798 for item_idx, item_hash in enumerate(hashes):
1792 - kwargs: MultiModalKwargsItem | None
1799 if not mm_is_cached[modality][item_idx]:
1800 missing_next_idx = mm_missing_next_idx[modality]
1795 - kwargs = missing_kwargs[missing_next_idx]
1796 - updates = missing_prompt_updates[missing_next_idx]
1801 + missing_kwargs_item = missing_kwargs[missing_next_idx]
1802 + missing_updates_item = missing_prompt_updates[missing_next_idx]
1803
1804 mm_missing_next_idx[modality] += 1
1805
1800 - item = kwargs, updates
1806 + item = missing_kwargs_item, missing_updates_item
1807 else:
1808 item = None
1809
@@ -1896,7 +1902,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1902 mm_uuids=mm_uuids,
1903 )
1904
1899 - mm_missing_data_items = self._get_cache_missing_items(
1905 + mm_is_cached, mm_missing_data_items = self._get_cache_missing_items(
1906 cache=cache,
1907 mm_data_items=mm_data_items,
1908 mm_hashes=mm_hashes,
@@ -1933,6 +1939,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1939 mm_kwargs, mm_prompt_updates = self._merge_mm_kwargs(
1940 cache,
1941 mm_hashes=mm_hashes,
1942 + mm_is_cached=mm_is_cached,
1943 mm_missing_kwargs=mm_missing_kwargs,
1944 mm_missing_prompt_updates=mm_missing_prompt_updates,
1945 )