A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 12:45 UTC
[Fix] Enable mm_processor_cache with vision LoRA (#31927)
Signed-off-by: prashanth058 <prashanth.dannamaneni@uipath.com>
prashanth058 committed
Jan 8, 2026 at 01:31 UTC
d3235cb503e33bff5f3b0ee11ba92dd3337d2a15
5 files changed
+52
-16
tests/multimodal/test_cache.py
+39
@@ -24,10 +24,12 @@ from vllm.multimodal.cache import (
24
)
25
from vllm.multimodal.hasher import MultiModalHasher
26
from vllm.multimodal.inputs import (
27
+ MultiModalFeatureSpec,
28
MultiModalFieldElem,
29
MultiModalKwargsItem,
30
MultiModalKwargsItems,
31
MultiModalSharedField,
32
+ PlaceholderRange,
33
)
34
from vllm.multimodal.processing import PromptInsertion
35
from vllm.utils.mem_constants import GiB_bytes, MiB_bytes
@@ -518,3 +520,40 @@ def test_cache_eviction_shm_cache():
520
receiver_cache = ShmObjectStoreReceiverCache(vllm_config, mp.Lock())
521
522
_run_test_cache_eviction_shm(sender_cache, receiver_cache, base_item_size=MiB_bytes)
523
+
524
+
525
+def test_processor_cache_shared_across_loras():
526
+ """Test that processor cache uses mm_hash to share data across LoRAs."""
527
+ model_config = ModelConfig(
528
+ model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
529
+ mm_processor_cache_gb=1,
530
+ )
531
+ receiver_cache = MultiModalReceiverCache(model_config)
532
+
533
+ base_mm_hash = "image_hash_abc123"
534
+ lora_a_identifier = f"12345:{base_mm_hash}"
535
+ lora_b_identifier = f"67890:{base_mm_hash}"
536
+
537
+ item_data = MultiModalKwargsItem.dummy("test_image", nbytes=1024)
538
+
539
+ feature_lora_a = MultiModalFeatureSpec(
540
+ data=item_data,
541
+ modality="image",
542
+ identifier=lora_a_identifier,
543
+ mm_position=PlaceholderRange(offset=0, length=100),
544
+ mm_hash=base_mm_hash,
545
+ )
546
+
547
+ receiver_cache.get_and_update_features([feature_lora_a])
548
+ assert base_mm_hash in receiver_cache._cache
549
+
550
+ feature_lora_b = MultiModalFeatureSpec(
551
+ data=None,
552
+ modality="image",
553
+ identifier=lora_b_identifier,
554
+ mm_position=PlaceholderRange(offset=0, length=100),
555
+ mm_hash=base_mm_hash,
556
+ )
557
+
558
+ receiver_cache.get_and_update_features([feature_lora_b])
559
+ assert feature_lora_b.data == item_data
vllm/engine/arg_utils.py
-13
@@ -1649,19 +1649,6 @@ class EngineArgs:
1649
else None
1650
)
1651
1652
- if (
1653
- lora_config is not None
1654
- and lora_config.enable_tower_connector_lora
1655
- and self.mm_processor_cache_gb != 0
1656
- ):
1657
- raise ValueError(
1658
- "Currently, enable_tower_connector_lora is "
1659
- "incompatible with the multi-modal processor cache. "
1660
- "When enable_tower_connector_lora is set, "
1661
- "mm_processor_cache_gb must be 0, got %s",
1662
- self.mm_processor_cache_gb,
1663
- )
1664
-
1652
if (
1653
lora_config is not None
1654
and speculative_config is not None
vllm/multimodal/cache.py
+7
-2
@@ -635,12 +635,17 @@ class BaseMultiModalReceiverCache(
635
Update multimodal features with cached encoder outputs.
636
Touch all identifier at first before update to avoid
637
item in updated list evict during update.
638
+
639
+ Uses mm_hash for cache key to share across LoRAs (falls back to
640
+ identifier for backward compatibility).
641
"""
642
for feature in mm_features:
640
- self.touch_receiver_cache_item(feature.identifier, feature.data)
643
+ cache_key = feature.mm_hash or feature.identifier
644
+ self.touch_receiver_cache_item(cache_key, feature.data)
645
646
for feature in mm_features:
643
- feature.data = self.get_and_update_item(feature.data, feature.identifier)
647
+ cache_key = feature.mm_hash or feature.identifier
648
+ feature.data = self.get_and_update_item(feature.data, cache_key)
649
return mm_features
650
651
@abstractmethod
vllm/multimodal/inputs.py
+3
@@ -330,6 +330,9 @@ class MultiModalFeatureSpec:
330
mm_position: PlaceholderRange
331
"""e.g., PlaceholderRange(offset=2, length=336)"""
332
333
+ mm_hash: str | None = None
334
+ """Base mm_hash for processor cache (without LoRA prefix)."""
335
+
336
@staticmethod
337
def gather_kwargs(features: list["MultiModalFeatureSpec"], keys: set[str]):
338
kwargs = defaultdict[str, list[NestedTensors]](list)
vllm/v1/engine/input_processor.py
+3
-1
@@ -562,15 +562,17 @@ class InputProcessor:
562
563
mm_features = []
564
for modality, idx in sorted_mm_idxs:
565
+ base_mm_hash = decoder_mm_hashes[modality][idx]
566
mm_features.append(
567
MultiModalFeatureSpec(
568
data=decoder_mm_inputs[modality][idx],
569
modality=modality,
570
identifier=self._get_mm_identifier(
570
- decoder_mm_hashes[modality][idx],
571
+ base_mm_hash,
572
lora_request,
573
),
574
mm_position=decoder_mm_positions[modality][idx],
575
+ mm_hash=base_mm_hash,
576
)
577
)
578