[Bugfix][Multimodal] Scope cache hash kwargs by modality (#54918)
Signed-off-by: waizuichougou <2082431897@qq.com>
waizuichougou committed
Sep 2, 2026 at 22:20 UTC
c6bca6e58540817bd6f192ae615dfa108cec1152
2 files changed
+88
-10
tests/multimodal/test_processing.py
+47
@@ -1284,3 +1284,50 @@ def test_processor_inputs_hashes_partial_uuids():
1284
),
1285
]
1286
}
1287
+
1288
+
1289
+def test_processor_inputs_hashes_scope_kwargs_by_modality():
1290
+ """Changing one modality's options must not invalidate another item."""
1291
+ rng = np.random.RandomState(0)
1292
+ mm_data_items = MultiModalDataParser().parse_mm_data(
1293
+ {
1294
+ "image": [random_image(rng, min_wh=8, max_wh=9)],
1295
+ "video": [np.zeros((2, 8, 8, 3), dtype=np.uint8)],
1296
+ }
1297
+ )
1298
+ mm_uuid_items = {"image": ["image-uuid"], "video": ["video-uuid"]}
1299
+
1300
+ def get_hashes(video_frames: int, image_size: int, video_size: int):
1301
+ return ProcessorInputs(
1302
+ prompt=[],
1303
+ mm_data_items=mm_data_items,
1304
+ mm_uuid_items=mm_uuid_items,
1305
+ media_io_kwargs={"video": {"num_frames": video_frames}},
1306
+ hf_processor_mm_kwargs={
1307
+ "images_kwargs": {"size": {"longest_edge": image_size}},
1308
+ "videos_kwargs": {"size": {"longest_edge": video_size}},
1309
+ },
1310
+ ).get_mm_hashes("test-model", "blake3")
1311
+
1312
+ base = get_hashes(video_frames=4, image_size=224, video_size=224)
1313
+ changed_video = get_hashes(video_frames=16, image_size=224, video_size=448)
1314
+ changed_image = get_hashes(video_frames=4, image_size=448, video_size=224)
1315
+
1316
+ assert changed_video["image"] == base["image"]
1317
+ assert changed_video["video"] != base["video"]
1318
+ assert changed_image["image"] != base["image"]
1319
+ assert changed_image["video"] == base["video"]
1320
+
1321
+
1322
+def test_processor_inputs_hashes_ignore_unrelated_kwargs():
1323
+ """An image-only request ignores video-only processing configuration."""
1324
+ image = random_image(np.random.RandomState(0), min_wh=8, max_wh=9)
1325
+ inputs = ProcessorInputs(
1326
+ prompt=[],
1327
+ mm_data_items=MultiModalDataParser().parse_mm_data({"image": [image]}),
1328
+ mm_uuid_items={"image": ["image-uuid"]},
1329
+ media_io_kwargs={"video": {"num_frames": 16}},
1330
+ hf_processor_mm_kwargs={"videos_kwargs": {"size": {"longest_edge": 448}}},
1331
+ )
1332
+
1333
+ assert inputs.get_mm_hashes("test-model", "blake3") == {"image": ["image-uuid"]}
vllm/multimodal/processing/inputs.py
+41
-10
@@ -9,6 +9,12 @@ from vllm.inputs import MultiModalHashes
9
from ..hasher import MultiModalHasher
10
from ..parse import MultiModalDataItems, MultiModalUUIDItems
11
12
+_HF_MODALITY_PROCESSOR_KWARGS = {
13
+ "image": "images_kwargs",
14
+ "video": "videos_kwargs",
15
+ "audio": "audio_kwargs",
16
+}
17
+
18
19
@dataclass
20
class ProcessorInputs:
@@ -31,17 +37,41 @@ class ProcessorInputs:
37
mm_data_items = self.mm_data_items
38
mm_uuid_items = self.mm_uuid_items or {}
39
34
- hash_factors = {
35
- "media_io_kwargs": dict(self.media_io_kwargs),
36
- "mm_processor_kwargs": dict(self.hf_processor_mm_kwargs),
37
- }
38
- has_hash_factors = any(hash_factors.values())
39
- hash_factor_kwargs = hash_factors if has_hash_factors else {}
40
-
40
mm_hashes = dict[str, list[str]]()
41
hasher = MultiModalHasher
42
43
for modality, data_items in mm_data_items.items():
44
+ # Each cache entry represents one item from one modality. Keep
45
+ # modality-scoped options out of unrelated entries, while
46
+ # retaining flat processor kwargs because they are shared by HF.
47
+ if modality in _HF_MODALITY_PROCESSOR_KWARGS:
48
+ modality_media_io_kwargs = self.media_io_kwargs.get(modality, {})
49
+ media_io_kwargs = (
50
+ {modality: modality_media_io_kwargs}
51
+ if modality_media_io_kwargs
52
+ else {}
53
+ )
54
+ scoped_key = _HF_MODALITY_PROCESSOR_KWARGS[modality]
55
+ mm_processor_kwargs = {
56
+ key: value
57
+ for key, value in self.hf_processor_mm_kwargs.items()
58
+ if key not in _HF_MODALITY_PROCESSOR_KWARGS.values()
59
+ or key == scoped_key
60
+ }
61
+ else:
62
+ # Unified modalities such as ``vision_chunk`` can contain
63
+ # items originating from multiple media modalities. There is
64
+ # no per-item origin metadata here, so preserve all factors.
65
+ media_io_kwargs = dict(self.media_io_kwargs)
66
+ mm_processor_kwargs = dict(self.hf_processor_mm_kwargs)
67
+
68
+ hash_factors = {
69
+ "media_io_kwargs": media_io_kwargs,
70
+ "mm_processor_kwargs": mm_processor_kwargs,
71
+ }
72
+ hash_factors = {key: value for key, value in hash_factors.items() if value}
73
+ has_hash_factors = bool(hash_factors)
74
+
75
uuid_items = (
76
mm_uuid_items[modality]
77
if modality in mm_uuid_items
@@ -53,8 +83,9 @@ class ProcessorInputs:
83
for i, item in enumerate(data_items.get_all_items_for_hash()):
84
uuid_item = uuid_items[i]
85
56
- # NOTE: Even if a uuid_item is provided, model output
57
- # depends on hash_factor_kwargs, so they are taken into account.
86
+ # NOTE: Even if a uuid_item is provided, model output depends
87
+ # on the current modality's hash factors, so they are taken
88
+ # into account.
89
if uuid_item is None or has_hash_factors:
90
# NOTE: use provided hash string to hash with kwargs
91
# if available for better performance.
@@ -64,7 +95,7 @@ class ProcessorInputs:
95
hash_algorithm,
96
model_id=model_id,
97
**{modality: item},
67
- **hash_factor_kwargs,
98
+ **hash_factors,
99
)
100
)
101
else: