[Feat][MM Hashing] include media_io_kwargs in multi-modal hashes (#54241)
Signed-off-by: Jan Kwiecinski <jkwiecinski@nvidia.com>
Jan Kwieciński committed
Sep 1, 2026 at 15:17 UTC
1f1f628859dd0dc2a3b7f43810254884799619e6
6 files changed
+163
-35
tests/multimodal/test_processing.py
+22
@@ -10,10 +10,13 @@ import pytest
10
from vllm.config import ModelConfig
11
from vllm.exceptions import VLLMValidationError
12
from vllm.multimodal import MULTIMODAL_REGISTRY
13
+from vllm.multimodal.hasher import MultiModalHasher
14
+from vllm.multimodal.parse import MultiModalDataParser
15
from vllm.multimodal.processing.context import (
16
InputProcessingContext,
17
overlay_modality_mm_kwargs,
18
)
19
+from vllm.multimodal.processing.inputs import ProcessorInputs
20
from vllm.multimodal.processing.processor import (
21
BaseMultiModalProcessor,
22
PlaceholderFeaturesInfo,
@@ -1262,3 +1265,22 @@ def test_mm_processor_kwargs_merge_then_overlay_preserves_scoping():
1265
assert overlay_modality_mm_kwargs(merged, "video")["size"] == size
1266
assert "size" not in overlay_modality_mm_kwargs(merged, "image")
1267
assert "size" not in overlay_modality_mm_kwargs(merged, None)
1268
+
1269
+
1270
+def test_processor_inputs_hashes_partial_uuids():
1271
+ rng = np.random.RandomState(0)
1272
+ images = [random_image(rng, min_wh=8, max_wh=9) for _ in range(2)]
1273
+ inputs = ProcessorInputs(
1274
+ prompt=[],
1275
+ mm_data_items=MultiModalDataParser().parse_mm_data({"image": images}),
1276
+ mm_uuid_items={"image": ["image-uuid", None]},
1277
+ )
1278
+
1279
+ assert inputs.get_mm_hashes("test-model", "blake3") == {
1280
+ "image": [
1281
+ "image-uuid",
1282
+ MultiModalHasher.hash_kwargs(
1283
+ "blake3", model_id="test-model", image=images[1]
1284
+ ),
1285
+ ]
1286
+ }
tests/renderers/test_multimodal_hashes.py
new
+92
@@ -0,0 +1,92 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import pytest
5
+
6
+from vllm.assets.video import VideoAsset
7
+from vllm.config import ModelConfig, VllmConfig
8
+from vllm.multimodal.utils import encode_video_url
9
+from vllm.renderers.hf import HfRenderer
10
+from vllm.renderers.params import ChatParams
11
+from vllm.tokenizers.registry import cached_tokenizer_from_config
12
+
13
+pytestmark = pytest.mark.cpu_test
14
+
15
+video_data_url = encode_video_url(VideoAsset("baby_reading").np_ndarrays)
16
+
17
+DEFAULT_MEDIA_IO_KWARGS = {"video": {"num_frames": 32}}
18
+DEFAULT_MM_PROCESSOR_KWARGS = {"videos_kwargs": {"do_resize": True}}
19
+
20
+
21
+def _build_video_renderer() -> HfRenderer:
22
+ model_config = ModelConfig(
23
+ model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
24
+ max_model_len=4096,
25
+ mm_processor_cache_gb=4.0,
26
+ )
27
+
28
+ return HfRenderer(
29
+ VllmConfig(model_config=model_config),
30
+ cached_tokenizer_from_config(model_config),
31
+ )
32
+
33
+
34
+def _get_message_hash(
35
+ renderer: HfRenderer,
36
+ messages: list[dict],
37
+ media_io_kwargs: dict | None = None,
38
+ mm_processor_kwargs: dict | None = None,
39
+) -> str:
40
+ media_io_kwargs = media_io_kwargs or DEFAULT_MEDIA_IO_KWARGS
41
+ mm_processor_kwargs = mm_processor_kwargs or DEFAULT_MM_PROCESSOR_KWARGS
42
+ _, inputs = renderer.render_chat(
43
+ [messages],
44
+ ChatParams(
45
+ media_io_kwargs=media_io_kwargs,
46
+ mm_processor_kwargs=mm_processor_kwargs,
47
+ ),
48
+ prompt_extras={"mm_processor_kwargs": mm_processor_kwargs},
49
+ )
50
+
51
+ return inputs[0]["mm_hashes"]["video"][0]
52
+
53
+
54
+def _video_chat_messages(client_uuid: str):
55
+ return [
56
+ {
57
+ "role": "user",
58
+ "content": [
59
+ {
60
+ "type": "video_url",
61
+ "video_url": {"url": video_data_url},
62
+ "uuid": client_uuid,
63
+ },
64
+ {"type": "text", "text": "Describe the video."},
65
+ ],
66
+ }
67
+ ]
68
+
69
+
70
+def test_mm_hash_includes_uuid_and_processing_kwargs():
71
+ renderer = _build_video_renderer()
72
+ messages = _video_chat_messages(video_data_url)
73
+
74
+ base_hash = _get_message_hash(renderer, messages)
75
+ media_io_hash = _get_message_hash(
76
+ renderer,
77
+ messages,
78
+ media_io_kwargs={"video": {"num_frames": 4}},
79
+ )
80
+ uuid_hash = _get_message_hash(
81
+ renderer,
82
+ _video_chat_messages("different-video-uuid"),
83
+ )
84
+ mm_processor_hash = _get_message_hash(
85
+ renderer,
86
+ messages,
87
+ mm_processor_kwargs={"videos_kwargs": {"do_resize": False}},
88
+ )
89
+
90
+ assert base_hash != media_io_hash
91
+ assert base_hash != uuid_hash
92
+ assert base_hash != mm_processor_hash
vllm/inputs/llm.py
+6
@@ -73,6 +73,12 @@ class _PromptOptions(TypedDict):
73
if the model supports it.
74
"""
75
76
+ media_io_kwargs: NotRequired[dict[str, dict[str, Any]] | None]
77
+ """
78
+ Optional per-modality media loading and decoding arguments.
79
+ Used only for hash derivation when using multi-modal UUIDs.
80
+ """
81
+
82
mm_processor_kwargs: NotRequired[dict[str, Any] | None]
83
"""
84
Optional multi-modal processor kwargs to be forwarded to the
vllm/multimodal/processing/inputs.py
+34
-35
@@ -21,6 +21,7 @@ class ProcessorInputs:
21
mm_data_items: MultiModalDataItems
22
mm_uuid_items: MultiModalUUIDItems | None = None
23
hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
24
+ media_io_kwargs: Mapping[str, Mapping[str, object]] = field(default_factory=dict)
25
26
def get_mm_hashes(
27
self,
@@ -29,49 +30,47 @@ class ProcessorInputs:
30
) -> MultiModalHashes:
31
mm_data_items = self.mm_data_items
32
mm_uuid_items = self.mm_uuid_items or {}
32
- hf_processor_mm_kwargs = self.hf_processor_mm_kwargs
33
+
34
+ hash_factors = {
35
+ "media_io_kwargs": dict(self.media_io_kwargs),
36
+ "mm_processor_kwargs": dict(self.hf_processor_mm_kwargs),
37
+ }
38
+ has_hash_factors = any(hash_factors.values())
39
+ hash_factor_kwargs = hash_factors if has_hash_factors else {}
40
41
mm_hashes = dict[str, list[str]]()
42
hasher = MultiModalHasher
43
44
for modality, data_items in mm_data_items.items():
38
- if modality in mm_uuid_items:
39
- uuid_items = mm_uuid_items[modality]
45
+ uuid_items = (
46
+ mm_uuid_items[modality]
47
+ if modality in mm_uuid_items
48
+ else ([None] * len(data_items))
49
+ )
50
41
- # For None entries, compute a hash; otherwise, use provided ID.
42
- hashes: list[str] = []
43
- for i, item in enumerate(data_items.get_all_items_for_hash()):
44
- uuid_item = uuid_items[i]
51
+ # For None entries, compute a hash; otherwise, use provided ID.
52
+ hashes: list[str] = []
53
+ for i, item in enumerate(data_items.get_all_items_for_hash()):
54
+ uuid_item = uuid_items[i]
55
46
- # NOTE: Even if a uuid_item is provided, we still compute a hash
47
- # if `hf_processor_mm_kwargs` is provided.
48
- # This is because the processed multimodal inputs can be different
49
- # depending on the processor kwargs.
50
- if uuid_item is None or hf_processor_mm_kwargs:
51
- # NOTE: use provided hash string to hash with kwargs
52
- # if available for better performance.
53
- item = uuid_item if uuid_item is not None else item
54
- hashes.append(
55
- hasher.hash_kwargs(
56
- hash_algorithm,
57
- model_id=model_id,
58
- **{modality: item},
59
- **hf_processor_mm_kwargs,
60
- )
56
+ # NOTE: Even if a uuid_item is provided, model output
57
+ # depends on hash_factor_kwargs, so they are taken into account.
58
+ if uuid_item is None or has_hash_factors:
59
+ # NOTE: use provided hash string to hash with kwargs
60
+ # if available for better performance.
61
+ item = uuid_item if uuid_item is not None else item
62
+ hashes.append(
63
+ hasher.hash_kwargs(
64
+ hash_algorithm,
65
+ model_id=model_id,
66
+ **{modality: item},
67
+ **hash_factor_kwargs,
68
)
62
- else:
63
- hashes.append(uuid_item)
64
-
65
- mm_hashes[modality] = hashes
66
- else:
67
- mm_hashes[modality] = [
68
- hasher.hash_kwargs(
69
- hash_algorithm,
70
- model_id=model_id,
71
- **{modality: item},
72
- **hf_processor_mm_kwargs,
69
)
74
- for item in data_items.get_all_items_for_hash()
75
- ]
70
+ else:
71
+ # If there are no extra kwargs, use the client-provided UUID.
72
+ hashes.append(uuid_item)
73
+
74
+ mm_hashes[modality] = hashes
75
76
return mm_hashes
vllm/multimodal/processing/processor.py
+1
@@ -1811,6 +1811,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
1811
inputs.mm_data_items,
1812
inputs.mm_uuid_items,
1813
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
1814
+ media_io_kwargs=inputs.media_io_kwargs,
1815
)
1816
1817
encoder_inputs = super().apply(encoder_processor_inputs, timing_ctx)
vllm/renderers/base.py
+8
@@ -759,6 +759,7 @@ class BaseRenderer(ABC, Generic[_T]):
759
mm_data: MultiModalDataDict,
760
mm_uuids: MultiModalUUIDDict | None,
761
mm_processor_kwargs: Mapping[str, object] | None,
762
+ media_io_kwargs: Mapping[str, Mapping[str, object]] | None = None,
763
*,
764
skip_mm_cache: bool = False,
765
) -> "MultiModalInput":
@@ -781,6 +782,7 @@ class BaseRenderer(ABC, Generic[_T]):
782
mm_data_items,
783
mm_uuid_items,
784
hf_processor_mm_kwargs=mm_processor_kwargs or {},
785
+ media_io_kwargs=media_io_kwargs or {},
786
)
787
mm_timing_ctx = self._mm_timing_registry.get(mm_req_id)
788
@@ -809,6 +811,7 @@ class BaseRenderer(ABC, Generic[_T]):
811
multi_modal_data,
812
mm_processor_kwargs=prompt.get("mm_processor_kwargs"),
813
mm_uuids=prompt.get("multi_modal_uuids"),
814
+ media_io_kwargs=prompt.get("media_io_kwargs"),
815
skip_mm_cache=skip_mm_cache,
816
)
817
else:
@@ -871,6 +874,7 @@ class BaseRenderer(ABC, Generic[_T]):
874
multi_modal_data,
875
mm_processor_kwargs=prompt.get("mm_processor_kwargs"),
876
mm_uuids=prompt.get("multi_modal_uuids"),
877
+ media_io_kwargs=prompt.get("media_io_kwargs"),
878
skip_mm_cache=skip_mm_cache,
879
)
880
else:
@@ -1082,6 +1086,8 @@ class BaseRenderer(ABC, Generic[_T]):
1086
1087
tok_prompts = self.tokenize_prompts(dict_prompts, tok_params)
1088
1089
+ prompt_extras = dict(prompt_extras or {})
1090
+ prompt_extras["media_io_kwargs"] = chat_params.media_io_kwargs or {}
1091
self._apply_prompt_extras(tok_prompts, prompt_extras)
1092
1093
eng_prompts = [
@@ -1118,6 +1124,8 @@ class BaseRenderer(ABC, Generic[_T]):
1124
1125
tok_prompts = await self.tokenize_prompts_async(dict_prompts, tok_params)
1126
1127
+ prompt_extras = dict(prompt_extras or {})
1128
+ prompt_extras["media_io_kwargs"] = chat_params.media_io_kwargs or {}
1129
self._apply_prompt_extras(tok_prompts, prompt_extras)
1130
1131
eng_prompts = await asyncio.gather(