[Bugfix][Multimodal] Frame the multi-modal hash digest input (#54283)
Signed-off-by: hotragn <hotragn.pettugani_2024@woxsen.edu.in> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Hotragn Pettugani committed
Sep 19, 2026 at 00:06 UTC
a8d1aa9c99b8698a2a78b611b7a10c30e6b3995b
3 files changed
+138
-14
tests/multimodal/test_hasher.py
+40
@@ -268,3 +268,43 @@ def test_hash_media_io_noop_config_preserves_hash():
268
assert hasher.hash_kwargs("blake3", image=loaded) == hasher.hash_kwargs(
269
"blake3", image=plain
270
)
271
+
272
+
273
+# The digest input is a concatenation of byte chunks, so it has to be uniquely
274
+# decodable. Each case below is a pair of distinct processor kwargs that used to
275
+# serialize to the same bytes, which made two requests share an mm hash -- and
276
+# therefore share both the processor cache entry and the prefix-cache block key.
277
+IMAGE = b"\x89PNG\r\n\x1a\n"
278
+
279
+
280
+def _hash(**mm_processor_kwargs: object) -> str:
281
+ return MultiModalHasher.hash_kwargs(
282
+ "blake3", model_id="m", image=IMAGE, **mm_processor_kwargs
283
+ )
284
+
285
+
286
+def test_hash_collision_kwargs_key_value_boundary():
287
+ # Both used to flatten to b"ab" + b"c".
288
+ assert _hash(**{"ab": "c"}) != _hash(**{"a": "bc"})
289
+
290
+
291
+def test_hash_collision_nested_vs_flattened_key():
292
+ nested = _hash(size={"shortest_edge": 224})
293
+ flattened = _hash(**{"size.shortest_edge": 224})
294
+ assert nested != flattened
295
+
296
+
297
+def test_hash_collision_sequence_vs_mapping():
298
+ assert _hash(fps=[2, 4]) != _hash(fps={"0": 2, "1": 4})
299
+
300
+
301
+@pytest.mark.parametrize("empty", ["", b"", []])
302
+def test_hash_collision_none_vs_empty(empty):
303
+ assert _hash(video_pruning_rate=None) != _hash(video_pruning_rate=empty)
304
+
305
+
306
+def test_hash_collision_empty_container_vs_omitted():
307
+ omitted = _hash()
308
+ assert _hash(size={}) != omitted
309
+ assert _hash(size=[]) != omitted
310
+ assert _hash(size={}) != _hash(size=[])
tests/multimodal/test_processing.py
+37
@@ -1324,3 +1324,40 @@ def test_processor_inputs_hashes_ignore_unrelated_kwargs():
1324
)
1325
1326
assert inputs.get_mm_hashes("test-model", "blake3") == {"image": ["image-uuid"]}
1327
+
1328
+
1329
+@pytest.mark.parametrize(
1330
+ ("left", "right"),
1331
+ [
1332
+ # Shifting the key/value boundary: both flatten to the dotted key
1333
+ # "mm_processor_kwargs.abc" followed by no value bytes.
1334
+ ({"ab": "c"}, {"a": "bc"}),
1335
+ # A nested mapping and a caller-supplied dotted key flatten alike.
1336
+ ({"size": {"shortest_edge": 224}}, {"size.shortest_edge": 224}),
1337
+ # A sequence and a mapping keyed by stringified indices flatten alike.
1338
+ ({"fps": [2, 4]}, {"fps": {"0": 2, "1": 4}}),
1339
+ # None contributes the key alone, which a zero-byte value also does.
1340
+ ({"video_pruning_rate": None}, {"video_pruning_rate": ""}),
1341
+ # An empty container contributes nothing, as does omitting the key.
1342
+ ({"size": {}}, {}),
1343
+ ],
1344
+)
1345
+def test_processor_inputs_hashes_distinguish_kwargs_shapes(left, right):
1346
+ """Distinct processor kwargs must not share a multi-modal hash.
1347
+
1348
+ ``hf_processor_mm_kwargs`` is per-request input, so both the keys and the
1349
+ values here are caller-controlled. The hash is the identity of the
1350
+ processor cache entry and is mixed into the prefix-cache block key, so two
1351
+ requests sharing one is a cross-request cache hit.
1352
+ """
1353
+ image = random_image(np.random.RandomState(0), min_wh=8, max_wh=9)
1354
+ mm_data_items = MultiModalDataParser().parse_mm_data({"image": [image]})
1355
+
1356
+ def hash_with(hf_processor_mm_kwargs):
1357
+ return ProcessorInputs(
1358
+ prompt=[],
1359
+ mm_data_items=mm_data_items,
1360
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1361
+ ).get_mm_hashes("test-model", "blake3")["image"][0]
1362
+
1363
+ assert hash_with(left) != hash_with(right)
vllm/multimodal/hasher.py
+61
-14
@@ -18,6 +18,28 @@ from .media import MediaWithBytes
18
19
logger = init_logger(__name__)
20
21
+# Framing for the digest input. The hash is built by feeding a stream of byte
22
+# chunks to `hasher.update`, so the stream must be uniquely decodable: without
23
+# an explicit length in front of every chunk and a tag in front of every
24
+# container, distinct inputs serialize to identical bytes and share a cache
25
+# entry. See `test_hash_collision_*` in tests/multimodal/test_hasher.py.
26
+_LENGTH_BYTES = 8
27
+_TAG_NONE = b"\x00"
28
+_TAG_SEQUENCE = b"\x01"
29
+_TAG_MAPPING = b"\x02"
30
+_TAG_LEAF = b"\x03"
31
+
32
+
33
+def _encode_length(value: int) -> bytes:
34
+ return value.to_bytes(_LENGTH_BYTES, "little")
35
+
36
+
37
+def _framed(chunk: bytes | memoryview) -> Iterable[bytes | memoryview]:
38
+ """Yield *chunk* preceded by its size in bytes."""
39
+ size = chunk.nbytes if isinstance(chunk, memoryview) else len(chunk)
40
+ yield _encode_length(size)
41
+ yield chunk
42
+
43
44
@functools.lru_cache(maxsize=3)
45
def _get_hasher_factory(
@@ -61,20 +83,27 @@ def _get_image_id_bytes(image: Image.Image) -> bytes | None:
83
84
85
class MultiModalHasher:
86
+ """Derives multi-modal cache keys.
87
+
88
+ Every method here yields *framed* chunks: each chunk is preceded by its
89
+ length and each container by its kind, so that the concatenation fed to the
90
+ digest is uniquely decodable and distinct inputs cannot share a key.
91
+ """
92
+
93
@classmethod
94
def serialize_item(cls, obj: object) -> Iterable[bytes | memoryview]:
95
# Simple cases
96
if isinstance(obj, (bytes, memoryview)):
68
- return (obj,)
97
+ return _framed(obj)
98
if isinstance(obj, str):
70
- return (obj.encode("utf-8"),)
99
+ return _framed(obj.encode("utf-8"))
100
if isinstance(obj, (int, float)):
72
- return (np.array(obj).tobytes(),)
101
+ return _framed(np.array(obj).tobytes())
102
103
if isinstance(obj, Image.Image):
104
image_id = _get_image_id_bytes(obj)
105
if image_id is not None:
77
- return (image_id,)
106
+ return _framed(image_id)
107
108
data = {"mode": obj.mode, "data": np.asarray(obj)}
109
palette = obj.palette
@@ -88,7 +117,7 @@ class MultiModalHasher:
117
if isinstance(obj, MediaWithBytes) and isinstance(obj.media, Image.Image):
118
image_id = _get_image_id_bytes(obj.media)
119
if image_id is not None:
91
- return (image_id,)
120
+ return _framed(image_id)
121
122
if obj.io_config:
123
return cls.iter_item_to_bytes(
@@ -151,7 +180,7 @@ class MultiModalHasher:
180
"No serialization method found for %s. Falling back to pickle.", type(obj)
181
)
182
154
- return (pickle.dumps(obj),)
183
+ return _framed(pickle.dumps(obj))
184
185
@classmethod
186
def iter_item_to_bytes(
@@ -159,18 +188,36 @@ class MultiModalHasher:
188
key: str,
189
obj: object,
190
) -> Iterable[bytes | memoryview]:
191
+ """Yield the digest input for a single ``key``/``obj`` pair."""
192
+ yield from _framed(key.encode("utf-8"))
193
+ yield from cls.iter_value_to_bytes(obj)
194
+
195
+ @classmethod
196
+ def iter_value_to_bytes(
197
+ cls,
198
+ obj: object,
199
+ ) -> Iterable[bytes | memoryview]:
200
+ """Yield the digest input for a value, tagged by its container kind.
201
+
202
+ Containers carry their kind and length so that a nested structure can
203
+ never serialize to the same bytes as a differently shaped one (a list
204
+ and a mapping keyed by stringified indices, for example).
205
+ """
206
if obj is None:
163
- yield key.encode("utf-8")
164
- return
165
- # Recursive cases
166
- if isinstance(obj, (list, tuple)):
167
- for i, elem in enumerate(obj):
168
- yield from cls.iter_item_to_bytes(f"{key}.{i}", elem)
207
+ yield _TAG_NONE
208
+ elif isinstance(obj, (list, tuple)):
209
+ yield _TAG_SEQUENCE
210
+ yield _encode_length(len(obj))
211
+ for elem in obj:
212
+ yield from cls.iter_value_to_bytes(elem)
213
elif isinstance(obj, dict):
214
+ yield _TAG_MAPPING
215
+ yield _encode_length(len(obj))
216
for k, v in obj.items():
171
- yield from cls.iter_item_to_bytes(f"{key}.{k}", v)
217
+ yield from _framed(str(k).encode("utf-8"))
218
+ yield from cls.iter_value_to_bytes(v)
219
else:
173
- yield key.encode("utf-8")
220
+ yield _TAG_LEAF
221
yield from cls.serialize_item(obj)
222
223
@classmethod