| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import contextlib |
| 4 | import hashlib |
| 5 | import struct |
| 6 | import uuid |
| 7 | from io import BytesIO |
| 8 | from pathlib import Path |
| 9 | |
| 10 | import numpy as np |
| 11 | import pytest |
| 12 | import torch |
| 13 | from PIL import Image, ImageDraw, ImageOps |
| 14 | |
| 15 | from vllm.config.multimodal import MMHasherAlgorithm |
| 16 | from vllm.multimodal.hasher import MultiModalHasher |
| 17 | from vllm.multimodal.media.base import MediaWithBytes |
| 18 | from vllm.multimodal.media.image import ImageMediaIO |
| 19 | from vllm.multimodal.parse import MultiModalDataParser |
| 20 | |
| 21 | pytestmark = pytest.mark.cpu_test |
| 22 | |
| 23 | ASSETS_DIR = Path(__file__).parent / "assets" |
| 24 | assert ASSETS_DIR.exists() |
| 25 | |
| 26 | |
| 27 | @pytest.mark.parametrize("algorithm", ["sha256", "sha512"]) |
| 28 | def test_hash_algorithm(algorithm: MMHasherAlgorithm): |
| 29 | hasher = getattr(hashlib, algorithm)() |
| 30 | for bytes_ in MultiModalHasher.iter_item_to_bytes("value", "test"): |
| 31 | hasher.update(bytes_) |
| 32 | |
| 33 | assert MultiModalHasher.hash_kwargs(algorithm, value="test") == hasher.hexdigest() |
| 34 | |
| 35 | |
| 36 | def test_hash_algorithm_required(): |
| 37 | with pytest.raises(TypeError, match="algorithm"): |
| 38 | MultiModalHasher.hash_kwargs(value="test") # type: ignore[call-arg] |
| 39 | |
| 40 | |
| 41 | def test_hash_single_item_different_shape(): |
| 42 | x1 = torch.zeros(()) |
| 43 | x2 = torch.zeros((1,)) |
| 44 | |
| 45 | hasher = MultiModalHasher |
| 46 | assert hasher.hash_kwargs("blake3", x=x1) != hasher.hash_kwargs("blake3", x=x2) |
| 47 | |
| 48 | |
| 49 | def test_hash_key_order_invariant(): |
| 50 | x = torch.zeros((5, 10)) |
| 51 | y = torch.ones((5, 10)) |
| 52 | |
| 53 | hasher = MultiModalHasher |
| 54 | assert hasher.hash_kwargs("blake3", x=x, y=y) == hasher.hash_kwargs( |
| 55 | "blake3", y=y, x=x |
| 56 | ) |
| 57 | |
| 58 | |
| 59 | # NOTE: Images that are the same visually are allowed to have the same hash |
| 60 | @pytest.mark.parametrize("mode_pair", [("1", "L"), ("RGBA", "CMYK")]) |
| 61 | def test_hash_collision_image_mode(mode_pair): |
| 62 | mode1, mode2 = mode_pair |
| 63 | image1 = Image.new(mode1, size=(10, 10), color=1) |
| 64 | image2 = Image.new(mode2, size=(10, 10), color=1) |
| 65 | |
| 66 | hasher = MultiModalHasher |
| 67 | assert hasher.hash_kwargs("blake3", image=image1) != hasher.hash_kwargs( |
| 68 | "blake3", image=image2 |
| 69 | ) |
| 70 | |
| 71 | |
| 72 | def test_hash_collision_image_palette(): |
| 73 | # These images differ only in Image.palette._palette |
| 74 | image1 = Image.open(ASSETS_DIR / "image1.png") |
| 75 | image2 = Image.open(ASSETS_DIR / "image2.png") |
| 76 | |
| 77 | hasher = MultiModalHasher |
| 78 | assert hasher.hash_kwargs("blake3", image=image1) != hasher.hash_kwargs( |
| 79 | "blake3", image=image2 |
| 80 | ) |
| 81 | |
| 82 | |
| 83 | def test_hash_collision_image_transpose(): |
| 84 | image1 = Image.new("1", size=(10, 20)) |
| 85 | ImageDraw.Draw(image1).line([(0, 0), (10, 0)]) |
| 86 | |
| 87 | image2 = Image.new("1", size=(20, 10)) |
| 88 | ImageDraw.Draw(image2).line([(0, 0), (0, 10)]) |
| 89 | |
| 90 | hasher = MultiModalHasher |
| 91 | assert hasher.hash_kwargs("blake3", image=image1) != hasher.hash_kwargs( |
| 92 | "blake3", image=image2 |
| 93 | ) |
| 94 | |
| 95 | |
| 96 | @pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) |
| 97 | def test_hash_collision_tensor_shape(dtype): |
| 98 | # The hash should be different though the data is the same when flattened |
| 99 | arr1 = torch.zeros((5, 10, 20, 3), dtype=dtype) |
| 100 | arr2 = torch.zeros((10, 20, 5, 3), dtype=dtype) |
| 101 | |
| 102 | hasher = MultiModalHasher |
| 103 | assert hasher.hash_kwargs("blake3", data=arr1) != hasher.hash_kwargs( |
| 104 | "blake3", data=arr2 |
| 105 | ) |
| 106 | |
| 107 | |
| 108 | def test_hash_collision_array_shape(): |
| 109 | # The hash should be different though the data is the same when flattened |
| 110 | arr1 = np.zeros((5, 10, 20, 3)) |
| 111 | arr2 = np.zeros((10, 20, 5, 3)) |
| 112 | |
| 113 | hasher = MultiModalHasher |
| 114 | assert hasher.hash_kwargs("blake3", data=arr1) != hasher.hash_kwargs( |
| 115 | "blake3", data=arr2 |
| 116 | ) |
| 117 | |
| 118 | |
| 119 | def test_hash_collision_video_num_frames(): |
| 120 | source = b"x" * 100 |
| 121 | |
| 122 | def item_for_hash(num_frames: int): |
| 123 | frames: np.ndarray = np.zeros((num_frames, 8, 8, 3), dtype=np.uint8) |
| 124 | metadata = { |
| 125 | "total_num_frames": 16, |
| 126 | "fps": 2.0, |
| 127 | "duration": 8.0, |
| 128 | "video_backend": "opencv", |
| 129 | "frames_indices": list(range(num_frames)), |
| 130 | "do_sample_frames": False, |
| 131 | } |
| 132 | video = MediaWithBytes((frames, metadata), source) |
| 133 | items = MultiModalDataParser()._parse_video_data([video]) |
| 134 | assert items is not None |
| 135 | return items.get_all_items_for_hash()[0] |
| 136 | |
| 137 | hasher = MultiModalHasher |
| 138 | assert hasher.hash_kwargs("blake3", video=item_for_hash(2)) != hasher.hash_kwargs( |
| 139 | "blake3", video=item_for_hash(4) |
| 140 | ) |
| 141 | |
| 142 | |
| 143 | def test_hash_video_tensor_frames(): |
| 144 | """Videos holding tensor frames (e.g. NVDEC-decoded) hash like |
| 145 | array-framed ones, from the original bytes without a D2H copy.""" |
| 146 | source = b"x" * 100 |
| 147 | |
| 148 | def item_for_hash(frames): |
| 149 | metadata = { |
| 150 | "total_num_frames": 2, |
| 151 | "fps": 2.0, |
| 152 | "duration": 1.0, |
| 153 | "video_backend": "torchcodec", |
| 154 | "frames_indices": [0, 1], |
| 155 | "do_sample_frames": False, |
| 156 | } |
| 157 | video = MediaWithBytes((frames, metadata), source) |
| 158 | items = MultiModalDataParser()._parse_video_data([video]) |
| 159 | assert items is not None |
| 160 | return items.get_all_items_for_hash()[0] |
| 161 | |
| 162 | np_frames = np.zeros((2, 8, 8, 3), dtype=np.uint8) |
| 163 | torch_frames = torch.zeros((2, 8, 8, 3), dtype=torch.uint8) |
| 164 | |
| 165 | hasher = MultiModalHasher |
| 166 | assert hasher.hash_kwargs("blake3", video=item_for_hash(np_frames)) == ( |
| 167 | hasher.hash_kwargs("blake3", video=item_for_hash(torch_frames)) |
| 168 | ) |
| 169 | |
| 170 | |
| 171 | def test_hash_non_contiguous_array(): |
| 172 | arr = np.arange(24).reshape(4, 6).T |
| 173 | assert not arr.flags.c_contiguous |
| 174 | |
| 175 | arr_c = np.ascontiguousarray(arr) |
| 176 | assert arr_c.flags.c_contiguous |
| 177 | |
| 178 | hasher = MultiModalHasher |
| 179 | # Both should be hashable and produce the same hashes |
| 180 | assert hasher.hash_kwargs("blake3", data=arr) == hasher.hash_kwargs( |
| 181 | "blake3", data=arr_c |
| 182 | ) |
| 183 | |
| 184 | |
| 185 | def test_hash_image_exif_id(): |
| 186 | # Test that EXIF ImageId tag can be used to store UUID |
| 187 | # and the hasher will use that instead of the image data. |
| 188 | image1 = image2 = Image.new("1", size=(10, 20)) |
| 189 | id = uuid.uuid4() |
| 190 | image1.getexif()[Image.ExifTags.Base.ImageID] = id |
| 191 | image2 = Image.open(ASSETS_DIR / "image1.png") |
| 192 | image2.getexif()[Image.ExifTags.Base.ImageID] = "Not a UUID" |
| 193 | image2a = Image.open(ASSETS_DIR / "image1.png") |
| 194 | |
| 195 | hasher = MultiModalHasher |
| 196 | # first image has UUID in ImageID, so it should hash to that UUID |
| 197 | assert hasher.hash_kwargs("blake3", image=image1) == hasher.hash_kwargs( |
| 198 | "blake3", image=id.bytes |
| 199 | ) |
| 200 | # second image has non-UUID in ImageID, so it should hash to the image data |
| 201 | assert hasher.hash_kwargs("blake3", image=image2) == hasher.hash_kwargs( |
| 202 | "blake3", image=image2a |
| 203 | ) |
| 204 | |
| 205 | |
| 206 | def test_hash_image_malformed_exif(): |
| 207 | # Test that images with malformed EXIF headers (e.g. invalid TIFF header) |
| 208 | # do not raise an unhandled exception during hashing and fall back to image data. |
| 209 | buf = BytesIO() |
| 210 | Image.new("RGB", (64, 48)).save(buf, "JPEG") |
| 211 | jpg = buf.getvalue() |
| 212 | rest = jpg[2:] |
| 213 | rest = rest[2 + struct.unpack(">H", rest[2:4])[0] :] |
| 214 | payload = b"Exif\x00\x00XXXX\x00\x00\x00\x08" + bytes(32) |
| 215 | data = b"\xff\xd8\xff\xe1" + struct.pack(">H", len(payload) + 2) + payload + rest |
| 216 | |
| 217 | image = Image.open(BytesIO(data)) |
| 218 | with contextlib.suppress(Exception): |
| 219 | image = ImageOps.exif_transpose(image) |
| 220 | image.load() |
| 221 | |
| 222 | hasher = MultiModalHasher |
| 223 | # Should hash without raising SyntaxError or any other exception |
| 224 | hash_val = hasher.hash_kwargs("blake3", image=image) |
| 225 | assert isinstance(hash_val, str) and len(hash_val) > 0 |
| 226 | |
| 227 | # Also verify MediaWithBytes wrapping the image with malformed EXIF |
| 228 | media_item = MediaWithBytes(image, data) |
| 229 | hash_media = hasher.hash_kwargs("blake3", image=media_item) |
| 230 | assert isinstance(hash_media, str) and len(hash_media) > 0 |
| 231 | |
| 232 | |
| 233 | def _rgba_png_bytes() -> bytes: |
| 234 | image = Image.new("RGBA", (8, 8), (255, 0, 0, 128)) |
| 235 | buf = BytesIO() |
| 236 | image.save(buf, format="PNG") |
| 237 | return buf.getvalue() |
| 238 | |
| 239 | |
| 240 | def test_hash_collision_media_io_config(): |
| 241 | data = _rgba_png_bytes() |
| 242 | white = ImageMediaIO(rgba_background_color=(255, 255, 255)).load_bytes(data) |
| 243 | black = ImageMediaIO(rgba_background_color=(0, 0, 0)).load_bytes(data) |
| 244 | white2 = ImageMediaIO(rgba_background_color=(255, 255, 255)).load_bytes(data) |
| 245 | keep = ImageMediaIO(image_mode=None).load_bytes(data) |
| 246 | |
| 247 | hasher = MultiModalHasher |
| 248 | assert hasher.hash_kwargs("blake3", image=white) != hasher.hash_kwargs( |
| 249 | "blake3", image=black |
| 250 | ) |
| 251 | assert hasher.hash_kwargs("blake3", image=white) != hasher.hash_kwargs( |
| 252 | "blake3", image=keep |
| 253 | ) |
| 254 | assert hasher.hash_kwargs("blake3", image=white) == hasher.hash_kwargs( |
| 255 | "blake3", image=white2 |
| 256 | ) |
| 257 | |
| 258 | |
| 259 | def test_hash_media_io_noop_config_preserves_hash(): |
| 260 | image = Image.new("RGB", (8, 8), (0, 128, 255)) |
| 261 | buf = BytesIO() |
| 262 | image.save(buf, format="PNG") |
| 263 | data = buf.getvalue() |
| 264 | |
| 265 | loaded = ImageMediaIO().load_bytes(data) |
| 266 | assert loaded.io_config is None |
| 267 | |
| 268 | plain = MediaWithBytes(loaded.media, data) |
| 269 | hasher = MultiModalHasher |
| 270 | assert hasher.hash_kwargs("blake3", image=loaded) == hasher.hash_kwargs( |
| 271 | "blake3", image=plain |
| 272 | ) |
| 273 | |
| 274 | |
| 275 | # The digest input is a concatenation of byte chunks, so it has to be uniquely |
| 276 | # decodable. Each case below is a pair of distinct processor kwargs that used to |
| 277 | # serialize to the same bytes, which made two requests share an mm hash -- and |
| 278 | # therefore share both the processor cache entry and the prefix-cache block key. |
| 279 | IMAGE = b"\x89PNG\r\n\x1a\n" |
| 280 | |
| 281 | |
| 282 | def _hash(**mm_processor_kwargs: object) -> str: |
| 283 | return MultiModalHasher.hash_kwargs( |
| 284 | "blake3", model_id="m", image=IMAGE, **mm_processor_kwargs |
| 285 | ) |
| 286 | |
| 287 | |
| 288 | def test_hash_collision_kwargs_key_value_boundary(): |
| 289 | # Both used to flatten to b"ab" + b"c". |
| 290 | assert _hash(**{"ab": "c"}) != _hash(**{"a": "bc"}) |
| 291 | |
| 292 | |
| 293 | def test_hash_collision_nested_vs_flattened_key(): |
| 294 | nested = _hash(size={"shortest_edge": 224}) |
| 295 | flattened = _hash(**{"size.shortest_edge": 224}) |
| 296 | assert nested != flattened |
| 297 | |
| 298 | |
| 299 | def test_hash_collision_sequence_vs_mapping(): |
| 300 | assert _hash(fps=[2, 4]) != _hash(fps={"0": 2, "1": 4}) |
| 301 | |
| 302 | |
| 303 | @pytest.mark.parametrize("empty", ["", b"", []]) |
| 304 | def test_hash_collision_none_vs_empty(empty): |
| 305 | assert _hash(video_pruning_rate=None) != _hash(video_pruning_rate=empty) |
| 306 | |
| 307 | |
| 308 | def test_hash_collision_empty_container_vs_omitted(): |
| 309 | omitted = _hash() |
| 310 | assert _hash(size={}) != omitted |
| 311 | assert _hash(size=[]) != omitted |
| 312 | assert _hash(size={}) != _hash(size=[]) |