[Perf] Hash videos by source bytes (#49607)
Signed-off-by: Guan-Ming (Wesley) Chiu <105915352+guan404ming@users.noreply.github.com> Signed-off-by: Guan-Ming Chiu <105915352+guan404ming@users.noreply.github.com> Co-authored-by: Isotr0py <2037008807@qq.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Guan-Ming Chiu committed
Jul 27, 2026 at 23:57 UTC
04502deca2f66b874d0fceae446c668c987413c0
8 files changed
+94
-21
tests/multimodal/test_hasher.py
+25
@@ -9,6 +9,8 @@ import torch
9
from PIL import Image, ImageDraw
10
11
from vllm.multimodal.hasher import MultiModalHasher
12
+from vllm.multimodal.media.base import MediaWithBytes
13
+from vllm.multimodal.parse import MultiModalDataParser
14
15
pytestmark = pytest.mark.cpu_test
16
@@ -82,6 +84,29 @@ def test_hash_collision_array_shape():
84
assert hasher.hash_kwargs(data=arr1) != hasher.hash_kwargs(data=arr2)
85
86
87
+def test_hash_collision_video_num_frames():
88
+ source = b"x" * 100
89
+
90
+ def item_for_hash(num_frames: int):
91
+ frames: np.ndarray = np.zeros((num_frames, 8, 8, 3), dtype=np.uint8)
92
+ metadata = {
93
+ "total_num_frames": 16,
94
+ "fps": 2.0,
95
+ "duration": 8.0,
96
+ "video_backend": "opencv",
97
+ "frames_indices": list(range(num_frames)),
98
+ "do_sample_frames": False,
99
+ }
100
+ video = MediaWithBytes((frames, metadata), source)
101
+ items = MultiModalDataParser()._parse_video_data([video])
102
+ return items.get_all_items_for_hash()[0]
103
+
104
+ hasher = MultiModalHasher
105
+ assert hasher.hash_kwargs(video=item_for_hash(2)) != hasher.hash_kwargs(
106
+ video=item_for_hash(4)
107
+ )
108
+
109
+
110
def test_hash_non_contiguous_array():
111
arr = np.arange(24).reshape(4, 6).T
112
assert not arr.flags.c_contiguous
vllm/multimodal/hasher.py
+6
@@ -83,6 +83,12 @@ class MultiModalHasher:
83
84
return cls.iter_item_to_bytes("image", obj.original_bytes)
85
86
+ if isinstance(obj, MediaWithBytes) and isinstance(obj.media, np.ndarray):
87
+ frames = obj.media
88
+ if frames.nbytes < len(obj.original_bytes):
89
+ return cls.iter_item_to_bytes("video", frames)
90
+ return cls.iter_item_to_bytes("video", obj.original_bytes)
91
+
92
if isinstance(obj, torch.Tensor):
93
tensor_obj: torch.Tensor = obj.cpu()
94
tensor_dtype = tensor_obj.dtype
vllm/multimodal/inputs.py
+4
-1
@@ -66,7 +66,10 @@ these are directly passed to the model without HF processing.
66
"""
67
68
VideoItem: TypeAlias = Union[
69
- HfVideoItem, "torch.Tensor", tuple[HfVideoItem, dict[str, Any]]
69
+ HfVideoItem,
70
+ "torch.Tensor",
71
+ tuple[HfVideoItem, dict[str, Any]],
72
+ MediaWithBytes[tuple[HfVideoItem, dict[str, Any]]],
73
]
74
"""
75
A `transformers.video_utils.VideoInput` representing a single video item.
vllm/multimodal/media/base.py
+12
-2
@@ -2,9 +2,10 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
from abc import ABC, abstractmethod
5
+from collections.abc import Iterable, Iterator
6
from dataclasses import dataclass, field
7
from pathlib import Path
7
-from typing import Any, Generic, TypeVar
8
+from typing import Any, Generic, TypeVar, cast
9
10
import numpy as np
11
@@ -22,7 +23,8 @@ class MediaWithBytes(Generic[_T]):
23
The wrapper delegates attribute access to the underlying media object,
24
making it behave transparently like the wrapped type (e.g., PIL.Image).
25
25
- NOTE: Currently, this wrapper is used only for the image modality.
26
+ NOTE: Currently, this wrapper is used only for the image and video
27
+ modalities.
28
"""
29
30
media: _T
@@ -32,6 +34,14 @@ class MediaWithBytes(Generic[_T]):
34
"""Allow np.array(obj) to return np.array(obj.media)."""
35
return np.array(self.media, *args, **kwargs)
36
37
+ def __iter__(self) -> Iterator[Any]:
38
+ """Allow unpacking obj to unpack obj.media (e.g. video tuples)."""
39
+ return iter(cast(Iterable[Any], self.media))
40
+
41
+ def __getitem__(self, index: Any) -> Any:
42
+ """Allow obj[i] to index obj.media (e.g. video tuples)."""
43
+ return cast(Any, self.media)[index]
44
+
45
def __getstate__(self):
46
return self.__dict__.copy()
47
vllm/multimodal/media/connector.py
+3
-3
@@ -29,7 +29,7 @@ from vllm.multimodal.video import get_video_loader_backend_for_processor
29
from vllm.utils.registry import ExtensionManager
30
31
from .audio import AudioEmbeddingMediaIO, AudioMediaIO
32
-from .base import MediaIO
32
+from .base import MediaIO, MediaWithBytes
33
from .image import ImageEmbeddingMediaIO, ImageMediaIO
34
from .video import VideoMediaIO
35
@@ -536,7 +536,7 @@ class MediaConnector:
536
*,
537
image_mode: str | None = "RGB",
538
video_processor: str | None = None,
539
- ) -> tuple[npt.NDArray, dict[str, Any]]:
539
+ ) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
540
"""
541
Load video from an HTTP or base64 data URL.
542
"""
@@ -562,7 +562,7 @@ class MediaConnector:
562
*,
563
image_mode: str | None = "RGB",
564
video_processor: str | None = None,
565
- ) -> tuple[npt.NDArray, dict[str, Any]]:
565
+ ) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
566
"""
567
Asynchronously load video from an HTTP or base64 data URL.
568
vllm/multimodal/media/video.py
+12
-7
@@ -13,13 +13,13 @@ from vllm import envs
13
from vllm.logger import init_logger
14
15
from ..video import VIDEO_LOADER_REGISTRY
16
-from .base import MediaIO
16
+from .base import MediaIO, MediaWithBytes
17
from .image import ImageMediaIO
18
19
logger = init_logger(__name__)
20
21
22
-class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]):
22
+class VideoMediaIO(MediaIO[MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]]):
23
"""Configuration values can be user-provided either by --media-io-kwargs or
24
by the runtime API field "media_io_kwargs". Ensure proper validation and
25
error handling.
@@ -91,14 +91,17 @@ class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]):
91
self.kwargs = kwargs
92
self.video_loader = VIDEO_LOADER_REGISTRY.load(video_loader_backend)
93
94
- def load_bytes(self, data: bytes) -> tuple[npt.NDArray, dict[str, Any]]:
95
- return self.video_loader.load_bytes(
94
+ def load_bytes(
95
+ self, data: bytes
96
+ ) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
97
+ video = self.video_loader.load_bytes(
98
data, num_frames=self.num_frames, **self.kwargs
99
)
100
+ return MediaWithBytes(video, data)
101
102
def load_base64(
103
self, media_type: str, data: str
101
- ) -> tuple[npt.NDArray, dict[str, Any]]:
104
+ ) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
105
if media_type.lower() == "video/jpeg":
106
load_frame = partial(
107
self.image_io.load_base64,
@@ -160,11 +163,13 @@ class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]):
163
"frames_indices": frames_indices,
164
"do_sample_frames": self.kwargs.get("do_sample_frames", False),
165
}
163
- return frames, metadata
166
+ return MediaWithBytes((frames, metadata), data.encode())
167
168
return self.load_bytes(pybase64.b64decode(data))
169
167
- def load_file(self, filepath: Path) -> tuple[npt.NDArray, dict[str, Any]]:
170
+ def load_file(
171
+ self, filepath: Path
172
+ ) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
173
with filepath.open("rb") as f:
174
data = f.read()
175
vllm/multimodal/parse.py
+24
-6
@@ -366,6 +366,20 @@ class VideoProcessorItems(ProcessorBatchItems[HfVideoItem | None]):
366
367
self.metadata = metadata
368
369
+ def _unwrap(self, item: Any) -> Any:
370
+ if isinstance(item, tuple):
371
+ frames, metadata = item
372
+ return super()._unwrap(frames), metadata
373
+ return super()._unwrap(item)
374
+
375
+ def get_item_for_hash(self, index: int) -> Any:
376
+ item = self.data[index]
377
+ if isinstance(item, MediaWithBytes) and isinstance(self.metadata, list):
378
+ metadata = self.metadata[index]
379
+ if metadata is not None:
380
+ return item, metadata
381
+ return item
382
+
383
def get_num_frames(self, item_idx: int) -> int:
384
video = self.get(item_idx)
385
if video is None:
@@ -552,7 +566,10 @@ class MultiModalDataParser:
566
def _get_video_with_metadata(
567
self,
568
video: VideoItem,
555
- ) -> tuple[np.ndarray, dict[str, Any] | None]:
569
+ ) -> tuple[np.ndarray | MediaWithBytes[np.ndarray], dict[str, Any] | None]:
570
+ if isinstance(video, MediaWithBytes):
571
+ new_video, metadata = self._get_video_with_metadata(video.media)
572
+ return MediaWithBytes(new_video, video.original_bytes), metadata
573
if isinstance(video, tuple):
574
return video
575
if isinstance(video, list):
@@ -653,7 +670,11 @@ class MultiModalDataParser:
670
else:
671
data_items = data # type: ignore[assignment]
672
656
- new_videos = list[tuple[np.ndarray, dict[str, Any] | None]]()
673
+ new_videos = list[
674
+ np.ndarray
675
+ | MediaWithBytes[np.ndarray]
676
+ | tuple[np.ndarray | MediaWithBytes[np.ndarray], dict[str, Any]]
677
+ ]()
678
metadata_lst: list[dict[str, Any] | None] = []
679
for data_item in data_items:
680
video, metadata = self._get_video_with_metadata(data_item)
@@ -664,12 +685,9 @@ class MultiModalDataParser:
685
"Please check your video input in `multi_modal_data`"
686
)
687
new_videos.append((video, metadata))
667
- metadata_lst.append(metadata)
688
else:
689
new_videos.append(video)
670
-
671
- if not self.video_needs_metadata:
672
- metadata = None
690
+ metadata_lst.append(metadata)
691
692
return VideoProcessorItems(new_videos, metadata=metadata_lst)
693
vllm/multimodal/utils.py
+8
-2
@@ -24,7 +24,13 @@ from .inputs import (
24
MultiModalKwargsItem,
25
MultiModalSharedField,
26
)
27
-from .media import AudioMediaIO, ImageMediaIO, MediaConnector, VideoMediaIO
27
+from .media import (
28
+ AudioMediaIO,
29
+ ImageMediaIO,
30
+ MediaConnector,
31
+ MediaWithBytes,
32
+ VideoMediaIO,
33
+)
34
35
if TYPE_CHECKING:
36
import torch.types
@@ -332,7 +338,7 @@ def fetch_image(
338
def fetch_video(
339
video_url: str,
340
video_io_kwargs: dict[str, Any] | None = None,
335
-) -> tuple[npt.NDArray, dict[str, Any]]:
341
+) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
342
"""
343
Args:
344
video_url: URL of the video file to fetch.