A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[Misc] Allow using OpenCV as video IO fallback (#15055)
Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Isotr0py committed
Apr 1, 2025 at 23:55 UTC
4e5a0f6ae208c56c169de58a2f3a02c533d9ec00
6 files changed
+84
-35
requirements/common.txt
+1
@@ -30,6 +30,7 @@ msgspec
30
gguf == 0.10.0
31
importlib_metadata
32
mistral_common[opencv] >= 1.5.4
33
+opencv-python-headless >= 4.11.0 # required for video IO
34
pyyaml
35
six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that needs to be the latest version for python 3.12
36
setuptools>=74.1.1; python_version > '3.11' # Setuptools is used by triton, we need to ensure a modern version is installed for 3.12+ so that it does not try to import distutils, which was removed in 3.12
requirements/test.in
+1
-1
@@ -9,7 +9,6 @@ pytest-shard
9
# testing utils
10
awscli
11
backoff # required for phi4mm test
12
-decord # required for video tests
12
einops # required for MPT, qwen-vl and Mamba
13
httpx
14
librosa # required for audio tests
@@ -28,6 +27,7 @@ torchvision==0.21.0
27
transformers_stream_generator # required for qwen-vl test
28
matplotlib # required for qwen-vl test
29
mistral_common[opencv] >= 1.5.4 # required for pixtral test
30
+opencv-python-headless >= 4.11.0 # required for video test
31
datamodel_code_generator # required for minicpm3 test
32
lm-eval[api]==0.4.4 # required for model evaluation test
33
transformers==4.50.3
requirements/test.txt
+4
-5
@@ -93,8 +93,6 @@ datasets==3.0.2
93
# lm-eval
94
decorator==5.1.1
95
# via librosa
96
-decord==0.6.0
97
- # via -r requirements/test.in
96
dill==0.3.8
97
# via
98
# datasets
@@ -276,7 +274,6 @@ numpy==1.26.4
274
# contourpy
275
# cupy-cuda12x
276
# datasets
279
- # decord
277
# einx
278
# encodec
279
# evaluate
@@ -337,8 +334,10 @@ nvidia-nvjitlink-cu12==12.4.127
334
# torch
335
nvidia-nvtx-cu12==12.4.127
336
# via torch
340
-opencv-python-headless==4.10.0.84
341
- # via mistral-common
337
+opencv-python-headless==4.11.0.86
338
+ # via
339
+ # -r requirements/test.in
340
+ # mistral-common
341
packaging==24.1
342
# via
343
# accelerate
setup.py
+1
-1
@@ -684,7 +684,7 @@ setup(
684
"fastsafetensors": ["fastsafetensors >= 0.1.10"],
685
"runai": ["runai-model-streamer", "runai-model-streamer-s3", "boto3"],
686
"audio": ["librosa", "soundfile"], # Required for audio processing
687
- "video": ["decord"] # Required for video processing
687
+ "video": [] # Kept for backwards compatibility
688
},
689
cmdclass=cmdclass,
690
package_data=package_data,
vllm/assets/video.py
+11
-8
@@ -10,8 +10,6 @@ import numpy.typing as npt
10
from huggingface_hub import hf_hub_download
11
from PIL import Image
12
13
-from vllm.multimodal.video import sample_frames_from_video
14
-
13
from .base import get_cache_dir
14
15
@@ -43,14 +41,19 @@ def video_to_ndarrays(path: str, num_frames: int = -1) -> npt.NDArray:
41
42
total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
43
frames = []
46
- for i in range(total_frames):
47
- ret, frame = cap.read()
48
- if ret:
49
- frames.append(frame)
50
- cap.release()
44
+
45
+ num_frames = num_frames if num_frames > 0 else total_frames
46
+ frame_indices = np.linspace(0, total_frames - 1, num_frames, dtype=int)
47
+ for idx in range(total_frames):
48
+ ok = cap.grab() # next img
49
+ if not ok:
50
+ break
51
+ if idx in frame_indices: # only decompress needed
52
+ ret, frame = cap.retrieve()
53
+ if ret:
54
+ frames.append(frame)
55
56
frames = np.stack(frames)
53
- frames = sample_frames_from_video(frames, num_frames)
57
if len(frames) < num_frames:
58
raise ValueError(f"Could not read enough frames from video file {path}"
59
f" (expected {num_frames} frames, got {len(frames)})")
vllm/multimodal/video.py
+66
-20
@@ -13,7 +13,7 @@ from PIL import Image
13
from vllm.inputs.registry import InputContext
14
from vllm.logger import init_logger
15
from vllm.transformers_utils.processor import cached_get_video_processor
16
-from vllm.utils import PlaceholderModule, is_list_of
16
+from vllm.utils import is_list_of
17
18
from .base import MediaIO, ModalityData
19
from .image import ImageMediaIO, ImagePlugin
@@ -22,11 +22,6 @@ from .inputs import MultiModalKwargs, VideoItem
22
if TYPE_CHECKING:
23
from vllm.config import ModelConfig
24
25
-try:
26
- import decord
27
-except ImportError:
28
- decord = PlaceholderModule("decord") # type: ignore[assignment]
29
-
25
logger = init_logger(__name__)
26
27
@@ -117,6 +112,69 @@ def sample_frames_from_video(frames: npt.NDArray,
112
return sampled_frames
113
114
115
+class VideoLoader:
116
+
117
+ @classmethod
118
+ def load_bytes(self, data: bytes, num_frames: int = -1) -> npt.NDArray:
119
+ raise NotImplementedError
120
+
121
+
122
+class OpenCVVideoBackend(VideoLoader):
123
+
124
+ def get_cv2_video_api(self):
125
+ import cv2.videoio_registry as vr
126
+
127
+ api_pref = None
128
+ for backend in vr.getStreamBufferedBackends():
129
+ if not vr.hasBackend(backend):
130
+ continue
131
+ if not vr.isBackendBuiltIn(backend):
132
+ _, abi, api = vr.getStreamBufferedBackendPluginVersion(backend)
133
+ if (abi < 1 or (abi == 1 and api < 2)):
134
+ continue
135
+ api_pref = backend
136
+ break
137
+ return api_pref
138
+
139
+ @classmethod
140
+ def load_bytes(cls, data: bytes, num_frames: int = -1) -> npt.NDArray:
141
+ import cv2
142
+
143
+ backend = cls().get_cv2_video_api()
144
+ cap = cv2.VideoCapture(BytesIO(data), backend, [])
145
+ if not cap.isOpened():
146
+ raise ValueError("Could not open video stream")
147
+
148
+ total_frames_num = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
149
+ full_read = num_frames == -1 or total_frames_num < num_frames
150
+ if full_read:
151
+ frame_idx = list(range(0, total_frames_num))
152
+ else:
153
+ uniform_sampled_frames = np.linspace(0,
154
+ total_frames_num - 1,
155
+ num_frames,
156
+ dtype=int)
157
+ frame_idx = uniform_sampled_frames.tolist()
158
+
159
+ width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
160
+ height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
161
+ frames = np.empty((len(frame_idx), height, width, 3), dtype=np.uint8)
162
+
163
+ i = 0
164
+ for idx in range(total_frames_num):
165
+ ok = cap.grab() # next img
166
+ if not ok:
167
+ break
168
+ if idx in frame_idx: # only decompress needed
169
+ ret, frame = cap.retrieve()
170
+ if ret:
171
+ frames[i] = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
172
+ i += 1
173
+ # we expect all frames loaded
174
+ assert i == num_frames
175
+ return frames
176
+
177
+
178
class VideoMediaIO(MediaIO[npt.NDArray]):
179
180
def __init__(
@@ -129,22 +187,10 @@ class VideoMediaIO(MediaIO[npt.NDArray]):
187
188
self.image_io = image_io
189
self.num_frames = num_frames
190
+ self.video_loader = OpenCVVideoBackend
191
192
def load_bytes(self, data: bytes) -> npt.NDArray:
134
- vr = decord.VideoReader(BytesIO(data), num_threads=1)
135
- total_frame_num = len(vr)
136
-
137
- num_frames = self.num_frames
138
- if total_frame_num > num_frames:
139
- uniform_sampled_frames = np.linspace(0,
140
- total_frame_num - 1,
141
- num_frames,
142
- dtype=int)
143
- frame_idx = uniform_sampled_frames.tolist()
144
- else:
145
- frame_idx = list(range(0, total_frame_num))
146
-
147
- return vr.get_batch(frame_idx).asnumpy()
193
+ return self.video_loader.load_bytes(data, self.num_frames)
194
195
def load_base64(self, media_type: str, data: str) -> npt.NDArray:
196
if media_type.lower() == "video/jpeg":