A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Frontend] Enable decord to load video from base64 (#11492)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Dec 26, 2024 at 00:33 UTC
b689ada91e381ac8a1b197b5a52d7f1fa32fac05
1 file changed
+19
-20
vllm/multimodal/utils.py
+19
-20
@@ -125,17 +125,7 @@ async def async_fetch_image(image_url: str,
125
return image.convert(image_mode)
126
127
128
-def _load_video_frames_from_bytes(b: bytes):
129
- frame = Image.open(BytesIO(b))
130
- return np.array(frame)
131
-
132
-
133
-def load_video_frames_from_base64(frame: Union[bytes, str]):
134
- """Load frame from base64 format."""
135
- return _load_video_frames_from_bytes(base64.b64decode(frame))
136
-
137
-
138
-def _load_video_from_bytes(b: bytes, num_frames: int = 32):
128
+def _load_video_from_bytes(b: bytes, num_frames: int = 32) -> npt.NDArray:
129
_, decord = try_import_video_packages()
130
131
video_path = BytesIO(b)
@@ -155,13 +145,17 @@ def _load_video_from_bytes(b: bytes, num_frames: int = 32):
145
return frames
146
147
158
-def _load_video_from_data_url(video_url: str):
159
- # Only split once and assume the second part is the base64 encoded image
160
- frames_base64 = video_url.split(",")[1:]
161
- return np.stack([
162
- load_video_frames_from_base64(frame_base64)
163
- for frame_base64 in frames_base64
164
- ])
148
+def _load_video_from_data_url(video_url: str) -> npt.NDArray:
149
+ # Only split once and assume the second part is the base64 encoded video
150
+ _, video_base64 = video_url.split(",", 1)
151
+
152
+ if video_url.startswith("data:video/jpeg;"):
153
+ return np.stack([
154
+ np.array(load_image_from_base64(frame_base64))
155
+ for frame_base64 in video_base64.split(",")
156
+ ])
157
+
158
+ return load_video_from_base64(video_base64)
159
160
161
def fetch_video(video_url: str, *, num_frames: int = 32) -> npt.NDArray:
@@ -342,7 +336,7 @@ def rescale_image_size(image: Image.Image,
336
return image
337
338
345
-def try_import_video_packages() -> Any:
339
+def try_import_video_packages():
340
try:
341
import cv2
342
import decord
@@ -384,7 +378,7 @@ def sample_frames_from_video(frames: npt.NDArray,
378
return sampled_frames
379
380
387
-def encode_video_base64(frames: npt.NDArray):
381
+def encode_video_base64(frames: npt.NDArray) -> str:
382
base64_frames = []
383
frames_list = [frames[i] for i in range(frames.shape[0])]
384
for frame in frames_list:
@@ -393,6 +387,11 @@ def encode_video_base64(frames: npt.NDArray):
387
return ",".join(base64_frames)
388
389
390
+def load_video_from_base64(video: Union[bytes, str]) -> npt.NDArray:
391
+ """Load video from base64 format."""
392
+ return _load_video_from_bytes(base64.b64decode(video))
393
+
394
+
395
def resolve_visual_encoder_outputs(
396
encoder_outputs: Union[torch.Tensor, list[torch.Tensor]],
397
feature_sample_layers: Optional[list[int]],