@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bcbe2a4d9e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[VLM] Optimize GLM4.5-V-style video processing to only decode necessary frames (#24161)

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>

Isotr0py committed Sep 12, 2025 at 00:44 UTC bcbe2a4d9e792dea8b76ac1a006aa8596af02c41
5 files changed +233 -55
tests/models/multimodal/processing/test_glm4_1v.py
+47
@@ -5,6 +5,7 @@ import pytest
5
6 from vllm.assets.video import VideoAsset
7 from vllm.multimodal import MULTIMODAL_REGISTRY
8 +from vllm.multimodal.video import OpenCVDynamicVideoBackend, OpenCVVideoBackend
9
10 from ...utils import build_model_context
11
@@ -50,3 +51,49 @@ def test_processor_override(
51
52 assert grid_t == expected_grid_t
53 assert video_tok_count == expected_toks_per_frame * grid_t
54 +
55 +
56 +@pytest.mark.parametrize("model_id", ["zai-org/GLM-4.1V-9B-Thinking"])
57 +@pytest.mark.parametrize("fps", [2])
58 +def test_video_loader_consistency(
59 + model_id: str,
60 + fps: int,
61 +):
62 + """
63 + Ensure dynamic video loader (pre-sampled by loader) and normal video
64 + loader (post-sampled by processor) produce same video processing outputs.
65 + """
66 + ctx = build_model_context(
67 + model_id,
68 + mm_processor_kwargs=None,
69 + limit_mm_per_prompt={"video": 1},
70 + )
71 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
72 + hf_processor_mm_kwargs = {"fps": fps}
73 +
74 + # Build the image str / prompt based on the number of images we pass
75 + prompt = "<|begin_of_video|><|video|><|end_of_video|>"
76 +
77 + video_path = VideoAsset(name="baby_reading", num_frames=-1).video_path
78 + with open(video_path, "rb") as f:
79 + video_bytes = f.read()
80 +
81 + static_video, static_metadata = OpenCVVideoBackend.load_bytes(video_bytes)
82 + dynamic_video, dynamic_metadata = OpenCVDynamicVideoBackend.load_bytes(
83 + video_bytes, requested_fps=fps)
84 +
85 + # pre-sampled loader shouldn't read all frames
86 + assert len(dynamic_video) < len(static_video)
87 +
88 + static_mm_data = {"video": [(static_video, static_metadata)]}
89 + dynamic_mm_data = {"video": [(dynamic_video, dynamic_metadata)]}
90 +
91 + static_outputs = processor.apply(prompt, static_mm_data,
92 + hf_processor_mm_kwargs)
93 + dynamic_outputs = processor.apply(prompt, dynamic_mm_data,
94 + hf_processor_mm_kwargs)
95 +
96 + assert static_outputs["prompt_token_ids"] == dynamic_outputs[
97 + "prompt_token_ids"]
98 + assert static_outputs["mm_kwargs"].get_data(
99 + ) == dynamic_outputs["mm_kwargs"].get_data()
tests/multimodal/test_utils.py
+26
@@ -204,6 +204,32 @@ async def test_fetch_video_http(video_url: str, num_frames: int):
204 assert metadata_sync == metadata_async
205
206
207 +@pytest.mark.asyncio
208 +@pytest.mark.parametrize("video_url", TEST_VIDEO_URLS)
209 +@pytest.mark.parametrize("max_duration", [1, 60, 1800])
210 +@pytest.mark.parametrize("requested_fps", [2, 24])
211 +async def test_fetch_video_http_with_dynamic_loader(
212 + video_url: str, max_duration: int, requested_fps: int,
213 + monkeypatch: pytest.MonkeyPatch):
214 + with monkeypatch.context() as m:
215 + m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv_dynamic")
216 + connector = MediaConnector(
217 + media_io_kwargs={
218 + "video": {
219 + "max_duration": max_duration,
220 + "requested_fps": requested_fps,
221 + }
222 + })
223 +
224 + video_sync, metadata_sync = connector.fetch_video(video_url)
225 + video_async, metadata_async = await connector.fetch_video_async(
226 + video_url)
227 +
228 + assert np.array_equal(video_sync, video_async)
229 + assert metadata_sync == metadata_async
230 + assert metadata_sync["video_backend"] == "opencv_dynamic"
231 +
232 +
233 # Used for `test_argsort_mm_positions`.
234 class TestCase(NamedTuple):
235 mm_positions: "MultiModalPlaceholderDict"
vllm/assets/video.py
+8 -8
@@ -110,22 +110,23 @@ class VideoAsset:
110 def filename(self) -> str:
111 return self._NAME_TO_FILE[self.name]
112
113 + @property
114 + def video_path(self) -> str:
115 + return download_video_asset(self.filename)
116 +
117 @property
118 def pil_images(self) -> list[Image.Image]:
115 - video_path = download_video_asset(self.filename)
116 - ret = video_to_pil_images_list(video_path, self.num_frames)
119 + ret = video_to_pil_images_list(self.video_path, self.num_frames)
120 return ret
121
122 @property
123 def np_ndarrays(self) -> npt.NDArray:
121 - video_path = download_video_asset(self.filename)
122 - ret = video_to_ndarrays(video_path, self.num_frames)
124 + ret = video_to_ndarrays(self.video_path, self.num_frames)
125 return ret
126
127 @property
128 def metadata(self) -> dict[str, Any]:
127 - video_path = download_video_asset(self.filename)
128 - ret = video_get_metadata(video_path)
129 + ret = video_get_metadata(self.video_path)
130 return ret
131
132 def get_audio(self, sampling_rate: Optional[float] = None) -> npt.NDArray:
@@ -134,5 +135,4 @@ class VideoAsset:
135
136 See also: examples/offline_inference/qwen2_5_omni/only_thinker.py
137 """
137 - video_path = download_video_asset(self.filename)
138 - return librosa.load(video_path, sr=sampling_rate)[0]
138 + return librosa.load(self.video_path, sr=sampling_rate)[0]
vllm/model_executor/models/glm4_1v.py
+60 -40
@@ -1023,6 +1023,43 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
1023 selected_timestamps.append(timestamps_list[idx])
1024 return selected_timestamps
1025
1026 + def _construct_video_placeholder(
1027 + self,
1028 + video_array: np.ndarray,
1029 + metadata: dict[str, Any],
1030 + grid_thw: torch.Tensor,
1031 + ) -> str:
1032 + hf_processor = self.get_hf_processor()
1033 + tokenizer = self.get_tokenizer()
1034 + image_processor = hf_processor.image_processor
1035 +
1036 + hf_config = self.get_hf_config()
1037 + boi_token_id = hf_config.image_start_token_id
1038 + eoi_token_id = hf_config.image_end_token_id
1039 + bov_token_id = hf_config.video_start_token_id
1040 + eov_token_id = hf_config.video_end_token_id
1041 + merge_length = image_processor.merge_size**2
1042 +
1043 + assert isinstance(grid_thw, torch.Tensor)
1044 + timestamps = self._get_video_second_idx(metadata, len(video_array))
1045 + frames_idx_token = [
1046 + tokenizer.encode(str(i), add_special_tokens=False)
1047 + for i in timestamps
1048 + ]
1049 + T, H, W = grid_thw
1050 + num_tokens_per_frame = int(H * W) // merge_length
1051 + placeholder = []
1052 + placeholder.append(bov_token_id)
1053 + for frame_idx in frames_idx_token:
1054 + placeholder.append(boi_token_id)
1055 + placeholder.extend([hf_processor.video_token_id] *
1056 + num_tokens_per_frame)
1057 + placeholder.append(eoi_token_id)
1058 + placeholder.extend(frame_idx)
1059 + placeholder.append(eov_token_id)
1060 +
1061 + return placeholder
1062 +
1063
1064 class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]):
1065
@@ -1118,17 +1155,10 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
1155 for item in mm_data.pop("videos", []):
1156 video_array, metadata = item
1157
1121 - # FIXME(Isotr0py): Activate the below logic after we can disable
1122 - # resampling from video loader backend.
1123 - # assert metadata["total_num_frames"] == len(video_array), (
1124 - # f"Total frames {metadata['total_num_frames']} does not "
1125 - # f"match the length of video array {len(video_array)}.")
1158 + if metadata["video_backend"] == "opencv_dynamic":
1159 + mm_kwargs["do_sample_frames"] = False
1160
1127 - # NOTE: Temporary workaround for resampled videos.
1128 - # this can cause a divergence with HF implementation if
1129 - # the input video is resampled in advance.
1130 -
1131 - if metadata["total_num_frames"] != len(video_array):
1161 + elif metadata["total_num_frames"] != len(video_array):
1162 logger.warning(
1163 "Total frames in metadata "
1164 "(%s) does not match the length of "
@@ -1140,11 +1170,10 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
1170 len(video_array),
1171 )
1172 metadata["total_num_frames"] = len(video_array)
1143 - metadata = VideoMetadata(**metadata)
1173
1174 video_mm_data = dict()
1175 video_mm_data["videos"] = [[video_array]]
1147 - video_mm_data["video_metadata"] = [[metadata]]
1176 + video_mm_data["video_metadata"] = [[VideoMetadata(**metadata)]]
1177
1178 video_outputs = super()._call_hf_processor(
1179 prompt="<|begin_of_video|><|video|><|end_of_video|>",
@@ -1152,11 +1181,23 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
1181 mm_kwargs=mm_kwargs,
1182 tok_kwargs=tok_kwargs,
1183 )
1155 - input_ids = video_outputs.pop("input_ids")
1156 - input_ids[input_ids == processor.image_token_id] = (
1157 - processor.video_token_id)
1158 - video_placeholder = processor.tokenizer.batch_decode(
1159 - input_ids)[0]
1184 + if "do_sample_frames" in mm_kwargs and not mm_kwargs[
1185 + "do_sample_frames"]:
1186 + # Transformers v4.55 has incorrect timestamps issue for
1187 + # skip sampling. We construct the placeholder manually to
1188 + # get placeholders with correct timestamps.
1189 + placeholder = self.info._construct_video_placeholder(
1190 + video_array,
1191 + metadata,
1192 + video_outputs["video_grid_thw"].squeeze(0),
1193 + )
1194 + video_placeholder = processor.tokenizer.decode(placeholder)
1195 + else:
1196 + input_ids = video_outputs.pop("input_ids")
1197 + input_ids[input_ids == processor.image_token_id] = (
1198 + processor.video_token_id)
1199 + video_placeholder = processor.tokenizer.batch_decode(
1200 + input_ids)[0]
1201 prompt = prompt.replace(
1202 "<|begin_of_video|><|video|><|end_of_video|>",
1203 video_placeholder,
@@ -1202,14 +1243,6 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
1243 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
1244 image_processor = self.info.get_image_processor(
1245 **hf_processor_mm_kwargs)
1205 - tokenizer = self.info.get_tokenizer()
1206 - hf_config = self.info.get_hf_config()
1207 -
1208 - boi_token_id = hf_config.image_start_token_id
1209 - eoi_token_id = hf_config.image_end_token_id
1210 -
1211 - bov_token_id = hf_config.video_start_token_id
1212 - eov_token_id = hf_config.video_end_token_id
1246
1247 merge_length = image_processor.merge_size**2
1248
@@ -1227,21 +1260,8 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
1260 assert isinstance(grid_thw, torch.Tensor)
1261
1262 video, metadata = mm_items["video"][item_idx]
1230 - timestamps = self.info._get_video_second_idx(metadata, len(video))
1231 - frames_idx_token = [
1232 - tokenizer.encode(str(i), add_special_tokens=False)
1233 - for i in timestamps
1234 - ]
1235 - num_tokens_per_frame = int(grid_thw[1:].prod()) // merge_length
1236 - placeholder = []
1237 - placeholder.append(bov_token_id)
1238 - for frame_idx in frames_idx_token:
1239 - placeholder.append(boi_token_id)
1240 - placeholder.extend([hf_processor.video_token_id] *
1241 - num_tokens_per_frame)
1242 - placeholder.append(eoi_token_id)
1243 - placeholder.extend(frame_idx)
1244 - placeholder.append(eov_token_id)
1263 + placeholder = self.info._construct_video_placeholder(
1264 + video, metadata, grid_thw)
1265 return PromptUpdateDetails.select_token_id(
1266 placeholder,
1267 embed_token_id=hf_processor.video_token_id,
vllm/multimodal/video.py
+92 -7
@@ -1,12 +1,12 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
3 import base64
4 +import math
5 from abc import abstractmethod
6 from functools import partial
7 from io import BytesIO
8 from pathlib import Path
9 -from typing import Any
9 +from typing import Any, Union
10
11 import numpy as np
12 import numpy.typing as npt
@@ -104,10 +104,12 @@ class OpenCVVideoBackend(VideoLoader):
104 return api_pref
105
106 @classmethod
107 - def load_bytes(cls,
108 - data: bytes,
109 - num_frames: int = -1,
110 - **kwargs) -> tuple[npt.NDArray, dict[str, Any]]:
107 + def load_bytes(
108 + cls,
109 + data: bytes,
110 + num_frames: int = -1,
111 + **kwargs,
112 + ) -> tuple[npt.NDArray, dict[str, Any]]:
113 import cv2
114
115 backend = cls().get_cv2_video_api()
@@ -119,6 +121,15 @@ class OpenCVVideoBackend(VideoLoader):
121 original_fps = cap.get(cv2.CAP_PROP_FPS)
122 duration = total_frames_num / original_fps if original_fps > 0 else 0
123
124 + # Use transformers transformers.video_utils.VideoMetadata format
125 + metadata = {
126 + "total_num_frames": total_frames_num,
127 + "fps": original_fps,
128 + "duration": duration,
129 + "video_backend": "opencv"
130 + }
131 +
132 + # resample video to target num_frames
133 full_read = num_frames == -1 or total_frames_num < num_frames
134 if full_read:
135 num_frames = total_frames_num
@@ -148,14 +159,88 @@ class OpenCVVideoBackend(VideoLoader):
159 assert i == num_frames, (f"Expected reading {num_frames} frames, "
160 f"but only loaded {i} frames from video.")
161
162 + return frames, metadata
163 +
164 +
165 +@VIDEO_LOADER_REGISTRY.register("opencv_dynamic")
166 +class OpenCVDynamicVideoBackend(OpenCVVideoBackend):
167 +
168 + @classmethod
169 + def load_bytes(
170 + cls,
171 + data: bytes,
172 + num_frames: int = -1,
173 + requested_fps: int = 2,
174 + max_duration: int = 300,
175 + **kwargs,
176 + ) -> tuple[npt.NDArray, dict[str, Any]]:
177 + import cv2
178 +
179 + backend = cls().get_cv2_video_api()
180 + cap = cv2.VideoCapture(BytesIO(data), backend, [])
181 + if not cap.isOpened():
182 + raise ValueError("Could not open video stream")
183 +
184 + total_frames_num = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
185 + original_fps = cap.get(cv2.CAP_PROP_FPS)
186 + duration = total_frames_num / original_fps if original_fps > 0 else 0
187 +
188 # Use transformers transformers.video_utils.VideoMetadata format
189 metadata = {
190 "total_num_frames": total_frames_num,
191 "fps": original_fps,
192 "duration": duration,
156 - "video_backend": "opencv"
193 + "video_backend": "opencv_dynamic"
194 }
195
196 + # resample video to target num_frames
197 + max_frame_idx = total_frames_num - 1
198 + duration = duration or round(max_frame_idx / original_fps) + 1
199 +
200 + # Refer to:
201 + # https://github.com/huggingface/transformers/blob/v4.55.4/src/transformers/models/glm4v/video_processing_glm4v.py#L103-L140
202 + frame_indices: Union[range, list[int]]
203 + if duration <= max_duration:
204 + n = int(math.floor(duration * requested_fps))
205 + frame_indices = sorted({
206 + min(max_frame_idx,
207 + int(math.ceil(i * original_fps / requested_fps)))
208 + for i in range(n)
209 + })
210 + else:
211 + num_samples = int(max_duration * requested_fps)
212 + if num_samples >= total_frames_num:
213 + frame_indices = range(total_frames_num)
214 + else:
215 + target_seconds = np.linspace(0,
216 + duration,
217 + num_samples,
218 + endpoint=True)
219 + frame_indices = sorted({
220 + min(max_frame_idx, int(math.ceil(t * original_fps)))
221 + for t in target_seconds
222 + })
223 +
224 + width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
225 + height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
226 + frames = np.empty((len(frame_indices), height, width, 3),
227 + dtype=np.uint8)
228 +
229 + i = 0
230 + for idx in range(total_frames_num):
231 + ok = cap.grab()
232 + if not ok:
233 + break
234 + if idx in frame_indices:
235 + ret, frame = cap.retrieve()
236 + if ret:
237 + frames[i] = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
238 + i += 1
239 +
240 + assert i == len(frame_indices), (
241 + f"Expected reading {len(frame_indices)} frames, "
242 + f"but only loaded {i} frames from video.")
243 +
244 return frames, metadata
245
246