A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Fix GLM4.1V multimodal processor with compatability for Transformers v4.56 (#24822)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Sep 15, 2025 at 20:45 UTC
0e219cd50b189800573e469bc8184724f52a91a1
6 files changed
+118
-70
examples/offline_inference/vision_language.py
+9
-1
@@ -1716,6 +1716,13 @@ model_example_map = {
1716
}
1717
1718
1719
+MODELS_NEED_VIDEO_METADATA = [
1720
+ "glm4_1v",
1721
+ "glm4_5v",
1722
+ "glm4_5v_fp8",
1723
+]
1724
+
1725
+
1726
def get_multi_modal_input(args):
1727
"""
1728
return {
@@ -1740,12 +1747,13 @@ def get_multi_modal_input(args):
1747
1748
if args.modality == "video":
1749
# Input video and question
1750
+ needs_metadata = args.model_type in MODELS_NEED_VIDEO_METADATA
1751
video = VideoAsset(name="baby_reading", num_frames=args.num_frames).np_ndarrays
1752
metadata = VideoAsset(name="baby_reading", num_frames=args.num_frames).metadata
1753
vid_questions = ["Why is this video funny?"]
1754
1755
return {
1748
- "data": [(video, metadata)] if args.model_type == "glm4_1v" else video,
1756
+ "data": ([(video, metadata)] if needs_metadata else video),
1757
"questions": vid_questions,
1758
}
1759
tests/models/multimodal/processing/test_common.py
+6
-3
@@ -32,11 +32,14 @@ def glm4_1v_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
32
# Ensure video metadata is included
33
if "video" in mm_data:
34
video = mm_data["video"]
35
+ num_frames = len(video)
36
mm_data["video"] = (video, {
36
- "total_num_frames": len(video),
37
- "fps": len(video),
37
+ "total_num_frames": num_frames,
38
+ "fps": num_frames,
39
"duration": 1,
39
- "video_backend": "opencv"
40
+ "frames_indices": [i for i in range(num_frames)],
41
+ "video_backend": "opencv",
42
+ "do_sample_frames": True,
43
})
44
return mm_data
45
tests/models/multimodal/processing/test_glm4_1v.py
+14
-3
@@ -12,8 +12,19 @@ from ...utils import build_model_context
12
13
@pytest.mark.parametrize("model_id", ["zai-org/GLM-4.1V-9B-Thinking"])
14
@pytest.mark.parametrize("expected_toks_per_frame", [299])
15
-@pytest.mark.parametrize("num_frames", [32, 128])
16
-@pytest.mark.parametrize("fps, expected_grid_t", [(1, 5), (2, 10)])
15
+@pytest.mark.parametrize(
16
+ "num_frames, fps, expected_grid_t",
17
+ [
18
+ # pre-sampled fixed frames (unexpected behavior,
19
+ # but we still expect it to work without errors)
20
+ (32, 1, 16),
21
+ (32, 2, 16),
22
+ (128, 1, 64),
23
+ (128, 2, 64),
24
+ # post-sampled frames (expected behavior)
25
+ (-1, 1, 5),
26
+ (-1, 2, 10),
27
+ ])
28
def test_processor_override(
29
model_id: str,
30
expected_toks_per_frame: int,
@@ -80,7 +91,7 @@ def test_video_loader_consistency(
91
92
static_video, static_metadata = OpenCVVideoBackend.load_bytes(video_bytes)
93
dynamic_video, dynamic_metadata = OpenCVDynamicVideoBackend.load_bytes(
83
- video_bytes, requested_fps=fps)
94
+ video_bytes, fps=fps)
95
96
# pre-sampled loader shouldn't read all frames
97
assert len(dynamic_video) < len(static_video)
vllm/assets/video.py
+11
-4
@@ -76,7 +76,7 @@ def video_to_pil_images_list(path: str,
76
return [Image.fromarray(frame) for frame in frames]
77
78
79
-def video_get_metadata(path: str) -> dict[str, Any]:
79
+def video_get_metadata(path: str, num_frames: int = -1) -> dict[str, Any]:
80
cap = cv2.VideoCapture(path)
81
if not cap.isOpened():
82
raise ValueError(f"Could not open video file {path}")
@@ -85,11 +85,18 @@ def video_get_metadata(path: str) -> dict[str, Any]:
85
fps = cap.get(cv2.CAP_PROP_FPS)
86
duration = total_frames / fps if fps > 0 else 0
87
88
+ if num_frames == -1 or num_frames > total_frames:
89
+ num_frames = total_frames
90
+
91
metadata = {
89
- "total_num_frames": total_frames,
92
+ "total_num_frames": num_frames,
93
"fps": fps,
94
"duration": duration,
92
- "video_backend": "opencv"
95
+ "video_backend": "opencv",
96
+ "frames_indices": list(range(num_frames)),
97
+ # extra field used to control hf processor's video
98
+ # sampling behavior
99
+ "do_sample_frames": num_frames == total_frames,
100
}
101
return metadata
102
@@ -126,7 +133,7 @@ class VideoAsset:
133
134
@property
135
def metadata(self) -> dict[str, Any]:
129
- ret = video_get_metadata(self.video_path)
136
+ ret = video_get_metadata(self.video_path, self.num_frames)
137
return ret
138
139
def get_audio(self, sampling_rate: Optional[float] = None) -> npt.NDArray:
vllm/model_executor/models/glm4_1v.py
+50
-38
@@ -36,7 +36,9 @@ import torch
36
import torch.nn as nn
37
import torch.nn.functional as F
38
from einops import rearrange
39
+from packaging.version import Version
40
from transformers import BatchFeature
41
+from transformers import __version__ as TRANSFORMERS_VERSION
42
from transformers.models.glm4v.configuration_glm4v import Glm4vVisionConfig
43
from transformers.models.glm4v.image_processing_glm4v import (
44
Glm4vImageProcessor, smart_resize)
@@ -1001,28 +1003,32 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
1003
max_frame_idx = meta_frames - 1
1004
duration = metadata.get("duration",
1005
round(max_frame_idx / video_fps) + 1)
1004
- if duration <= video_processor.max_duration:
1005
- n = int(math.floor(duration * video_processor.fps))
1006
- frame_indices = [
1007
- min(
1008
- max_frame_idx,
1009
- int(math.ceil(i * video_fps / video_processor.fps)),
1010
- ) for i in range(n)
1011
- ]
1006
+ do_sample_frames = metadata["do_sample_frames"]
1007
+ if not do_sample_frames:
1008
+ frame_indices = metadata["frames_indices"]
1009
else:
1013
- num_samples = int(video_processor.max_duration *
1014
- video_processor.fps)
1015
- if num_samples >= meta_frames:
1016
- frame_indices = list(range(meta_frames))
1017
- else:
1018
- target_seconds = np.linspace(0,
1019
- duration,
1020
- num_samples,
1021
- endpoint=True)
1010
+ if duration <= video_processor.max_duration:
1011
+ n = int(math.floor(duration * video_processor.fps))
1012
frame_indices = [
1023
- min(max_frame_idx, int(math.ceil(t * video_fps)))
1024
- for t in target_seconds
1013
+ min(
1014
+ max_frame_idx,
1015
+ int(math.ceil(i * video_fps / video_processor.fps)),
1016
+ ) for i in range(n)
1017
]
1018
+ else:
1019
+ num_samples = int(video_processor.max_duration *
1020
+ video_processor.fps)
1021
+ if num_samples >= meta_frames:
1022
+ frame_indices = list(range(meta_frames))
1023
+ else:
1024
+ target_seconds = np.linspace(0,
1025
+ duration,
1026
+ num_samples,
1027
+ endpoint=True)
1028
+ frame_indices = [
1029
+ min(max_frame_idx, int(math.ceil(t * video_fps)))
1030
+ for t in target_seconds
1031
+ ]
1032
1033
seen, uniq = set(), []
1034
for idx in frame_indices:
@@ -1139,7 +1145,9 @@ class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]):
1145
"fps": 2.0,
1146
"duration": num_frames / 2.0,
1147
"total_num_frames": num_frames,
1148
+ "frames_indices": [i for i in range(num_frames)],
1149
"video_backend": "opencv",
1150
+ "do_sample_frames": False,
1151
}
1152
video_item = (video.copy(), video_metadata)
1153
video_items.append(video_item)
@@ -1172,34 +1180,37 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
1180
for item in mm_data.pop("videos", []):
1181
video_array, metadata = item
1182
1175
- if metadata["video_backend"] == "opencv_dynamic":
1176
- mm_kwargs["do_sample_frames"] = False
1177
-
1178
- elif metadata["total_num_frames"] != len(video_array):
1179
- logger.warning(
1180
- "Total frames in metadata "
1181
- "(%s) does not match the length of "
1182
- "video array %s. This can "
1183
- "be because the video is resampled "
1184
- "in advance. This may cause "
1185
- "a divergence with HF implementation.",
1186
- metadata["total_num_frames"],
1187
- len(video_array),
1188
- )
1189
- metadata["total_num_frames"] = len(video_array)
1183
+ # don't update mm_kwargs inplace
1184
+ video_mm_kwargs = dict(**mm_kwargs)
1185
+ video_mm_kwargs["do_sample_frames"] = metadata.get(
1186
+ "do_sample_frames", True)
1187
1188
video_mm_data = dict()
1189
video_mm_data["videos"] = [[video_array]]
1193
- video_mm_data["video_metadata"] = [[VideoMetadata(**metadata)]]
1190
+
1191
+ # backward compatibility for Transformers 4.55
1192
+ unuse_metadata = ["do_sample_frames"]
1193
+ if not hasattr(
1194
+ VideoMetadata,
1195
+ "frames_indices") and "frames_indices" in metadata:
1196
+ unuse_metadata.append("frames_indices")
1197
+
1198
+ video_mm_data["video_metadata"] = [[
1199
+ VideoMetadata(
1200
+ **{
1201
+ k: metadata[k]
1202
+ for k in metadata if k not in unuse_metadata
1203
+ })
1204
+ ]]
1205
1206
video_outputs = super()._call_hf_processor(
1207
prompt="<|begin_of_video|><|video|><|end_of_video|>",
1208
mm_data=video_mm_data,
1198
- mm_kwargs=mm_kwargs,
1209
+ mm_kwargs=video_mm_kwargs,
1210
tok_kwargs=tok_kwargs,
1211
)
1201
- if "do_sample_frames" in mm_kwargs and not mm_kwargs[
1202
- "do_sample_frames"]:
1212
+ if not video_mm_kwargs["do_sample_frames"] and Version(
1213
+ TRANSFORMERS_VERSION) < Version("4.56.0"):
1214
# Transformers v4.55 has incorrect timestamps issue for
1215
# skip sampling. We construct the placeholder manually to
1216
# get placeholders with correct timestamps.
@@ -1218,6 +1229,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
1229
prompt = prompt.replace(
1230
"<|begin_of_video|><|video|><|end_of_video|>",
1231
video_placeholder,
1232
+ 1,
1233
)
1234
1235
video_grid_thw_lst.append(video_outputs["video_grid_thw"])
vllm/multimodal/video.py
+28
-21
@@ -121,14 +121,6 @@ class OpenCVVideoBackend(VideoLoader):
121
original_fps = cap.get(cv2.CAP_PROP_FPS)
122
duration = total_frames_num / original_fps if original_fps > 0 else 0
123
124
- # Use transformers transformers.video_utils.VideoMetadata format
125
- metadata = {
126
- "total_num_frames": total_frames_num,
127
- "fps": original_fps,
128
- "duration": duration,
129
- "video_backend": "opencv"
130
- }
131
-
124
# resample video to target num_frames
125
full_read = num_frames == -1 or total_frames_num < num_frames
126
if full_read:
@@ -159,6 +151,20 @@ class OpenCVVideoBackend(VideoLoader):
151
assert i == num_frames, (f"Expected reading {num_frames} frames, "
152
f"but only loaded {i} frames from video.")
153
154
+ # Use transformers transformers.video_utils.VideoMetadata format
155
+ # NOTE(Isotr0py): For models like Qwen3-VL/GLM4.5V, this metadata
156
+ # can cause incorrect timestamp calculation without num_frames=-1.
157
+ metadata = {
158
+ "total_num_frames": num_frames,
159
+ "fps": original_fps,
160
+ "duration": duration,
161
+ "video_backend": "opencv",
162
+ "frames_indices": list(range(num_frames)),
163
+ # extra field used to control hf processor's video
164
+ # sampling behavior
165
+ "do_sample_frames": num_frames == total_frames_num,
166
+ }
167
+
168
return frames, metadata
169
170
@@ -170,7 +176,7 @@ class OpenCVDynamicVideoBackend(OpenCVVideoBackend):
176
cls,
177
data: bytes,
178
num_frames: int = -1,
173
- requested_fps: int = 2,
179
+ fps: int = 2,
180
max_duration: int = 300,
181
**kwargs,
182
) -> tuple[npt.NDArray, dict[str, Any]]:
@@ -185,14 +191,6 @@ class OpenCVDynamicVideoBackend(OpenCVVideoBackend):
191
original_fps = cap.get(cv2.CAP_PROP_FPS)
192
duration = total_frames_num / original_fps if original_fps > 0 else 0
193
188
- # Use transformers transformers.video_utils.VideoMetadata format
189
- metadata = {
190
- "total_num_frames": total_frames_num,
191
- "fps": original_fps,
192
- "duration": duration,
193
- "video_backend": "opencv_dynamic"
194
- }
195
-
194
# resample video to target num_frames
195
max_frame_idx = total_frames_num - 1
196
duration = duration or round(max_frame_idx / original_fps) + 1
@@ -201,14 +199,13 @@ class OpenCVDynamicVideoBackend(OpenCVVideoBackend):
199
# https://github.com/huggingface/transformers/blob/v4.55.4/src/transformers/models/glm4v/video_processing_glm4v.py#L103-L140
200
frame_indices: Union[range, list[int]]
201
if duration <= max_duration:
204
- n = int(math.floor(duration * requested_fps))
202
+ n = int(math.floor(duration * fps))
203
frame_indices = sorted({
206
- min(max_frame_idx,
207
- int(math.ceil(i * original_fps / requested_fps)))
204
+ min(max_frame_idx, int(math.ceil(i * original_fps / fps)))
205
for i in range(n)
206
})
207
else:
211
- num_samples = int(max_duration * requested_fps)
208
+ num_samples = int(max_duration * fps)
209
if num_samples >= total_frames_num:
210
frame_indices = range(total_frames_num)
211
else:
@@ -241,6 +238,16 @@ class OpenCVDynamicVideoBackend(OpenCVVideoBackend):
238
f"Expected reading {len(frame_indices)} frames, "
239
f"but only loaded {i} frames from video.")
240
241
+ # Use transformers transformers.video_utils.VideoMetadata format
242
+ metadata = {
243
+ "total_num_frames": total_frames_num,
244
+ "fps": original_fps,
245
+ "duration": duration,
246
+ "video_backend": "opencv_dynamic",
247
+ "frames_indices": list(frame_indices),
248
+ "do_sample_frames": False,
249
+ }
250
+
251
return frames, metadata
252
253