[Bugfix][GLM4V] Fix video dummy profiling and memory usage (#48729)
Signed-off-by: xiaoming <1259730330@qq.com>
labAxiaoming committed
Jul 18, 2026 at 01:44 UTC
bf578e1abdffc2d25232783ff59a3132279e6bdd
2 files changed
+72
-49
tests/models/multimodal/processing/test_glm4_1v.py
+52
@@ -1,9 +1,15 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+from unittest.mock import Mock
5
+
6
import pytest
7
8
from vllm.assets.video import VideoAsset
9
+from vllm.model_executor.models.glm4_1v import (
10
+ Glm4vForConditionalGeneration,
11
+ Glm4vProcessingInfo,
12
+)
13
from vllm.multimodal import MULTIMODAL_REGISTRY
14
from vllm.multimodal.inputs import batched_tensors_equal
15
from vllm.multimodal.video import DynamicVideoBackend, VideoBackend
@@ -11,6 +17,52 @@ from vllm.multimodal.video import DynamicVideoBackend, VideoBackend
17
from ...utils import build_model_context
18
19
20
+@pytest.mark.parametrize(
21
+ (
22
+ "max_video_pixels",
23
+ "max_tokens",
24
+ "expected_num_frames",
25
+ ),
26
+ [
27
+ (47_040_000, 124_988, 11),
28
+ (47_040_000, 30_000, 24),
29
+ (100_352_000, 124_988, 21),
30
+ (100_352_000, 30_000, 7),
31
+ (100_352_000, 0, 1),
32
+ ],
33
+)
34
+def test_get_max_video_frames_matches_glm_resize(
35
+ max_video_pixels: int,
36
+ max_tokens: int,
37
+ expected_num_frames: int,
38
+):
39
+ info = Mock(spec=Glm4vProcessingInfo)
40
+ info.get_image_size_with_most_features.return_value = (2184, 2184)
41
+ info._get_video_max_pixels.return_value = max_video_pixels
42
+ vision_config = info.get_hf_config.return_value.vision_config
43
+ vision_config.patch_size = 14
44
+ vision_config.spatial_merge_size = 2
45
+ vision_config.temporal_patch_size = 2
46
+ info._get_vision_info.side_effect = lambda **kwargs: (
47
+ Glm4vProcessingInfo._get_vision_info(info, **kwargs)
48
+ )
49
+
50
+ num_frames = Glm4vProcessingInfo._get_max_video_frames(
51
+ info,
52
+ max_tokens=max_tokens,
53
+ )
54
+
55
+ assert num_frames == expected_num_frames
56
+ assert info._get_video_max_pixels.call_count == 1
57
+ assert info._get_vision_info.call_count == 600
58
+
59
+
60
+def test_encoder_cudagraph_uses_model_video_frame_limit():
61
+ model = Mock()
62
+
63
+ assert Glm4vForConditionalGeneration.get_max_frames_per_video(model) == 600
64
+
65
+
66
@pytest.mark.parametrize("model_id", ["zai-org/GLM-4.1V-9B-Thinking"])
67
@pytest.mark.parametrize("expected_toks_per_frame", [299])
68
@pytest.mark.parametrize(
vllm/model_executor/models/glm4_1v.py
+20
-49
@@ -40,6 +40,7 @@ import transformers
40
from einops import rearrange
41
from packaging.version import Version
42
from transformers import BatchFeature, Glm4vProcessor
43
+from transformers.image_processing_base import ImageProcessingMixin
44
from transformers.models.glm4v.configuration_glm4v import (
45
Glm4vTextConfig,
46
Glm4vVisionConfig,
@@ -49,6 +50,7 @@ from transformers.models.glm4v.image_processing_glm4v import (
50
smart_resize,
51
)
52
from transformers.models.glm4v.video_processing_glm4v import Glm4vVideoProcessor
53
+from transformers.video_processing_utils import BaseVideoProcessor
54
from transformers.video_utils import VideoMetadata
55
56
from vllm.config import VllmConfig
@@ -94,6 +96,8 @@ from vllm.multimodal.processing import (
96
PromptUpdateDetails,
97
)
98
from vllm.sequence import IntermediateTensors
99
+from vllm.transformers_utils.processor import get_processor_cls_name_from_config
100
+from vllm.transformers_utils.utils import convert_model_repo_to_path
101
from vllm.utils.tensor_schema import TensorSchema, TensorShape
102
from vllm.v1.attention.backends.registry import AttentionBackendEnum
103
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
@@ -982,11 +986,6 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
986
return self.get_hf_processor(**kwargs).video_processor
987
988
def _get_processor_class_name(self) -> str | None:
985
- from vllm.transformers_utils.processor import (
986
- get_processor_cls_name_from_config,
987
- )
988
- from vllm.transformers_utils.utils import convert_model_repo_to_path
989
-
989
return get_processor_cls_name_from_config(
990
convert_model_repo_to_path(self.ctx.model_config.model),
991
revision=self.ctx.model_config.revision,
@@ -1109,8 +1108,6 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
1108
1109
image_processor_config = self.ctx.get_hf_image_processor_config()
1110
if not image_processor_config.get("size"):
1112
- from transformers.image_processing_base import ImageProcessingMixin
1113
-
1111
image_processor_config, _ = ImageProcessingMixin.get_image_processor_dict(
1112
self.ctx.model_config.model,
1113
revision=self.ctx.model_config.revision,
@@ -1122,8 +1119,6 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
1119
return self._get_longest_edge(size, "GLM4V image processor size")
1120
1121
def _get_video_max_pixels(self) -> int:
1125
- from transformers.video_processing_utils import BaseVideoProcessor
1126
-
1122
mm_kwargs = self.ctx.get_merged_mm_kwargs({})
1123
if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
1124
return int(override_max_pixels)
@@ -1175,39 +1170,29 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
1170
image_height=target_height,
1171
)
1172
1178
- def get_num_video_tokens(
1179
- self,
1180
- *,
1181
- image_width: int,
1182
- image_height: int,
1183
- num_frames: int,
1184
- ) -> int:
1185
- _, num_video_tokens = self._get_vision_info(
1186
- image_width=image_width,
1187
- image_height=image_height,
1188
- num_frames=num_frames,
1189
- max_image_pixels=28 * 28 * 2 * 30000,
1190
- )
1191
- return num_video_tokens
1192
-
1173
def _get_max_video_frames(self, max_tokens: int) -> int:
1174
target_width, target_height = self.get_image_size_with_most_features()
1175
1196
- num_frames = 0
1176
+ max_video_pixels = self._get_video_max_pixels()
1177
+ num_frames_with_most_features = 1
1178
+ max_vision_tokens = 0
1179
1198
- while True:
1199
- next_num_frames = num_frames + 1
1200
- next_max_tokens = self.get_num_video_tokens(
1180
+ for num_frames in range(1, _MAX_FRAMES_PER_VIDEO + 1):
1181
+ _, num_vision_tokens = self._get_vision_info(
1182
image_width=target_width,
1183
image_height=target_height,
1203
- num_frames=next_num_frames,
1184
+ num_frames=num_frames,
1185
+ max_image_pixels=max_video_pixels,
1186
)
1205
- if next_max_tokens > max_tokens or next_max_tokens == 0:
1206
- break
1187
1208
- num_frames = next_num_frames
1188
+ if (
1189
+ 0 < num_vision_tokens <= max_tokens
1190
+ and num_vision_tokens > max_vision_tokens
1191
+ ):
1192
+ num_frames_with_most_features = num_frames
1193
+ max_vision_tokens = num_vision_tokens
1194
1210
- return num_frames
1195
+ return num_frames_with_most_features
1196
1197
def get_num_frames_with_most_features(
1198
self,
@@ -1215,15 +1200,9 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
1200
mm_counts: Mapping[str, int],
1201
) -> int:
1202
max_images = mm_counts.get("image", 0)
1218
- max_videos = mm_counts.get("video", 0)
1203
1204
max_image_tokens = self.get_max_image_tokens() * max_images
1221
- max_total_frames = self._get_max_video_frames(seq_len - max_image_tokens)
1222
- max_frames_per_video = min(
1223
- max_total_frames // max(max_videos, 1), _MAX_FRAMES_PER_VIDEO
1224
- )
1225
-
1226
- return max(max_frames_per_video, 1)
1205
+ return self._get_max_video_frames(seq_len - max_image_tokens)
1206
1207
def _get_video_second_idx_glm4v(
1208
self, metadata: dict[str, Any], total_frames: int
@@ -2001,15 +1980,7 @@ class Glm4vForConditionalGeneration(
1980
raise AssertionError("This line should be unreachable.")
1981
1982
def get_max_frames_per_video(self) -> int:
2004
- mm_registry = MULTIMODAL_REGISTRY
2005
- info = mm_registry.get_processing_info(self.model_config)
2006
- max_frames_per_video = info.get_num_frames_with_most_features(
2007
- seq_len=self.model_config.max_model_len,
2008
- mm_counts={"video": self.multimodal_config.get_limit_per_prompt("video")},
2009
- )
2010
- # Small 'max_frames_per_video' will cause 'tensor mismatch' in PR#43403
2011
- # 16 is the default 'num_frames' of '_get_vision_info'
2012
- return max(max_frames_per_video, 16)
1983
+ return _MAX_FRAMES_PER_VIDEO
1984
1985
def get_encoder_cudagraph_budget_range(
1986
self,