[Bugfix][Multimodal] Fix video temporal padding estimates (#49030)
Signed-off-by: xiaoming <1259730330@qq.com>
labAxiaoming committed
Jul 29, 2026 at 01:57 UTC
1db989bbf14b9896c1306fd722ced9a9b1238465
7 files changed
+34
-9
tests/models/multimodal/processing/test_glm4_1v.py
+25
@@ -63,6 +63,31 @@ def test_encoder_cudagraph_uses_model_video_frame_limit():
63
assert Glm4vForConditionalGeneration.get_max_frames_per_video(model) == 600
64
65
66
+@pytest.mark.parametrize(
67
+ ("temporal_patch_size", "expected_grid_t"),
68
+ [(2, 9), (4, 5), (8, 3)],
69
+)
70
+def test_vision_info_rounds_up_temporal_frames(
71
+ temporal_patch_size: int,
72
+ expected_grid_t: int,
73
+):
74
+ info = Mock(spec=Glm4vProcessingInfo)
75
+ vision_config = info.get_hf_config.return_value.vision_config
76
+ vision_config.patch_size = 14
77
+ vision_config.spatial_merge_size = 2
78
+ vision_config.temporal_patch_size = temporal_patch_size
79
+
80
+ _, num_vision_tokens = Glm4vProcessingInfo._get_vision_info(
81
+ info,
82
+ image_width=28,
83
+ image_height=28,
84
+ num_frames=17,
85
+ do_resize=False,
86
+ )
87
+
88
+ assert num_vision_tokens == expected_grid_t
89
+
90
+
91
@pytest.mark.parametrize("model_id", ["zai-org/GLM-4.1V-9B-Thinking"])
92
@pytest.mark.parametrize("expected_toks_per_frame", [299])
93
@pytest.mark.parametrize(
vllm/model_executor/models/glm4_1v.py
+2
-2
@@ -1082,8 +1082,8 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
1082
preprocessed_size = ImageSize(width=image_width, height=image_height)
1083
1084
# NOTE: Frames are padded to be divisible by `temporal_patch_size`
1085
- # https://github.com/huggingface/transformers/blob/v4.48.3/src/transformers/models/qwen2_vl/image_processing_qwen2_vl.py#L294
1086
- padded_num_frames = num_frames + num_frames % temporal_patch_size
1085
+ # https://github.com/huggingface/transformers/blob/v5.13.0/src/transformers/models/qwen2_vl/video_processing_qwen2_vl.py#L249-L252
1086
+ padded_num_frames = num_frames + (-num_frames % temporal_patch_size)
1087
1088
grid_t = max(padded_num_frames // temporal_patch_size, 1)
1089
grid_h = preprocessed_size.height // patch_size
vllm/model_executor/models/kanana_v.py
+2
-2
@@ -409,8 +409,8 @@ class KananaVProcessingInfo(BaseProcessingInfo):
409
preprocessed_size = ImageSize(width=image_width, height=image_height)
410
411
# NOTE: Frames are padded to be divisible by `temporal_patch_size`
412
- # https://github.com/huggingface/transformers/blob/v4.48.3/src/transformers/models/qwen2_vl/image_processing_qwen2_vl.py#L294
413
- padded_num_frames = num_frames + num_frames % temporal_patch_size
412
+ # https://github.com/huggingface/transformers/blob/v5.13.0/src/transformers/models/qwen2_vl/video_processing_qwen2_vl.py#L249-L252
413
+ padded_num_frames = num_frames + (-num_frames % temporal_patch_size)
414
415
grid_t = max(padded_num_frames // temporal_patch_size, 1)
416
grid_h = preprocessed_size.height // patch_size
vllm/model_executor/models/keye.py
+1
-1
@@ -983,7 +983,7 @@ class KeyeProcessingInfo(BaseProcessingInfo):
983
else:
984
preprocessed_size = ImageSize(width=image_width, height=image_height)
985
986
- padded_num_frames = num_frames + num_frames % temporal_patch_size
986
+ padded_num_frames = num_frames + (-num_frames % temporal_patch_size)
987
988
grid_t = max(padded_num_frames // temporal_patch_size, 1)
989
grid_h = preprocessed_size.height // patch_size
vllm/model_executor/models/llava_onevision2.py
+1
-1
@@ -1345,7 +1345,7 @@ class LlavaOnevision2ProcessingInfo(BaseProcessingInfo):
1345
preprocessed = ImageSize(width=rw, height=rh)
1346
else:
1347
preprocessed = ImageSize(width=image_width, height=image_height)
1348
- padded_frames = num_frames + num_frames % temporal_patch_size
1348
+ padded_frames = num_frames + (-num_frames % temporal_patch_size)
1349
grid_t = max(padded_frames // temporal_patch_size, 1)
1350
grid_h = preprocessed.height // patch_size
1351
grid_w = preprocessed.width // patch_size
vllm/model_executor/models/mimo_v2_omni.py
+1
-1
@@ -715,7 +715,7 @@ class MiMoV2OmniProcessingInfo(BaseProcessingInfo):
715
effective_frames = num_frames * tokens_per_second
716
else:
717
effective_frames = num_frames
718
- padded_num_frames = effective_frames + effective_frames % temporal_patch_size
718
+ padded_num_frames = effective_frames + (-effective_frames % temporal_patch_size)
719
grid_t = max(padded_num_frames // temporal_patch_size, 1)
720
grid_h = preprocessed_size.height // patch_size
721
grid_w = preprocessed_size.width // patch_size
vllm/model_executor/models/qwen2_vl.py
+2
-2
@@ -898,8 +898,8 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
898
preprocessed_size = ImageSize(width=image_width, height=image_height)
899
900
# NOTE: Frames are padded to be divisible by `temporal_patch_size`
901
- # https://github.com/huggingface/transformers/blob/v4.48.3/src/transformers/models/qwen2_vl/image_processing_qwen2_vl.py#L294
902
- padded_num_frames = num_frames + num_frames % temporal_patch_size
901
+ # https://github.com/huggingface/transformers/blob/v5.13.0/src/transformers/models/qwen2_vl/video_processing_qwen2_vl.py#L249-L252
902
+ padded_num_frames = num_frames + (-num_frames % temporal_patch_size)
903
904
grid_t = max(padded_num_frames // temporal_patch_size, 1)
905
grid_h = preprocessed_size.height // patch_size