| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from unittest.mock import Mock |
| 5 | |
| 6 | import pytest |
| 7 | |
| 8 | from vllm.assets.video import VideoAsset |
| 9 | from vllm.model_executor.models.glm4_1v import ( |
| 10 | Glm4vForConditionalGeneration, |
| 11 | Glm4vProcessingInfo, |
| 12 | ) |
| 13 | from vllm.multimodal import MULTIMODAL_REGISTRY |
| 14 | from vllm.multimodal.inputs import batched_tensors_equal |
| 15 | from vllm.multimodal.video import DynamicVideoBackend, VideoBackend |
| 16 | |
| 17 | from ...utils import build_model_context |
| 18 | |
| 19 | |
| 20 | @pytest.mark.parametrize( |
| 21 | ( |
| 22 | "max_video_pixels", |
| 23 | "max_tokens", |
| 24 | "expected_num_frames", |
| 25 | ), |
| 26 | [ |
| 27 | (47_040_000, 124_988, 11), |
| 28 | (47_040_000, 30_000, 24), |
| 29 | (100_352_000, 124_988, 21), |
| 30 | (100_352_000, 30_000, 7), |
| 31 | (100_352_000, 0, 1), |
| 32 | ], |
| 33 | ) |
| 34 | def test_get_max_video_frames_matches_glm_resize( |
| 35 | max_video_pixels: int, |
| 36 | max_tokens: int, |
| 37 | expected_num_frames: int, |
| 38 | ): |
| 39 | info = Mock(spec=Glm4vProcessingInfo) |
| 40 | info.get_image_size_with_most_features.return_value = (2184, 2184) |
| 41 | info._get_video_max_pixels.return_value = max_video_pixels |
| 42 | vision_config = info.get_hf_config.return_value.vision_config |
| 43 | vision_config.patch_size = 14 |
| 44 | vision_config.spatial_merge_size = 2 |
| 45 | vision_config.temporal_patch_size = 2 |
| 46 | info._get_vision_info.side_effect = lambda **kwargs: ( |
| 47 | Glm4vProcessingInfo._get_vision_info(info, **kwargs) |
| 48 | ) |
| 49 | |
| 50 | num_frames = Glm4vProcessingInfo._get_max_video_frames( |
| 51 | info, |
| 52 | max_tokens=max_tokens, |
| 53 | ) |
| 54 | |
| 55 | assert num_frames == expected_num_frames |
| 56 | assert info._get_video_max_pixels.call_count == 1 |
| 57 | assert info._get_vision_info.call_count == 600 |
| 58 | |
| 59 | |
| 60 | def test_encoder_cudagraph_uses_model_video_frame_limit(): |
| 61 | model = Mock() |
| 62 | |
| 63 | assert Glm4vForConditionalGeneration.get_max_frames_per_video(model) == 600 |
| 64 | |
| 65 | |
| 66 | @pytest.mark.parametrize( |
| 67 | ("temporal_patch_size", "expected_grid_t"), |
| 68 | [(2, 9), (4, 5), (8, 3)], |
| 69 | ) |
| 70 | def test_vision_info_rounds_up_temporal_frames( |
| 71 | temporal_patch_size: int, |
| 72 | expected_grid_t: int, |
| 73 | ): |
| 74 | info = Mock(spec=Glm4vProcessingInfo) |
| 75 | vision_config = info.get_hf_config.return_value.vision_config |
| 76 | vision_config.patch_size = 14 |
| 77 | vision_config.spatial_merge_size = 2 |
| 78 | vision_config.temporal_patch_size = temporal_patch_size |
| 79 | |
| 80 | _, num_vision_tokens = Glm4vProcessingInfo._get_vision_info( |
| 81 | info, |
| 82 | image_width=28, |
| 83 | image_height=28, |
| 84 | num_frames=17, |
| 85 | do_resize=False, |
| 86 | ) |
| 87 | |
| 88 | assert num_vision_tokens == expected_grid_t |
| 89 | |
| 90 | |
| 91 | @pytest.mark.parametrize("model_id", ["zai-org/GLM-4.1V-9B-Thinking"]) |
| 92 | @pytest.mark.parametrize("expected_toks_per_frame", [299]) |
| 93 | @pytest.mark.parametrize( |
| 94 | "num_frames, fps, expected_grid_t", |
| 95 | [ |
| 96 | # pre-sampled fixed frames (unexpected behavior, |
| 97 | # but we still expect it to work without errors) |
| 98 | (32, 1, 16), |
| 99 | (32, 2, 16), |
| 100 | (128, 1, 64), |
| 101 | (128, 2, 64), |
| 102 | # post-sampled frames (expected behavior) |
| 103 | (-1, 1, 5), |
| 104 | (-1, 2, 10), |
| 105 | ], |
| 106 | ) |
| 107 | def test_processor_override( |
| 108 | model_id: str, |
| 109 | expected_toks_per_frame: int, |
| 110 | expected_grid_t: int, |
| 111 | fps: int, |
| 112 | num_frames: int, |
| 113 | ): |
| 114 | """Ensure GLM4vMultiModalProcessor can handle video frames properly.""" |
| 115 | ctx = build_model_context( |
| 116 | model_id, |
| 117 | mm_processor_kwargs=None, |
| 118 | limit_mm_per_prompt={"video": 1}, |
| 119 | ) |
| 120 | processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) |
| 121 | hf_processor_mm_kwargs = {"fps": fps} |
| 122 | |
| 123 | # Build the image str / prompt based on the number of images we pass |
| 124 | video_assets = VideoAsset(name="baby_reading", num_frames=num_frames) |
| 125 | prompt = "<|begin_of_video|><|video|><|end_of_video|>" |
| 126 | |
| 127 | video, metadata = video_assets.np_ndarrays, video_assets.metadata |
| 128 | metadata["fps"] = fps |
| 129 | mm_data = {"video": [(video, metadata)]} |
| 130 | |
| 131 | processed_inputs = processor( |
| 132 | prompt, |
| 133 | mm_items=processor.info.parse_mm_data(mm_data), |
| 134 | hf_processor_mm_kwargs=hf_processor_mm_kwargs, |
| 135 | ) |
| 136 | |
| 137 | # Ensure we have the right number of placeholders per num_crops size |
| 138 | hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs) |
| 139 | image_token_id = hf_processor.image_token_id |
| 140 | video_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id) |
| 141 | grid_t, _, _ = processed_inputs["mm_kwargs"].get_data()["video_grid_thw"][0] |
| 142 | |
| 143 | assert grid_t == expected_grid_t |
| 144 | assert video_tok_count == expected_toks_per_frame * grid_t |
| 145 | |
| 146 | |
| 147 | @pytest.mark.parametrize("model_id", ["zai-org/GLM-4.1V-9B-Thinking"]) |
| 148 | @pytest.mark.parametrize("fps", [2]) |
| 149 | @pytest.mark.parametrize("backend", ["opencv"]) |
| 150 | def test_video_loader_consistency( |
| 151 | model_id: str, |
| 152 | fps: int, |
| 153 | backend: str, |
| 154 | ): |
| 155 | """Ensure dynamic video loader (pre-sampled by loader) and normal video |
| 156 | loader (post-sampled by processor) produce same video processing outputs. |
| 157 | """ |
| 158 | ctx = build_model_context( |
| 159 | model_id, |
| 160 | mm_processor_kwargs=None, |
| 161 | limit_mm_per_prompt={"video": 1}, |
| 162 | ) |
| 163 | processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) |
| 164 | hf_processor_mm_kwargs = {"fps": fps} |
| 165 | |
| 166 | # Build the image str / prompt based on the number of images we pass |
| 167 | prompt = "<|begin_of_video|><|video|><|end_of_video|>" |
| 168 | |
| 169 | video_path = VideoAsset(name="baby_reading", num_frames=-1).video_path |
| 170 | with open(video_path, "rb") as f: |
| 171 | video_bytes = f.read() |
| 172 | |
| 173 | static_video, static_metadata = VideoBackend.load_bytes( |
| 174 | video_bytes, backend=backend |
| 175 | ) |
| 176 | dynamic_video, dynamic_metadata = DynamicVideoBackend.load_bytes( |
| 177 | video_bytes, fps=fps, backend=backend |
| 178 | ) |
| 179 | |
| 180 | # pre-sampled loader shouldn't read all frames |
| 181 | assert len(dynamic_video) < len(static_video) |
| 182 | |
| 183 | static_mm_data = {"video": [(static_video, static_metadata)]} |
| 184 | dynamic_mm_data = {"video": [(dynamic_video, dynamic_metadata)]} |
| 185 | |
| 186 | static_outputs = processor( |
| 187 | prompt, |
| 188 | mm_items=processor.info.parse_mm_data(static_mm_data), |
| 189 | hf_processor_mm_kwargs=hf_processor_mm_kwargs, |
| 190 | ) |
| 191 | dynamic_outputs = processor( |
| 192 | prompt, |
| 193 | mm_items=processor.info.parse_mm_data(dynamic_mm_data), |
| 194 | hf_processor_mm_kwargs=hf_processor_mm_kwargs, |
| 195 | ) |
| 196 | |
| 197 | assert static_outputs["prompt_token_ids"] == dynamic_outputs["prompt_token_ids"] |
| 198 | assert batched_tensors_equal( |
| 199 | static_outputs["mm_kwargs"].get_data(), |
| 200 | dynamic_outputs["mm_kwargs"].get_data(), |
| 201 | ) |