| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Video placeholder accounting for GLM-5.3-Flash. |
| 4 | |
| 5 | ``Glm4vProcessingInfo._construct_video_placeholder`` emits one frame of |
| 6 | placeholders per timestamp, so the timestamps returned by |
| 7 | ``_get_video_second_idx_glm46v`` decide how many placeholders the prompt gets |
| 8 | while ``video_grid_thw`` decides how many rows the vision tower produces. If |
| 9 | the two disagree they collide in ``_merge_multimodal_embeddings``, which raises |
| 10 | inside a worker and takes the engine down with it. |
| 11 | |
| 12 | The checks below are arithmetic: no weights, no GPU. |
| 13 | """ |
| 14 | |
| 15 | import pytest |
| 16 | import torch |
| 17 | from PIL import Image |
| 18 | from transformers.models.glm5_next.video_processing_glm5_next import ( |
| 19 | Glm5NextVideoProcessor, |
| 20 | smart_resize, |
| 21 | ) |
| 22 | from transformers.video_utils import VideoMetadata |
| 23 | |
| 24 | from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm |
| 25 | from vllm.multimodal import MULTIMODAL_REGISTRY |
| 26 | from vllm.platforms import current_platform |
| 27 | |
| 28 | from ...utils import build_model_context |
| 29 | |
| 30 | |
| 31 | @pytest.fixture(scope="module") |
| 32 | def processor(): |
| 33 | ctx = build_model_context( |
| 34 | "zai-org/GLM-5.3-Flash", |
| 35 | limit_mm_per_prompt={"video": 1}, |
| 36 | ) |
| 37 | return MULTIMODAL_REGISTRY.create_processor( |
| 38 | ctx.model_config, |
| 39 | tokenizer=ctx.tokenizer, |
| 40 | ) |
| 41 | |
| 42 | |
| 43 | def _pixel_path_grid( |
| 44 | video_processor: Glm5NextVideoProcessor, |
| 45 | num_frames: int, |
| 46 | height: int, |
| 47 | width: int, |
| 48 | ) -> tuple[int, int, int]: |
| 49 | """The ``video_grid_thw`` ``Glm5NextVideoProcessor._preprocess`` builds.""" |
| 50 | resized_height, resized_width = smart_resize( |
| 51 | num_frames=num_frames, |
| 52 | height=height, |
| 53 | width=width, |
| 54 | temporal_factor=video_processor.temporal_patch_size, |
| 55 | factor=video_processor.patch_size |
| 56 | * video_processor.merge_size |
| 57 | * video_processor.patch_expand_factor, |
| 58 | min_pixels=video_processor.min_image_tokens, |
| 59 | max_pixels=video_processor.max_image_tokens, |
| 60 | ) |
| 61 | padded_frames = num_frames + (-num_frames % video_processor.temporal_patch_size) |
| 62 | return ( |
| 63 | padded_frames // video_processor.temporal_patch_size, |
| 64 | resized_height // video_processor.patch_size, |
| 65 | resized_width // video_processor.patch_size, |
| 66 | ) |
| 67 | |
| 68 | |
| 69 | @pytest.mark.parametrize( |
| 70 | ("total_num_frames", "fps", "duration", "height", "width", "expected_grid"), |
| 71 | [ |
| 72 | # 4 s at 8 fps: the GLM-4.6V sampler asks for 3x as many timestamps. |
| 73 | (32, 8.0, 4.0, 480, 640, (4, 36, 46)), |
| 74 | # 1080p, 20 s: same factor of 3 at a full-size canvas. |
| 75 | (600, 30.0, 20.0, 1080, 1920, (20, 58, 102)), |
| 76 | # 1080p, 60 s: the one duration window where the two samplers agree |
| 77 | # anyway -- a regression guard, the count must not move. |
| 78 | (1800, 30.0, 60.0, 1080, 1920, (60, 34, 58)), |
| 79 | # Past 300 s the GLM-4.6V sampler asks for half as many instead. |
| 80 | (9030, 30.0, 301.0, 720, 1280, (301, 14, 26)), |
| 81 | ], |
| 82 | ) |
| 83 | def test_video_placeholders_match_encoder_rows( |
| 84 | processor, |
| 85 | total_num_frames: int, |
| 86 | fps: float, |
| 87 | duration: float, |
| 88 | height: int, |
| 89 | width: int, |
| 90 | expected_grid: tuple[int, int, int], |
| 91 | ): |
| 92 | info = processor.info |
| 93 | video_processor = info.get_video_processor() |
| 94 | |
| 95 | frame_indices = video_processor.sample_frames( |
| 96 | VideoMetadata(total_num_frames=total_num_frames, fps=fps, duration=duration) |
| 97 | ) |
| 98 | grid_t, grid_h, grid_w = _pixel_path_grid( |
| 99 | video_processor, len(frame_indices), height, width |
| 100 | ) |
| 101 | assert (grid_t, grid_h, grid_w) == expected_grid |
| 102 | |
| 103 | timestamps = info._get_video_second_idx_glm46v( |
| 104 | { |
| 105 | "total_num_frames": total_num_frames, |
| 106 | "fps": fps, |
| 107 | "duration": duration, |
| 108 | "do_sample_frames": True, |
| 109 | }, |
| 110 | total_num_frames, |
| 111 | ) |
| 112 | |
| 113 | merge_length = video_processor.merge_size**2 |
| 114 | tokens_per_frame = grid_h * grid_w // merge_length |
| 115 | encoder_rows = grid_t * grid_h * grid_w // merge_length |
| 116 | |
| 117 | assert len(timestamps) == grid_t |
| 118 | assert len(timestamps) * tokens_per_frame == encoder_rows |
| 119 | assert timestamps == sorted(timestamps) |
| 120 | assert timestamps[0] == 0 |
| 121 | assert timestamps[-1] <= duration |
| 122 | |
| 123 | |
| 124 | def test_video_placeholders_match_encoder_rows_when_presampled(processor): |
| 125 | """The loader may pre-sample and hand the frames over as they are.""" |
| 126 | info = processor.info |
| 127 | video_processor = info.get_video_processor() |
| 128 | |
| 129 | num_frames = 32 |
| 130 | grid_t, _, _ = _pixel_path_grid(video_processor, num_frames, 480, 640) |
| 131 | |
| 132 | timestamps = info._get_video_second_idx_glm46v( |
| 133 | { |
| 134 | "total_num_frames": 256, |
| 135 | "fps": 8.0, |
| 136 | "duration": 32.0, |
| 137 | "do_sample_frames": False, |
| 138 | "frames_indices": list(range(0, 256, 256 // num_frames)), |
| 139 | }, |
| 140 | num_frames, |
| 141 | ) |
| 142 | |
| 143 | assert len(timestamps) == grid_t |
| 144 | |
| 145 | |
| 146 | def test_video_shorter_than_one_sampling_interval_is_rejected(processor): |
| 147 | """A clip the sampler cannot pick a single frame from is a bad request.""" |
| 148 | with pytest.raises(ValueError, match="selected no frames"): |
| 149 | processor.info._get_video_second_idx_glm46v( |
| 150 | { |
| 151 | "total_num_frames": 2, |
| 152 | "fps": 8.0, |
| 153 | "duration": 0.25, |
| 154 | "do_sample_frames": True, |
| 155 | }, |
| 156 | 2, |
| 157 | ) |
| 158 | |
| 159 | |
| 160 | @pytest.mark.usefixtures("default_vllm_config") |
| 161 | def test_mm_device_do_normalize(): |
| 162 | device = current_platform.device_type |
| 163 | ctx = build_model_context( |
| 164 | "zai-org/GLM-5.3-Flash", |
| 165 | limit_mm_per_prompt={"image": 2}, |
| 166 | ) |
| 167 | assert ctx.model_config.multimodal_config.mm_device_do_normalize |
| 168 | |
| 169 | ctx.model_config.multimodal_config.mm_device_do_normalize = False |
| 170 | processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) |
| 171 | images = [ |
| 172 | Image.new("RGB", (310, 470), color=(17, 89, 231)), |
| 173 | Image.new("RGB", (480, 320), color=(201, 13, 127)), |
| 174 | ] |
| 175 | prompt = "<|begin_of_image|><|image|><|end_of_image|>" * len(images) |
| 176 | mm_items = processor.info.parse_mm_data({"image": images}) |
| 177 | |
| 178 | normalized_inputs = processor(prompt, mm_items=mm_items) |
| 179 | normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"] |
| 180 | |
| 181 | ctx.model_config.multimodal_config.mm_device_do_normalize = True |
| 182 | raw_inputs = processor(prompt, mm_items=mm_items) |
| 183 | raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"] |
| 184 | assert raw_values.dtype == torch.uint8 |
| 185 | |
| 186 | input_norm = build_mm_input_norm(ctx.model_config).to(device) |
| 187 | output = input_norm(raw_values.to(device), normalized_values.dtype) |
| 188 | torch.testing.assert_close( |
| 189 | output, normalized_values.to(device), rtol=1e-5, atol=1e-6 |
| 190 | ) |
| 191 | |
| 192 | |
| 193 | def _image_info(**kwargs): |
| 194 | ctx = build_model_context( |
| 195 | "zai-org/GLM-5.3-Flash", |
| 196 | limit_mm_per_prompt={"image": 1}, |
| 197 | **kwargs, |
| 198 | ) |
| 199 | return MULTIMODAL_REGISTRY.create_processor( |
| 200 | ctx.model_config, |
| 201 | tokenizer=ctx.tokenizer, |
| 202 | ).info |
| 203 | |
| 204 | |
| 205 | def test_image_encoder_cache_covers_full_token_budget(): |
| 206 | """The most-features probe must reach the processor's token ceiling. |
| 207 | |
| 208 | The inherited square probe refits to 89x89 = 7921 tokens under the |
| 209 | max_image_tokens=8000 budget, so the encoder cache came up short and |
| 210 | ordinary non-square images in 7922-8000 tokens were refused with |
| 211 | HTTP 400 (#59539). |
| 212 | """ |
| 213 | info = _image_info() |
| 214 | assert info.get_max_image_tokens() == 8000 |
| 215 | |
| 216 | # The shapes from the issue, with the token counts the processor |
| 217 | # actually produces; every one must fit the cache. |
| 218 | for width, height, expected_tokens in [ |
| 219 | (4032, 3024, 7931), # phone photo, 4:3 |
| 220 | (3840, 2160, 7973), # 4K frame, 16:9 |
| 221 | (3508, 2480, 7950), # A4 at 300 dpi |
| 222 | (2600, 2400, 7998), # 13:12 |
| 223 | (3000, 3000, 7921), # square worst case before the fix |
| 224 | ]: |
| 225 | num_tokens = info.get_num_image_tokens(image_width=width, image_height=height) |
| 226 | assert num_tokens == expected_tokens |
| 227 | assert num_tokens <= info.get_max_image_tokens() |
| 228 | |
| 229 | # The profiling dummy covers the real worst case, not the square one. |
| 230 | size = info.get_image_size_with_most_features() |
| 231 | assert size.width * size.height == 2240 * 2800 |
| 232 | |
| 233 | |
| 234 | def test_image_encoder_cache_follows_max_pixels_override(): |
| 235 | info = _image_info(mm_processor_kwargs={"max_pixels": 1568 * 100}) |
| 236 | assert info.get_max_image_tokens() == 100 |
| 237 | size = info.get_image_size_with_most_features() |
| 238 | assert ( |
| 239 | info.get_num_image_tokens(image_width=size.width, image_height=size.height) |
| 240 | == 100 |
| 241 | ) |