| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Regression tests for Qwen3-VL multimodal processing.""" |
| 4 | |
| 5 | from copy import deepcopy |
| 6 | from types import SimpleNamespace |
| 7 | from typing import Any |
| 8 | |
| 9 | import numpy as np |
| 10 | import pytest |
| 11 | import torch |
| 12 | from transformers.models.qwen3_vl import Qwen3VLVideoProcessor |
| 13 | |
| 14 | from vllm.config import ModelConfig |
| 15 | from vllm.config.multimodal import MultiModalDummyOptions |
| 16 | from vllm.model_executor.models.qwen3_vl import Qwen3VLProcessingInfo |
| 17 | from vllm.multimodal import MULTIMODAL_REGISTRY |
| 18 | from vllm.multimodal.processing.context import BaseProcessingInfo |
| 19 | |
| 20 | from ...registry import HF_EXAMPLE_MODELS |
| 21 | from ...utils import build_model_context |
| 22 | |
| 23 | MODEL_ID = "Qwen/Qwen3-VL-4B-Instruct" |
| 24 | |
| 25 | |
| 26 | def test_qwen3_video_routing_schema_adds_vllm_compatibility_kwargs( |
| 27 | monkeypatch: pytest.MonkeyPatch, |
| 28 | ) -> None: |
| 29 | native_video_kwargs = set(Qwen3VLVideoProcessor.valid_kwargs.__annotations__) |
| 30 | assert "size" in native_video_kwargs |
| 31 | assert "min_pixels" not in native_video_kwargs |
| 32 | assert "max_pixels" not in native_video_kwargs |
| 33 | |
| 34 | info = Qwen3VLProcessingInfo(SimpleNamespace()) |
| 35 | parent_supported = { |
| 36 | "text_kwargs": {"padding"}, |
| 37 | "images_kwargs": {"size", "min_pixels", "max_pixels"}, |
| 38 | "videos_kwargs": set(native_video_kwargs), |
| 39 | } |
| 40 | monkeypatch.setattr( |
| 41 | BaseProcessingInfo, |
| 42 | "get_supported_mm_processor_kwargs", |
| 43 | lambda self: {scope: set(kwargs) for scope, kwargs in parent_supported.items()}, |
| 44 | ) |
| 45 | |
| 46 | supported = info.supported_mm_processor_kwargs |
| 47 | |
| 48 | assert {"min_pixels", "max_pixels"} <= supported["videos_kwargs"] |
| 49 | assert parent_supported["videos_kwargs"] == native_video_kwargs |
| 50 | |
| 51 | |
| 52 | def _build_video_mm_data( |
| 53 | num_frames: int, |
| 54 | width: int = 128, |
| 55 | height: int = 128, |
| 56 | original_fps: float = 30.0, |
| 57 | ) -> dict[str, Any]: |
| 58 | """Create synthetic video data with metadata indicating that |
| 59 | HF processor should re-sample frames (do_sample_frames=True). |
| 60 | |
| 61 | ``total_num_frames`` is set equal to the ndarray frame count so |
| 62 | that HF's ``sample_frames`` indices stay within bounds of the |
| 63 | actual tensor that is passed.""" |
| 64 | video = np.zeros((num_frames, height, width, 3), dtype=np.uint8) |
| 65 | metadata = { |
| 66 | "fps": original_fps, |
| 67 | "duration": num_frames / original_fps, |
| 68 | "total_num_frames": num_frames, |
| 69 | "frames_indices": list(range(num_frames)), |
| 70 | "video_backend": "opencv", |
| 71 | "do_sample_frames": True, |
| 72 | } |
| 73 | return {"video": [(video, metadata)]} |
| 74 | |
| 75 | |
| 76 | @pytest.mark.parametrize("model_id", [MODEL_ID]) |
| 77 | def test_processor_partial_scoped_image_size(model_id: str) -> None: |
| 78 | """Partial image ``size`` must be completed and applied before the HF call. |
| 79 | |
| 80 | Keep video compatibility and per-video kwargs in an image-only call as well: |
| 81 | Qwen3 must sanitize the video scope before HF validates it. Regression for |
| 82 | #56363. |
| 83 | """ |
| 84 | ctx = build_model_context( |
| 85 | model_id, |
| 86 | limit_mm_per_prompt={"image": 1, "video": 0}, |
| 87 | ) |
| 88 | processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) |
| 89 | |
| 90 | image = np.zeros((128, 128, 3), dtype=np.uint8) |
| 91 | min_pixels = 256 * 32 * 32 |
| 92 | default_size = dict(processor.info.get_image_processor().size) |
| 93 | complete_size = default_size | {"shortest_edge": min_pixels} |
| 94 | |
| 95 | def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor: |
| 96 | mm_items = processor.info.parse_mm_data({"image": [image]}) |
| 97 | return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[ |
| 98 | "image_grid_thw" |
| 99 | ] |
| 100 | |
| 101 | stock_grid = process({}) |
| 102 | complete_grid = process({"images_kwargs": {"size": complete_size}}) |
| 103 | partial_grid = process( |
| 104 | { |
| 105 | "images_kwargs": { |
| 106 | "size": {"shortest_edge": min_pixels}, |
| 107 | }, |
| 108 | "videos_kwargs": { |
| 109 | "min_pixels": 4096, |
| 110 | "max_pixels": 469_762_048, |
| 111 | "fps": [2.0], |
| 112 | "num_frames": [8], |
| 113 | }, |
| 114 | } |
| 115 | ) |
| 116 | |
| 117 | assert torch.equal(partial_grid, complete_grid) |
| 118 | assert not torch.equal(partial_grid, stock_grid) |
| 119 | |
| 120 | |
| 121 | @pytest.mark.parametrize("model_id", [MODEL_ID]) |
| 122 | def test_processor_partial_scoped_video_size(model_id: str) -> None: |
| 123 | """Partial scoped video ``size`` must be completed and applied before HF.""" |
| 124 | ctx = build_model_context( |
| 125 | model_id, |
| 126 | limit_mm_per_prompt={"image": 0, "video": 1}, |
| 127 | ) |
| 128 | processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) |
| 129 | |
| 130 | min_pixels = 512 * 32 * 32 |
| 131 | default_size = dict(processor.info.get_video_processor().size) |
| 132 | complete_size = default_size | {"shortest_edge": min_pixels} |
| 133 | |
| 134 | def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor: |
| 135 | mm_items = processor.info.parse_mm_data(_build_video_mm_data(num_frames=8)) |
| 136 | return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[ |
| 137 | "video_grid_thw" |
| 138 | ] |
| 139 | |
| 140 | stock_grid = process({}) |
| 141 | complete_grid = process({"videos_kwargs": {"size": complete_size}}) |
| 142 | partial_grid = process( |
| 143 | { |
| 144 | "videos_kwargs": { |
| 145 | "size": {"shortest_edge": min_pixels}, |
| 146 | }, |
| 147 | } |
| 148 | ) |
| 149 | |
| 150 | assert torch.equal(partial_grid, complete_grid) |
| 151 | assert not torch.equal(partial_grid, stock_grid) |
| 152 | |
| 153 | |
| 154 | @pytest.mark.parametrize("model_id", [MODEL_ID]) |
| 155 | def test_processor_flat_video_pixel_aliases_match_scoped_size(model_id: str) -> None: |
| 156 | """Flat min/max pixel aliases keep their historical Qwen3 video effect.""" |
| 157 | ctx = build_model_context( |
| 158 | model_id, |
| 159 | limit_mm_per_prompt={"image": 0, "video": 1}, |
| 160 | ) |
| 161 | processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) |
| 162 | pixel_budget = 512 * 32 * 32 |
| 163 | |
| 164 | def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor: |
| 165 | mm_items = processor.info.parse_mm_data(_build_video_mm_data(num_frames=8)) |
| 166 | return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[ |
| 167 | "video_grid_thw" |
| 168 | ] |
| 169 | |
| 170 | stock_grid = process({}) |
| 171 | scoped_grid = process( |
| 172 | { |
| 173 | "videos_kwargs": { |
| 174 | "size": { |
| 175 | "shortest_edge": pixel_budget, |
| 176 | "longest_edge": pixel_budget, |
| 177 | }, |
| 178 | }, |
| 179 | } |
| 180 | ) |
| 181 | flat_grid = process( |
| 182 | { |
| 183 | "min_pixels": pixel_budget, |
| 184 | "max_pixels": pixel_budget, |
| 185 | } |
| 186 | ) |
| 187 | |
| 188 | assert torch.equal(flat_grid, scoped_grid) |
| 189 | assert not torch.equal(flat_grid, stock_grid) |
| 190 | |
| 191 | |
| 192 | @pytest.mark.parametrize("model_id", [MODEL_ID]) |
| 193 | @pytest.mark.parametrize( |
| 194 | "num_frames", |
| 195 | [8, 16], |
| 196 | ) |
| 197 | def test_processor_num_frames_timestamp( |
| 198 | model_id: str, |
| 199 | num_frames: int, |
| 200 | ) -> None: |
| 201 | """Regression test: using ``num_frames`` (without ``fps``) must not |
| 202 | cause a timestamp / token-count mismatch. |
| 203 | |
| 204 | Before the fix, ``_get_video_second_idx`` ignored the explicit |
| 205 | ``num_frames`` and fell back to an fps-based calculation, which |
| 206 | produced a different number of timestamp entries and ultimately led |
| 207 | to shape mismatches in downstream token construction. |
| 208 | |
| 209 | We deliberately choose ``num_frames`` values (8, 16) that differ |
| 210 | from what the default fps-based path would compute (which clamps |
| 211 | to ``min_frames=4`` for a short video at 30 fps), so this test |
| 212 | would fail without the fix. |
| 213 | """ |
| 214 | ctx = build_model_context( |
| 215 | model_id, |
| 216 | limit_mm_per_prompt={"image": 0, "video": 1}, |
| 217 | ) |
| 218 | processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) |
| 219 | |
| 220 | prompt = "<|vision_start|><|video_pad|><|vision_end|>" |
| 221 | mm_data = _build_video_mm_data(num_frames=num_frames) |
| 222 | |
| 223 | # Process with explicit num_frames (no fps) -- this is the path |
| 224 | # that was broken before the fix. |
| 225 | hf_mm_kwargs: dict[str, Any] = {"num_frames": num_frames} |
| 226 | processed = processor( |
| 227 | prompt, |
| 228 | mm_items=processor.info.parse_mm_data(mm_data), |
| 229 | hf_processor_mm_kwargs=hf_mm_kwargs, |
| 230 | ) |
| 231 | |
| 232 | # Basic sanity: the processor must produce video tokens. |
| 233 | token_ids = processed["prompt_token_ids"] |
| 234 | assert len(token_ids) > 0, "Processor produced empty token list" |
| 235 | |
| 236 | # Verify that video placeholders were actually inserted. |
| 237 | assert "mm_placeholders" in processed |
| 238 | video_phs = processed["mm_placeholders"].get("video", []) |
| 239 | assert len(video_phs) == 1, ( |
| 240 | f"Expected exactly 1 video placeholder, got {len(video_phs)}" |
| 241 | ) |
| 242 | |
| 243 | |
| 244 | @pytest.mark.parametrize("model_id", [MODEL_ID]) |
| 245 | @pytest.mark.parametrize("num_videos", [2, 4]) |
| 246 | def test_processor_multi_video( |
| 247 | model_id: str, |
| 248 | num_videos: int, |
| 249 | ) -> None: |
| 250 | """Verify that multi-video processing produces correct placeholders. |
| 251 | |
| 252 | This exercises the token-level replacement path in |
| 253 | ``_apply_hf_processor_main`` which avoids the quadratic text-level |
| 254 | prompt expansion. |
| 255 | """ |
| 256 | ctx = build_model_context( |
| 257 | model_id, |
| 258 | limit_mm_per_prompt={"image": 0, "video": num_videos}, |
| 259 | ) |
| 260 | processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) |
| 261 | |
| 262 | prompt = "<|vision_start|><|video_pad|><|vision_end|>" * num_videos |
| 263 | mm_data = {"video": [_build_video_mm_data(num_frames=8)["video"][0]] * num_videos} |
| 264 | |
| 265 | processed = processor( |
| 266 | prompt, |
| 267 | mm_items=processor.info.parse_mm_data(mm_data), |
| 268 | hf_processor_mm_kwargs={"num_frames": 8}, |
| 269 | ) |
| 270 | |
| 271 | token_ids = processed["prompt_token_ids"] |
| 272 | assert len(token_ids) > 0 |
| 273 | |
| 274 | video_phs = processed["mm_placeholders"].get("video", []) |
| 275 | assert len(video_phs) == num_videos, ( |
| 276 | f"Expected {num_videos} video placeholders, got {len(video_phs)}" |
| 277 | ) |
| 278 | |
| 279 | # All placeholders should have the same length (same video params) |
| 280 | # and must not overlap. |
| 281 | lengths = {ph.length for ph in video_phs} |
| 282 | assert len(lengths) == 1, f"Placeholder lengths differ: {lengths}" |
| 283 | for i in range(1, len(video_phs)): |
| 284 | prev_end = video_phs[i - 1].offset + video_phs[i - 1].length |
| 285 | assert video_phs[i].offset >= prev_end, ( |
| 286 | f"Placeholder {i} overlaps with placeholder {i - 1}" |
| 287 | ) |
| 288 | |
| 289 | |
| 290 | # Qwen3-VL / Qwen3.8 "Long Video Understanding" pixel budget from the |
| 291 | # model card. Used to check --mm-processor-kwargs scoping (#52834). |
| 292 | _LONG_VIDEO_SIZE = {"longest_edge": 469762048, "shortest_edge": 4096} |
| 293 | |
| 294 | |
| 295 | def _probe_mm_token_budgets( |
| 296 | model_id: str, mm_processor_kwargs: dict[str, Any] | None |
| 297 | ) -> tuple[int, int]: |
| 298 | ctx = build_model_context( |
| 299 | model_id, |
| 300 | mm_processor_kwargs=mm_processor_kwargs, |
| 301 | limit_mm_per_prompt={"image": 1, "video": 1}, |
| 302 | ) |
| 303 | info = MULTIMODAL_REGISTRY.create_processor(ctx.model_config).info |
| 304 | video = info.get_max_video_tokens( |
| 305 | seq_len=500000, mm_counts={"video": 1, "image": 1} |
| 306 | ) |
| 307 | return video, info.get_max_image_tokens() |
| 308 | |
| 309 | |
| 310 | @pytest.mark.skip_global_cleanup |
| 311 | @pytest.mark.parametrize("model_id", [MODEL_ID]) |
| 312 | def test_processor_kwargs_videos_kwargs_does_not_leak_into_image_budget( |
| 313 | model_id: str, |
| 314 | ) -> None: |
| 315 | """``videos_kwargs.size`` must raise only the video token budget. |
| 316 | |
| 317 | A flat ``size`` override still applies to both modalities (the previous |
| 318 | shared-namespace behavior). Regression for #52834. |
| 319 | """ |
| 320 | stock_video, stock_image = _probe_mm_token_budgets(model_id, None) |
| 321 | scoped_video, scoped_image = _probe_mm_token_budgets( |
| 322 | model_id, {"videos_kwargs": {"size": _LONG_VIDEO_SIZE}} |
| 323 | ) |
| 324 | flat_video, flat_image = _probe_mm_token_budgets( |
| 325 | model_id, {"size": _LONG_VIDEO_SIZE} |
| 326 | ) |
| 327 | |
| 328 | assert scoped_video == flat_video |
| 329 | assert scoped_video > stock_video |
| 330 | assert scoped_image == stock_image |
| 331 | assert flat_image > stock_image |
| 332 | |
| 333 | |
| 334 | @pytest.mark.parametrize("model_id", [MODEL_ID]) |
| 335 | @pytest.mark.parametrize( |
| 336 | ("hf_mm_kwargs", "scalar_kwargs"), |
| 337 | [ |
| 338 | ({"num_frames": [8, 16]}, [{"num_frames": 8}, {"num_frames": 16}]), |
| 339 | ({"fps": [4.0, 12.0]}, [{"fps": 4.0}, {"fps": 12.0}]), |
| 340 | ], |
| 341 | ) |
| 342 | def test_processor_multi_video_list_kwargs( |
| 343 | model_id: str, |
| 344 | hf_mm_kwargs: dict[str, Any], |
| 345 | scalar_kwargs: list[dict[str, Any]], |
| 346 | ) -> None: |
| 347 | """List-valued video kwargs must be scalarized per item without mutation. |
| 348 | |
| 349 | Each video in the multi-video request must match processing the same input |
| 350 | independently with its corresponding scalar ``fps``/``num_frames`` value. |
| 351 | """ |
| 352 | ctx = build_model_context( |
| 353 | model_id, |
| 354 | limit_mm_per_prompt={"image": 0, "video": 2}, |
| 355 | ) |
| 356 | processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) |
| 357 | |
| 358 | prompt = ( |
| 359 | "<|vision_start|><|video_pad|><|vision_end|>" |
| 360 | "<|vision_start|><|video_pad|><|vision_end|>" |
| 361 | ) |
| 362 | mm_data = { |
| 363 | "video": [ |
| 364 | _build_video_mm_data(num_frames=32)["video"][0], |
| 365 | _build_video_mm_data(num_frames=32)["video"][0], |
| 366 | ] |
| 367 | } |
| 368 | |
| 369 | hf_mm_kwargs_before = deepcopy(hf_mm_kwargs) |
| 370 | processed = processor( |
| 371 | prompt, |
| 372 | mm_items=processor.info.parse_mm_data(mm_data), |
| 373 | hf_processor_mm_kwargs=hf_mm_kwargs, |
| 374 | ) |
| 375 | |
| 376 | def process_single(hf_kwargs: dict[str, Any]) -> torch.Tensor: |
| 377 | single_prompt = "<|vision_start|><|video_pad|><|vision_end|>" |
| 378 | single_mm_data = _build_video_mm_data(num_frames=32) |
| 379 | single = processor( |
| 380 | single_prompt, |
| 381 | mm_items=processor.info.parse_mm_data(single_mm_data), |
| 382 | hf_processor_mm_kwargs=hf_kwargs, |
| 383 | ) |
| 384 | return single["mm_kwargs"].get_data()["video_grid_thw"][0] |
| 385 | |
| 386 | expected_grids = [process_single(kwargs) for kwargs in scalar_kwargs] |
| 387 | multi_grids = processed["mm_kwargs"].get_data()["video_grid_thw"] |
| 388 | |
| 389 | assert not torch.equal(expected_grids[0], expected_grids[1]) |
| 390 | assert torch.equal(multi_grids[0], expected_grids[0]) |
| 391 | assert torch.equal(multi_grids[1], expected_grids[1]) |
| 392 | assert hf_mm_kwargs == hf_mm_kwargs_before |
| 393 | |
| 394 | video_phs = processed["mm_placeholders"].get("video", []) |
| 395 | assert len(video_phs) == 2, ( |
| 396 | f"Expected exactly 2 video placeholders, got {len(video_phs)}" |
| 397 | ) |
| 398 | |
| 399 | |
| 400 | def _build_video_embeds_mm_data( |
| 401 | hidden_size: int, |
| 402 | grid_thw: tuple[int, int, int] = (2, 4, 4), |
| 403 | ) -> dict[str, Any]: |
| 404 | """Create an embeds-only video item as an EPD consumer receives it: |
| 405 | pre-computed embeddings plus the metadata published by the encoder.""" |
| 406 | import torch |
| 407 | |
| 408 | t, h, w = grid_thw |
| 409 | num_tokens = t * h * w // 4 # spatial_merge_size ** 2 |
| 410 | return { |
| 411 | "video": { |
| 412 | "video_embeds": torch.zeros(num_tokens, hidden_size), |
| 413 | "video_grid_thw": torch.tensor([grid_thw]), |
| 414 | "timestamps": torch.tensor([[float(i) for i in range(t)]]), |
| 415 | } |
| 416 | } |
| 417 | |
| 418 | |
| 419 | @pytest.mark.parametrize("model_id", [MODEL_ID]) |
| 420 | def test_processor_video_embeds_with_timestamps(model_id: str) -> None: |
| 421 | """Embeds-only video input must size the placeholder range from the |
| 422 | grid and the real timestamps published by the encoder (EC consumer |
| 423 | path); synthesized or missing timestamps would change the token count |
| 424 | and break embedding merging downstream.""" |
| 425 | # `build_model_context` forces enable_mm_embeds off, so build the |
| 426 | # config directly; keep the same online-availability skip behavior. |
| 427 | HF_EXAMPLE_MODELS.find_hf_info(model_id).check_available_online(on_fail="skip") |
| 428 | model_config = ModelConfig( |
| 429 | model_id, |
| 430 | runner="generate", |
| 431 | limit_mm_per_prompt={"image": 0, "video": 1}, |
| 432 | enable_mm_embeds=True, |
| 433 | ) |
| 434 | processor = MULTIMODAL_REGISTRY.create_processor(model_config) |
| 435 | tokenizer = processor.info.get_tokenizer() |
| 436 | |
| 437 | grid_thw = (2, 4, 4) |
| 438 | mm_data = _build_video_embeds_mm_data( |
| 439 | model_config.get_inputs_embeds_size(), grid_thw |
| 440 | ) |
| 441 | |
| 442 | prompt = "<|vision_start|><|video_pad|><|vision_end|>" |
| 443 | processed = processor( |
| 444 | prompt, |
| 445 | mm_items=processor.info.parse_mm_data(mm_data), |
| 446 | hf_processor_mm_kwargs={}, |
| 447 | ) |
| 448 | |
| 449 | video_phs = processed["mm_placeholders"].get("video", []) |
| 450 | assert len(video_phs) == 1, ( |
| 451 | f"Expected exactly 1 video placeholder, got {len(video_phs)}" |
| 452 | ) |
| 453 | |
| 454 | t, h, w = grid_thw |
| 455 | tokens_per_frame = h * w // 4 |
| 456 | timestamp_tokens = sum( |
| 457 | len(tokenizer.encode(f"<{float(i):.1f} seconds>", add_special_tokens=False)) |
| 458 | for i in range(t) |
| 459 | ) |
| 460 | # per frame: timestamp tokens + vision_start + video tokens + vision_end |
| 461 | expected_len = timestamp_tokens + t * (tokens_per_frame + 2) |
| 462 | assert video_phs[0].length == expected_len |
| 463 | |
| 464 | |
| 465 | @pytest.mark.parametrize("model_id", [MODEL_ID]) |
| 466 | def test_processor_video_embeds_missing_timestamps(model_id: str) -> None: |
| 467 | """Timestamps are required metadata for video embeds: they size the |
| 468 | placeholder range, so omitting them must fail loudly at parse time |
| 469 | instead of silently producing a wrong prompt.""" |
| 470 | ctx = build_model_context( |
| 471 | model_id, |
| 472 | limit_mm_per_prompt={"image": 0, "video": 1}, |
| 473 | ) |
| 474 | processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) |
| 475 | |
| 476 | mm_data = _build_video_embeds_mm_data(ctx.model_config.get_inputs_embeds_size()) |
| 477 | del mm_data["video"]["timestamps"] |
| 478 | |
| 479 | with pytest.raises(ValueError, match="timestamps"): |
| 480 | processor.info.parse_mm_data(mm_data) |
| 481 | |
| 482 | |
| 483 | @pytest.mark.parametrize("model_id", [MODEL_ID]) |
| 484 | def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> None: |
| 485 | """Regression test for memory profiling with ``cap_pixels_per_frame``. |
| 486 | |
| 487 | With the HF per-frame pixel cap enabled (transformers#48071) and a |
| 488 | raised video budget, a 2-frame profiling dummy would be processed at |
| 489 | only 2 * cap pixels, underestimating the largest possible video (a |
| 490 | fully sampled one still fills the whole ``longest_edge`` budget). The |
| 491 | dummy builder must spread the budget over enough frames that the cap |
| 492 | is not binding. |
| 493 | """ |
| 494 | # 16 capped frames' worth of budget on top of the default per-frame |
| 495 | # ceiling (max_video_tokens=768 at patch 16, merge 2 -> 786,432 |
| 496 | # pixels per frame). |
| 497 | per_frame_cap = 768 * (16 * 2) ** 2 |
| 498 | budget = per_frame_cap * 16 |
| 499 | size = {"longest_edge": budget, "shortest_edge": 4096} |
| 500 | |
| 501 | capped_ctx = build_model_context( |
| 502 | model_id, |
| 503 | mm_processor_kwargs={ |
| 504 | "videos_kwargs": { |
| 505 | "size": size, |
| 506 | "cap_pixels_per_frame": True, |
| 507 | }, |
| 508 | }, |
| 509 | limit_mm_per_prompt={"image": 0, "video": 1}, |
| 510 | ) |
| 511 | capped = MULTIMODAL_REGISTRY.create_processor(capped_ctx.model_config) |
| 512 | capped_dummy = capped.dummy_inputs.get_dummy_mm_data( |
| 513 | 1024, {"video": 1}, MultiModalDummyOptions() |
| 514 | ) |
| 515 | capped_frames = capped_dummy["video"][0][0].shape[0] |
| 516 | assert capped_frames == 16, ( |
| 517 | f"Expected the dummy to spread the budget over 16 frames, got {capped_frames}" |
| 518 | ) |
| 519 | |
| 520 | uncapped_ctx = build_model_context( |
| 521 | model_id, |
| 522 | mm_processor_kwargs={"videos_kwargs": {"size": size}}, |
| 523 | limit_mm_per_prompt={"image": 0, "video": 1}, |
| 524 | ) |
| 525 | uncapped = MULTIMODAL_REGISTRY.create_processor(uncapped_ctx.model_config) |
| 526 | uncapped_dummy = uncapped.dummy_inputs.get_dummy_mm_data( |
| 527 | 1024, {"video": 1}, MultiModalDummyOptions() |
| 528 | ) |
| 529 | uncapped_frames = uncapped_dummy["video"][0][0].shape[0] |
| 530 | assert uncapped_frames == 2, ( |
| 531 | f"Expected the uncapped dummy to keep 2 frames, got {uncapped_frames}" |
| 532 | ) |