[Model] Honor cap_pixels_per_frame in Qwen3-VL memory profiling (#54380)
Signed-off-by: Devon Krisman <git@krisman.dev> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Devon Krisman committed
Aug 30, 2026 at 04:07 UTC
4f78a8fdd0b06421f0cedf1ef23220b76a527de7
2 files changed
+66
-1
tests/models/multimodal/processing/test_qwen3_vl.py
+43
@@ -184,3 +184,46 @@ def test_processor_multi_video_list_kwargs(
184
assert len(video_phs) == 2, (
185
f"Expected exactly 2 video placeholders, got {len(video_phs)}"
186
)
187
+
188
+
189
+@pytest.mark.parametrize("model_id", [MODEL_ID])
190
+def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> None:
191
+ """Regression test for memory profiling with ``cap_pixels_per_frame``.
192
+
193
+ With the HF per-frame pixel cap enabled (transformers#48071) and a
194
+ raised video budget, a 2-frame profiling dummy would be processed at
195
+ only 2 * cap pixels, underestimating the largest possible video (a
196
+ fully sampled one still fills the whole ``longest_edge`` budget). The
197
+ dummy builder must spread the budget over enough frames that the cap
198
+ is not binding.
199
+ """
200
+ # 16 capped frames' worth of budget on top of the default per-frame
201
+ # ceiling (max_video_tokens=768 at patch 16, merge 2 -> 786,432
202
+ # pixels per frame).
203
+ per_frame_cap = 768 * (16 * 2) ** 2
204
+ budget = per_frame_cap * 16
205
+ size = {"longest_edge": budget, "shortest_edge": 4096}
206
+
207
+ capped_ctx = build_model_context(
208
+ model_id,
209
+ mm_processor_kwargs={"size": size, "cap_pixels_per_frame": True},
210
+ limit_mm_per_prompt={"image": 0, "video": 1},
211
+ )
212
+ capped = MULTIMODAL_REGISTRY.create_processor(capped_ctx.model_config)
213
+ capped_dummy = capped.dummy_inputs.get_dummy_mm_data(1024, {"video": 1}, {})
214
+ capped_frames = capped_dummy["video"][0][0].shape[0]
215
+ assert capped_frames == 16, (
216
+ f"Expected the dummy to spread the budget over 16 frames, got {capped_frames}"
217
+ )
218
+
219
+ uncapped_ctx = build_model_context(
220
+ model_id,
221
+ mm_processor_kwargs={"size": size},
222
+ limit_mm_per_prompt={"image": 0, "video": 1},
223
+ )
224
+ uncapped = MULTIMODAL_REGISTRY.create_processor(uncapped_ctx.model_config)
225
+ uncapped_dummy = uncapped.dummy_inputs.get_dummy_mm_data(1024, {"video": 1}, {})
226
+ uncapped_frames = uncapped_dummy["video"][0][0].shape[0]
227
+ assert uncapped_frames == 2, (
228
+ f"Expected the uncapped dummy to keep 2 frames, got {uncapped_frames}"
229
+ )
vllm/model_executor/models/qwen3_vl.py
+23
-1
@@ -112,7 +112,7 @@ from vllm.tokenizers.registry import cached_tokenizer_from_config
112
from vllm.triton_utils import HAS_TRITON, tl, triton
113
from vllm.utils.cache import LRUCache
114
from vllm.utils.collection_utils import is_list_of
115
-from vllm.utils.math_utils import round_up
115
+from vllm.utils.math_utils import cdiv, round_up
116
from vllm.utils.torch_utils import PIN_MEMORY
117
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
118
@@ -1146,6 +1146,28 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
1146
# video_max_pixels contains the temporal compression factor,
1147
# so we divide by 2 to get the maximum number of image pixels.
1148
video_max_pixels = video_size["longest_edge"]
1149
+
1150
+ # With the HF processor's per-frame pixel cap enabled
1151
+ # (cap_pixels_per_frame, huggingface/transformers#48071), a 2-frame
1152
+ # dummy is processed at only 2 * cap pixels, so memory profiling
1153
+ # underestimates the largest possible item: a fully sampled video
1154
+ # still reaches the whole longest_edge budget. Spread the budget
1155
+ # over enough frames that the cap is not binding for the dummy.
1156
+ # This takes precedence over a shrinking num_frames override, which
1157
+ # would otherwise reintroduce the under-profiling.
1158
+ if mm_kwargs.get("cap_pixels_per_frame"):
1159
+ max_video_tokens = mm_kwargs.get(
1160
+ "max_video_tokens",
1161
+ getattr(video_processor, "max_video_tokens", 768),
1162
+ )
1163
+ patch_size = mm_kwargs.get("patch_size", video_processor.patch_size)
1164
+ merge_size = mm_kwargs.get("merge_size", video_processor.merge_size)
1165
+ per_frame_cap = max_video_tokens * (patch_size * merge_size) ** 2
1166
+ target_num_frames = max(
1167
+ target_num_frames,
1168
+ min(video_processor.max_frames, cdiv(video_max_pixels, per_frame_cap)),
1169
+ )
1170
+
1171
target_video_width, target_video_height = (
1172
self.info.get_image_size_with_most_features(
1173
max_pixels=video_max_pixels // temporal_patch_size