@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 295a0b8d64

[Bugfix][Multimodal] Fix flat/scoped mm_processor_kwargs merge and resolution (#56372)

Signed-off-by: Daniel M. García-Ocaña Hernández <danielgarciaocana@gmail.com>

Daniel M. García-Ocaña Hernández committed Sep 29, 2026 at 05:58 UTC 295a0b8d6456b765edc5b135c6d2b5c03bccbb90
11 files changed +1601 -187
tests/config/test_multimodal_config.py
+244
@@ -188,6 +188,250 @@ def test_mm_processor_device_type_normalizes(device: object, expected: str | Non
188 assert config.get_mm_processor_device_type() == expected
189
190
191 +def test_merge_mm_processor_kwargs_scoped_precedence():
192 + configured = {
193 + "size": {
194 + "shortest_edge": 64,
195 + "longest_edge": 512,
196 + },
197 + "do_resize": True,
198 + "padding": False,
199 + "images_kwargs": {
200 + "size": {"longest_edge": 1024},
201 + "do_resize": False,
202 + },
203 + "videos_kwargs": {
204 + "size": {"longest_edge": 2048},
205 + },
206 + }
207 + inference = {
208 + "size": {"shortest_edge": 128},
209 + "do_resize": True,
210 + "images_kwargs": {
211 + "size": {"shortest_edge": 256},
212 + },
213 + }
214 +
215 + config = MultiModalConfig(
216 + mm_processor_kwargs=configured,
217 + mm_device_do_normalize=False,
218 + )
219 +
220 + assert config.merge_mm_processor_kwargs(inference) == {
221 + "size": {
222 + "shortest_edge": 128,
223 + "longest_edge": 512,
224 + },
225 + "do_resize": True,
226 + "padding": False,
227 + "images_kwargs": {
228 + "size": {
229 + "shortest_edge": 256,
230 + "longest_edge": 1024,
231 + },
232 + "do_resize": True,
233 + },
234 + "videos_kwargs": {
235 + "size": {
236 + "shortest_edge": 128,
237 + "longest_edge": 2048,
238 + },
239 + },
240 + }
241 +
242 +
243 +def test_merge_mm_processor_kwargs_mapping_vs_replacement():
244 + config = MultiModalConfig(
245 + mm_processor_kwargs={
246 + "size": {"shortest_edge": 64},
247 + "images_kwargs": {"size": {"longest_edge": 1024}},
248 + },
249 + mm_device_do_normalize=False,
250 + )
251 +
252 + assert config.merge_mm_processor_kwargs({"size": 7}) == {
253 + "size": 7,
254 + "images_kwargs": {"size": 7},
255 + }
256 + assert config.merge_mm_processor_kwargs({"size": {}}) == {
257 + "size": {"shortest_edge": 64},
258 + "images_kwargs": {
259 + "size": {
260 + "shortest_edge": 64,
261 + "longest_edge": 1024,
262 + },
263 + },
264 + }
265 +
266 +
267 +_EMPTY_IMAGE_SCOPE_KWARGS: tuple[dict[str, object], ...] = (
268 + {},
269 + {"images_kwargs": None},
270 + {"images_kwargs": {}},
271 +)
272 +
273 +
274 +@pytest.mark.parametrize("configured", _EMPTY_IMAGE_SCOPE_KWARGS)
275 +@pytest.mark.parametrize("inference", _EMPTY_IMAGE_SCOPE_KWARGS)
276 +def test_merge_mm_processor_kwargs_empty_scopes_are_absent(
277 + configured: dict[str, object],
278 + inference: dict[str, object],
279 +):
280 + config = MultiModalConfig(
281 + mm_processor_kwargs=configured,
282 + mm_device_do_normalize=False,
283 + )
284 +
285 + assert config.merge_mm_processor_kwargs(inference) == {}
286 +
287 +
288 +@pytest.mark.parametrize("empty_inference", _EMPTY_IMAGE_SCOPE_KWARGS)
289 +def test_merge_mm_processor_kwargs_empty_inference_scope_preserves_config_merge(
290 + empty_inference: dict[str, object],
291 +):
292 + config = MultiModalConfig(
293 + mm_processor_kwargs={
294 + "images_kwargs": {
295 + "size": {
296 + "shortest_edge": 64,
297 + "longest_edge": 512,
298 + },
299 + },
300 + },
301 + mm_device_do_normalize=False,
302 + )
303 + inference = {
304 + "size": {"shortest_edge": 128},
305 + **empty_inference,
306 + }
307 +
308 + assert config.merge_mm_processor_kwargs(inference) == {
309 + "size": {"shortest_edge": 128},
310 + "images_kwargs": {
311 + "size": {
312 + "shortest_edge": 128,
313 + "longest_edge": 512,
314 + },
315 + },
316 + }
317 +
318 +
319 +@pytest.mark.parametrize("empty_config", _EMPTY_IMAGE_SCOPE_KWARGS)
320 +def test_merge_mm_processor_kwargs_empty_config_scope_allows_inference_merge(
321 + empty_config: dict[str, object],
322 +):
323 + config = MultiModalConfig(
324 + mm_processor_kwargs={
325 + "size": {"longest_edge": 512},
326 + **empty_config,
327 + },
328 + mm_device_do_normalize=False,
329 + )
330 +
331 + assert config.merge_mm_processor_kwargs(
332 + {
333 + "size": {"shortest_edge": 128},
334 + "images_kwargs": {"size": {"longest_edge": 1024}},
335 + }
336 + ) == {
337 + "size": {
338 + "shortest_edge": 128,
339 + "longest_edge": 512,
340 + },
341 + "images_kwargs": {
342 + "size": {
343 + "shortest_edge": 128,
344 + "longest_edge": 1024,
345 + },
346 + },
347 + }
348 +
349 +
350 +@pytest.mark.parametrize("non_mapping_scope", [False, 0, "", [], 123])
351 +def test_merge_mm_processor_kwargs_falsey_non_mapping_scopes_are_not_empty(
352 + non_mapping_scope: object,
353 +):
354 + config = MultiModalConfig(
355 + mm_processor_kwargs={"images_kwargs": non_mapping_scope},
356 + mm_device_do_normalize=False,
357 + )
358 +
359 + for empty_inference in _EMPTY_IMAGE_SCOPE_KWARGS:
360 + assert config.merge_mm_processor_kwargs(empty_inference) == {
361 + "images_kwargs": non_mapping_scope
362 + }
363 +
364 +
365 +@pytest.mark.parametrize(
366 + ("configured_scope", "inference_scope", "expected_scope"),
367 + [
368 + (123, {"size": {"shortest_edge": 128}}, {"size": {"shortest_edge": 128}}),
369 + ({"size": {"shortest_edge": 64}}, 123, 123),
370 + (123, 456, 456),
371 + ],
372 +)
373 +def test_merge_mm_processor_kwargs_non_mapping_scope_precedence(
374 + configured_scope: object,
375 + inference_scope: object,
376 + expected_scope: object,
377 +):
378 + config = MultiModalConfig(
379 + mm_processor_kwargs={"images_kwargs": configured_scope},
380 + mm_device_do_normalize=False,
381 + )
382 +
383 + merged = config.merge_mm_processor_kwargs({"images_kwargs": inference_scope})
384 +
385 + assert merged["images_kwargs"] == expected_scope
386 +
387 +
388 +def test_merge_mm_processor_kwargs_none_leaf_is_explicit():
389 + config = MultiModalConfig(mm_device_do_normalize=False)
390 +
391 + assert config.merge_mm_processor_kwargs({"images_kwargs": {"do_resize": None}}) == {
392 + "images_kwargs": {"do_resize": None}
393 + }
394 +
395 +
396 +def test_merge_mm_processor_kwargs_normalize_and_ownership():
397 + configured: dict[str, object] = {
398 + "images_kwargs": {
399 + "size": {"shortest_edge": 64},
400 + },
401 + }
402 + inference: dict[str, object] = {
403 + "do_normalize": True,
404 + "do_rescale": True,
405 + "images_kwargs": {
406 + "size": {"longest_edge": 1024},
407 + },
408 + }
409 + configured_before = copy.deepcopy(configured)
410 + inference_before = copy.deepcopy(inference)
411 +
412 + config = MultiModalConfig(
413 + mm_processor_kwargs=configured,
414 + mm_device_do_normalize=True,
415 + )
416 + merged = config.merge_mm_processor_kwargs(inference)
417 +
418 + assert merged["do_normalize"] is False
419 + assert merged["do_rescale"] is False
420 + assert configured == configured_before
421 + assert inference == inference_before
422 + assert config.mm_processor_kwargs == configured_before
423 +
424 + images_kwargs = merged["images_kwargs"]
425 + assert isinstance(images_kwargs, dict)
426 + size = images_kwargs["size"]
427 + assert isinstance(size, dict)
428 + size["shortest_edge"] = 999
429 +
430 + assert configured == configured_before
431 + assert inference == inference_before
432 + assert config.mm_processor_kwargs == configured_before
433 +
434 +
435 def _validate_mm_processor_device(*, device: str, ec_role: ECRole | None) -> None:
436 ec_config = (
437 None
tests/models/multimodal/processing/test_gemma4.py
+9 -16
@@ -202,19 +202,19 @@ def test_get_prompt_updates_respects_nested_max_soft_tokens(model_id: str):
202
203
204 @pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
205 -@pytest.mark.parametrize("kwargs_on_init", [False, True])
205 @pytest.mark.parametrize(
206 "image_kwargs", [{"rescale_factor": 1 / 127.5}, {"max_soft_tokens": 560}]
207 )
208 @pytest.mark.parametrize("video_uuid", [None, "same-video"])
209 def test_video_cache_is_independent_of_image_kwargs(
210 model_id: str,
212 - kwargs_on_init: bool,
211 image_kwargs: dict[str, object],
212 video_uuid: str | None,
213 ):
216 - """Image overrides must not change video frames or depend on cache warmth."""
217 - kwargs = {"images_kwargs": image_kwargs}
214 + """Request image overrides must not change video frames or make results
215 + depend on cache warmth.
216 + """
217 + request_kwargs = {"images_kwargs": image_kwargs}
218 ctx = build_model_context(
219 model_id,
220 limit_mm_per_prompt={"image": 1, "video": 1},
@@ -239,18 +239,11 @@ def test_video_cache_is_independent_of_image_kwargs(
239 cache=cache,
240 )
241
242 - baseline = process({}, cache=None)
243 - if kwargs_on_init:
244 - ctx = build_model_context(
245 - model_id,
246 - mm_processor_kwargs=kwargs,
247 - limit_mm_per_prompt={"image": 1, "video": 1},
248 - mm_processor_cache_gb=1,
249 - )
250 - cache = MultiModalProcessorOnlyCache(ctx.model_config)
251 - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
252 - request_kwargs = {} if kwargs_on_init else kwargs
253 - process(request_kwargs, cache=cache)
242 + # Populate the video cache without image overrides, then change only
243 + # images_kwargs on the next request. Since images_kwargs are excluded from
244 + # the video hash, and must not affect the processed video, the cached result
245 + # must match a fresh recomputation.
246 + baseline = process({}, cache=cache)
247 cached = process(request_kwargs, cache=cache)
248 cache.clear_cache()
249 fresh = process(request_kwargs, cache=cache)
tests/models/multimodal/processing/test_mistral3.py
+39
@@ -100,6 +100,11 @@ def _expected_placeholder_tokens_per_image(
100 [
101 ({}, (448, 448), 256),
102 ({"size": {"longest_edge": 1008}}, (1540, 1540), 1296),
103 + (
104 + {"images_kwargs": {"size": {"longest_edge": 1008}}},
105 + (1540, 1540),
106 + 1296,
107 + ),
108 ({"size": {"longest_edge": 1288}}, (1536, 1187), 1656),
109 ({"size": {"longest_edge": 1008}}, (29, 29), 4),
110 ({"size": {"longest_edge": 1000}}, (1540, 1700), 1188),
@@ -148,6 +153,40 @@ def test_processor_size_override(
153 assert prompt_update_tokens == hf_placeholder_tokens
154
155
156 +def test_scoped_request_size_overrides_configured_flat_size():
157 + ctx = build_model_context(
158 + _MODEL_ID,
159 + mm_processor_kwargs={"size": {"longest_edge": 1008}},
160 + limit_mm_per_prompt={"image": 1},
161 + model_config_kwargs=_MODEL_CONFIG_KWARGS,
162 + )
163 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
164 + hf_processor = AutoProcessor.from_pretrained(
165 + _MODEL_ID,
166 + fix_mistral_regex=True,
167 + )
168 +
169 + hf_processor_mm_kwargs: dict[str, object] = {
170 + "images_kwargs": {"size": {"longest_edge": 1288}}
171 + }
172 + image = Image.new("RGB", (1536, 1187), color=(127, 127, 127))
173 +
174 + merged_mm_kwargs = processor.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
175 + pixel_values, hf_placeholder_tokens = _process_images_with_hf(
176 + hf_processor, [image], merged_mm_kwargs
177 + )
178 + prompt_update_tokens = _placeholder_tokens_from_prompt_updates(
179 + processor, [image], pixel_values, hf_processor_mm_kwargs
180 + )
181 +
182 + expected_from_pixel_values = _expected_placeholder_tokens_per_image(
183 + hf_processor, pixel_values[0]
184 + )
185 + assert expected_from_pixel_values == 1656
186 + assert hf_placeholder_tokens.count(hf_processor.image_token_id) == 1656
187 + assert prompt_update_tokens == hf_placeholder_tokens
188 +
189 +
190 def test_lightonocr_keeps_vision_config_image_size():
191 ctx = build_model_context(
192 _LIGHTON_MODEL_ID,
tests/models/multimodal/processing/test_qwen2_vl.py
+224
@@ -1,6 +1,8 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 +from copy import deepcopy
5 +from types import SimpleNamespace
6 from typing import Any
7
8 import numpy as np
@@ -8,16 +10,238 @@ import pytest
10 import torch
11 from PIL import Image
12
13 +from vllm.config.multimodal import MultiModalConfig
14 from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
15 +from vllm.model_executor.models.qwen2_vl import Qwen2VLProcessingInfo
16 from vllm.multimodal import MULTIMODAL_REGISTRY
17 from vllm.multimodal.cache import MultiModalProcessorOnlyCache
18 from vllm.multimodal.inputs import batched_tensors_equal
19 +from vllm.multimodal.processing.context import InputProcessingContext
20 from vllm.platforms import current_platform
21
22 from ....conftest import ImageTestAssets
23 from ...utils import build_model_context
24
25
26 +@pytest.mark.parametrize(
27 + ("mm_kwargs", "expected"),
28 + [
29 + (
30 + {"size": {"shortest_edge": 64, "longest_edge": 1024}},
31 + {
32 + "size": {"shortest_edge": 64, "longest_edge": 1024},
33 + "min_pixels": 64,
34 + "max_pixels": 1024,
35 + },
36 + ),
37 + (
38 + {"min_pixels": 64, "max_pixels": 1024},
39 + {
40 + "min_pixels": 64,
41 + "max_pixels": 1024,
42 + "size": {"shortest_edge": 64, "longest_edge": 1024},
43 + },
44 + ),
45 + (
46 + {
47 + "size": {"shortest_edge": 64, "longest_edge": 1024},
48 + "min_pixels": 128,
49 + "max_pixels": 2048,
50 + },
51 + {
52 + "size": {"shortest_edge": 64, "longest_edge": 1024},
53 + "min_pixels": 128,
54 + "max_pixels": 2048,
55 + },
56 + ),
57 + (
58 + {
59 + "min_pixels": None,
60 + "size": {"shortest_edge": 0},
61 + "images_kwargs": {"max_pixels": 4096},
62 + "videos_kwargs": {"size": {"longest_edge": 8192}},
63 + },
64 + {
65 + "min_pixels": None,
66 + "size": {"shortest_edge": 0},
67 + "images_kwargs": {
68 + "max_pixels": 4096,
69 + "size": {"longest_edge": 4096},
70 + },
71 + "videos_kwargs": {
72 + "size": {"longest_edge": 8192},
73 + "max_pixels": 8192,
74 + },
75 + },
76 + ),
77 + (
78 + {"min_pixels": 0},
79 + {"min_pixels": 0, "size": {"shortest_edge": 0}},
80 + ),
81 + (
82 + {"size": None, "min_pixels": 64},
83 + {"size": {"shortest_edge": 64}, "min_pixels": 64},
84 + ),
85 + ],
86 +)
87 +def test_complete_mm_processor_size_aliases(
88 + mm_kwargs: dict[str, object],
89 + expected: dict[str, object],
90 +) -> None:
91 + original = deepcopy(mm_kwargs)
92 +
93 + actual = Qwen2VLProcessingInfo._complete_mm_processor_size_aliases(mm_kwargs)
94 +
95 + assert actual == expected
96 + assert mm_kwargs == original
97 +
98 +
99 +def test_merge_and_resolve_mm_processor_kwargs_preserves_request_alias_precedence(
100 + monkeypatch: pytest.MonkeyPatch,
101 +) -> None:
102 + mm_config = MultiModalConfig(
103 + mm_processor_kwargs={
104 + "images_kwargs": {"min_pixels": 64},
105 + },
106 + mm_device_do_normalize=False,
107 + )
108 + model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config)
109 + ctx = InputProcessingContext(model_config, tokenizer=None) # type: ignore[arg-type]
110 + info = Qwen2VLProcessingInfo(ctx)
111 + monkeypatch.setattr(
112 + info,
113 + "get_supported_mm_processor_kwargs",
114 + lambda: {"images_kwargs": {"size", "min_pixels"}},
115 + )
116 +
117 + assert info._merge_and_resolve_mm_processor_kwargs(
118 + {"size": {"shortest_edge": 128}}
119 + ) == {
120 + "images_kwargs": {
121 + "size": {"shortest_edge": 128},
122 + "min_pixels": 128,
123 + },
124 + }
125 +
126 +
127 +@pytest.mark.parametrize(
128 + ("mm_kwargs", "default_size", "expected"),
129 + [
130 + (
131 + {},
132 + {"shortest_edge": 32, "longest_edge": 1024},
133 + {"shortest_edge": 32, "longest_edge": 1024},
134 + ),
135 + (
136 + {"size": {"shortest_edge": 64}},
137 + {"shortest_edge": 32, "longest_edge": 1024},
138 + {"shortest_edge": 64, "longest_edge": 1024},
139 + ),
140 + (
141 + {
142 + "size": {"shortest_edge": 64, "longest_edge": 2048},
143 + "min_pixels": 128,
144 + "max_pixels": 4096,
145 + },
146 + {"shortest_edge": 32, "longest_edge": 1024},
147 + {"shortest_edge": 128, "longest_edge": 4096},
148 + ),
149 + (
150 + {
151 + "size": {"shortest_edge": 64},
152 + "max_pixels": 4096,
153 + },
154 + None,
155 + {"shortest_edge": 64, "longest_edge": 4096},
156 + ),
157 + (
158 + {
159 + "size": {"shortest_edge": 64, "longest_edge": 2048},
160 + "min_pixels": None,
161 + "max_pixels": 0,
162 + },
163 + {"shortest_edge": 32, "longest_edge": 1024},
164 + {"shortest_edge": 64, "longest_edge": 0},
165 + ),
166 + ],
167 +)
168 +def test_get_vision_size(
169 + mm_kwargs: dict[str, object],
170 + default_size: dict[str, object] | None,
171 + expected: dict[str, object],
172 +) -> None:
173 + original_mm_kwargs = {
174 + key: dict(value) if isinstance(value, dict) else value
175 + for key, value in mm_kwargs.items()
176 + }
177 + original_default_size = dict(default_size) if default_size is not None else None
178 +
179 + actual = Qwen2VLProcessingInfo._get_vision_size(
180 + mm_kwargs,
181 + default_size=default_size,
182 + )
183 +
184 + assert actual == expected
185 + assert mm_kwargs == original_mm_kwargs
186 + assert default_size == original_default_size
187 +
188 +
189 +@pytest.mark.parametrize(
190 + ("modality", "scope"),
191 + [
192 + ("image", "images_kwargs"),
193 + ("video", "videos_kwargs"),
194 + ],
195 +)
196 +def test_get_vision_info_reads_merged_modality_scope(
197 + monkeypatch: pytest.MonkeyPatch,
198 + modality: str,
199 + scope: str,
200 +) -> None:
201 + vision_config = SimpleNamespace(
202 + patch_size=14,
203 + spatial_merge_size=2,
204 + temporal_patch_size=2,
205 + )
206 + ctx = SimpleNamespace(
207 + get_hf_config=lambda *_: SimpleNamespace(vision_config=vision_config)
208 + )
209 + info = Qwen2VLProcessingInfo(ctx)
210 + merged = {
211 + "images_kwargs": {"size": {"shortest_edge": 111}},
212 + "videos_kwargs": {"size": {"shortest_edge": 222}},
213 + }
214 + monkeypatch.setattr(
215 + info,
216 + "_merge_and_resolve_mm_processor_kwargs",
217 + lambda _: merged,
218 + )
219 +
220 + seen: list[dict[str, object]] = []
221 +
222 + def get_size(
223 + mm_kwargs: dict[str, object],
224 + default_size: dict[str, object] | None = None,
225 + ) -> dict[str, object]:
226 + seen.append(mm_kwargs)
227 + return dict(default_size or {})
228 +
229 + monkeypatch.setattr(info, "_get_vision_size", get_size)
230 + image_processor = SimpleNamespace(size={"shortest_edge": 32, "longest_edge": 1024})
231 +
232 + info._get_vision_info(
233 + image_width=28,
234 + image_height=28,
235 + num_frames=2,
236 + do_resize=False,
237 + image_processor=image_processor,
238 + mm_kwargs={},
239 + modality=modality, # type: ignore[arg-type]
240 + )
241 +
242 + assert seen == [merged[scope]]
243 +
244 +
245 def test_jina_vl_processing_order() -> None:
246 """Jina's document-first prompt keeps cached features and hashes aligned."""
247 ctx = build_model_context(
tests/models/multimodal/processing/test_qwen3_vl.py
+185 -16
@@ -1,19 +1,21 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -"""Regression tests for Qwen3-VL processor.
4 -
5 -Covers the fix for num_frames-based timestamp calculation
6 -(issue vllm-project/vllm#35909).
7 -"""
3 +"""Regression tests for Qwen3-VL multimodal processing."""
4
5 +from copy import deepcopy
6 +from types import SimpleNamespace
7 from typing import Any
8
9 import numpy as np
10 import pytest
11 +import torch
12 +from transformers.models.qwen3_vl import Qwen3VLVideoProcessor
13
14 from vllm.config import ModelConfig
15 from vllm.config.multimodal import MultiModalDummyOptions
16 +from vllm.model_executor.models.qwen3_vl import Qwen3VLProcessingInfo
17 from vllm.multimodal import MULTIMODAL_REGISTRY
18 +from vllm.multimodal.processing.context import BaseProcessingInfo
19
20 from ...registry import HF_EXAMPLE_MODELS
21 from ...utils import build_model_context
@@ -21,6 +23,32 @@ from ...utils import build_model_context
23 MODEL_ID = "Qwen/Qwen3-VL-4B-Instruct"
24
25
26 +def test_qwen3_video_routing_schema_adds_vllm_compatibility_kwargs(
27 + monkeypatch: pytest.MonkeyPatch,
28 +) -> None:
29 + native_video_kwargs = set(Qwen3VLVideoProcessor.valid_kwargs.__annotations__)
30 + assert "size" in native_video_kwargs
31 + assert "min_pixels" not in native_video_kwargs
32 + assert "max_pixels" not in native_video_kwargs
33 +
34 + info = Qwen3VLProcessingInfo(SimpleNamespace())
35 + parent_supported = {
36 + "text_kwargs": {"padding"},
37 + "images_kwargs": {"size", "min_pixels", "max_pixels"},
38 + "videos_kwargs": set(native_video_kwargs),
39 + }
40 + monkeypatch.setattr(
41 + BaseProcessingInfo,
42 + "get_supported_mm_processor_kwargs",
43 + lambda self: {scope: set(kwargs) for scope, kwargs in parent_supported.items()},
44 + )
45 +
46 + supported = info.supported_mm_processor_kwargs
47 +
48 + assert {"min_pixels", "max_pixels"} <= supported["videos_kwargs"]
49 + assert parent_supported["videos_kwargs"] == native_video_kwargs
50 +
51 +
52 def _build_video_mm_data(
53 num_frames: int,
54 width: int = 128,
@@ -45,6 +73,122 @@ def _build_video_mm_data(
73 return {"video": [(video, metadata)]}
74
75
76 +@pytest.mark.parametrize("model_id", [MODEL_ID])
77 +def test_processor_partial_scoped_image_size(model_id: str) -> None:
78 + """Partial image ``size`` must be completed and applied before the HF call.
79 +
80 + Keep video compatibility and per-video kwargs in an image-only call as well:
81 + Qwen3 must sanitize the video scope before HF validates it. Regression for
82 + #56363.
83 + """
84 + ctx = build_model_context(
85 + model_id,
86 + limit_mm_per_prompt={"image": 1, "video": 0},
87 + )
88 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
89 +
90 + image = np.zeros((128, 128, 3), dtype=np.uint8)
91 + min_pixels = 256 * 32 * 32
92 + default_size = dict(processor.info.get_image_processor().size)
93 + complete_size = default_size | {"shortest_edge": min_pixels}
94 +
95 + def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
96 + mm_items = processor.info.parse_mm_data({"image": [image]})
97 + return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
98 + "image_grid_thw"
99 + ]
100 +
101 + stock_grid = process({})
102 + complete_grid = process({"images_kwargs": {"size": complete_size}})
103 + partial_grid = process(
104 + {
105 + "images_kwargs": {
106 + "size": {"shortest_edge": min_pixels},
107 + },
108 + "videos_kwargs": {
109 + "min_pixels": 4096,
110 + "max_pixels": 469_762_048,
111 + "fps": [2.0],
112 + "num_frames": [8],
113 + },
114 + }
115 + )
116 +
117 + assert torch.equal(partial_grid, complete_grid)
118 + assert not torch.equal(partial_grid, stock_grid)
119 +
120 +
121 +@pytest.mark.parametrize("model_id", [MODEL_ID])
122 +def test_processor_partial_scoped_video_size(model_id: str) -> None:
123 + """Partial scoped video ``size`` must be completed and applied before HF."""
124 + ctx = build_model_context(
125 + model_id,
126 + limit_mm_per_prompt={"image": 0, "video": 1},
127 + )
128 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
129 +
130 + min_pixels = 512 * 32 * 32
131 + default_size = dict(processor.info.get_video_processor().size)
132 + complete_size = default_size | {"shortest_edge": min_pixels}
133 +
134 + def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
135 + mm_items = processor.info.parse_mm_data(_build_video_mm_data(num_frames=8))
136 + return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
137 + "video_grid_thw"
138 + ]
139 +
140 + stock_grid = process({})
141 + complete_grid = process({"videos_kwargs": {"size": complete_size}})
142 + partial_grid = process(
143 + {
144 + "videos_kwargs": {
145 + "size": {"shortest_edge": min_pixels},
146 + },
147 + }
148 + )
149 +
150 + assert torch.equal(partial_grid, complete_grid)
151 + assert not torch.equal(partial_grid, stock_grid)
152 +
153 +
154 +@pytest.mark.parametrize("model_id", [MODEL_ID])
155 +def test_processor_flat_video_pixel_aliases_match_scoped_size(model_id: str) -> None:
156 + """Flat min/max pixel aliases keep their historical Qwen3 video effect."""
157 + ctx = build_model_context(
158 + model_id,
159 + limit_mm_per_prompt={"image": 0, "video": 1},
160 + )
161 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
162 + pixel_budget = 512 * 32 * 32
163 +
164 + def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
165 + mm_items = processor.info.parse_mm_data(_build_video_mm_data(num_frames=8))
166 + return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
167 + "video_grid_thw"
168 + ]
169 +
170 + stock_grid = process({})
171 + scoped_grid = process(
172 + {
173 + "videos_kwargs": {
174 + "size": {
175 + "shortest_edge": pixel_budget,
176 + "longest_edge": pixel_budget,
177 + },
178 + },
179 + }
180 + )
181 + flat_grid = process(
182 + {
183 + "min_pixels": pixel_budget,
184 + "max_pixels": pixel_budget,
185 + }
186 + )
187 +
188 + assert torch.equal(flat_grid, scoped_grid)
189 + assert not torch.equal(flat_grid, stock_grid)
190 +
191 +
192 @pytest.mark.parametrize("model_id", [MODEL_ID])
193 @pytest.mark.parametrize(
194 "num_frames",
@@ -189,20 +333,21 @@ def test_processor_kwargs_videos_kwargs_does_not_leak_into_image_budget(
333
334 @pytest.mark.parametrize("model_id", [MODEL_ID])
335 @pytest.mark.parametrize(
192 - "hf_mm_kwargs",
193 - [{"num_frames": [8, 16]}, {"fps": [2.0, 4.0]}],
336 + ("hf_mm_kwargs", "scalar_kwargs"),
337 + [
338 + ({"num_frames": [8, 16]}, [{"num_frames": 8}, {"num_frames": 16}]),
339 + ({"fps": [4.0, 12.0]}, [{"fps": 4.0}, {"fps": 12.0}]),
340 + ],
341 )
342 def test_processor_multi_video_list_kwargs(
343 model_id: str,
344 hf_mm_kwargs: dict[str, Any],
345 + scalar_kwargs: list[dict[str, Any]],
346 ) -> None:
199 - """Regression test: a multi-video request with list-valued per-video
200 - ``mm_processor_kwargs`` (one ``fps``/``num_frames`` per video) must not
201 - crash.
347 + """List-valued video kwargs must be scalarized per item without mutation.
348
203 - Before the fix, ``_apply_hf_processor_main`` copied the whole kwargs to every
204 - video without slicing, so ``_get_video_second_idx`` received the list
205 - where a scalar was expected and raised ``TypeError``.
349 + Each video in the multi-video request must match processing the same input
350 + independently with its corresponding scalar ``fps``/``num_frames`` value.
351 """
352 ctx = build_model_context(
353 model_id,
@@ -216,17 +361,36 @@ def test_processor_multi_video_list_kwargs(
361 )
362 mm_data = {
363 "video": [
219 - _build_video_mm_data(num_frames=16)["video"][0],
364 + _build_video_mm_data(num_frames=32)["video"][0],
365 _build_video_mm_data(num_frames=32)["video"][0],
366 ]
367 }
368
369 + hf_mm_kwargs_before = deepcopy(hf_mm_kwargs)
370 processed = processor(
371 prompt,
372 mm_items=processor.info.parse_mm_data(mm_data),
373 hf_processor_mm_kwargs=hf_mm_kwargs,
374 )
375
376 + def process_single(hf_kwargs: dict[str, Any]) -> torch.Tensor:
377 + single_prompt = "<|vision_start|><|video_pad|><|vision_end|>"
378 + single_mm_data = _build_video_mm_data(num_frames=32)
379 + single = processor(
380 + single_prompt,
381 + mm_items=processor.info.parse_mm_data(single_mm_data),
382 + hf_processor_mm_kwargs=hf_kwargs,
383 + )
384 + return single["mm_kwargs"].get_data()["video_grid_thw"][0]
385 +
386 + expected_grids = [process_single(kwargs) for kwargs in scalar_kwargs]
387 + multi_grids = processed["mm_kwargs"].get_data()["video_grid_thw"]
388 +
389 + assert not torch.equal(expected_grids[0], expected_grids[1])
390 + assert torch.equal(multi_grids[0], expected_grids[0])
391 + assert torch.equal(multi_grids[1], expected_grids[1])
392 + assert hf_mm_kwargs == hf_mm_kwargs_before
393 +
394 video_phs = processed["mm_placeholders"].get("video", [])
395 assert len(video_phs) == 2, (
396 f"Expected exactly 2 video placeholders, got {len(video_phs)}"
@@ -336,7 +500,12 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non
500
501 capped_ctx = build_model_context(
502 model_id,
339 - mm_processor_kwargs={"size": size, "cap_pixels_per_frame": True},
503 + mm_processor_kwargs={
504 + "videos_kwargs": {
505 + "size": size,
506 + "cap_pixels_per_frame": True,
507 + },
508 + },
509 limit_mm_per_prompt={"image": 0, "video": 1},
510 )
511 capped = MULTIMODAL_REGISTRY.create_processor(capped_ctx.model_config)
@@ -350,7 +519,7 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non
519
520 uncapped_ctx = build_model_context(
521 model_id,
353 - mm_processor_kwargs={"size": size},
522 + mm_processor_kwargs={"videos_kwargs": {"size": size}},
523 limit_mm_per_prompt={"image": 0, "video": 1},
524 )
525 uncapped = MULTIMODAL_REGISTRY.create_processor(uncapped_ctx.model_config)
tests/multimodal/test_processing.py
+291 -39
@@ -3,18 +3,23 @@
3
4 import time
5 from contextlib import nullcontext
6 +from copy import deepcopy
7 +from types import SimpleNamespace
8 +from typing import TypedDict
9
10 import numpy as np
11 import pytest
12
13 from vllm.config import ModelConfig, SchedulerConfig
14 +from vllm.config.multimodal import MultiModalConfig
15 from vllm.exceptions import VLLMValidationError
16 from vllm.multimodal import MULTIMODAL_REGISTRY
17 from vllm.multimodal.hasher import MultiModalHasher
18 from vllm.multimodal.parse import MultiModalDataParser
19 from vllm.multimodal.processing.context import (
20 + BaseProcessingInfo,
21 InputProcessingContext,
17 - overlay_modality_mm_kwargs,
22 + _resolve_mm_processor_kwargs,
23 )
24 from vllm.multimodal.processing.inputs import ProcessorInputs
25 from vllm.multimodal.processing.processor import (
@@ -1207,58 +1212,305 @@ def test_apply_prompt_updates_falls_back_with_index_targets():
1212 assert [p.tokens for p in placeholders["image"]] == [[200, 201], [9]]
1213
1214
1210 -@pytest.mark.skip_global_cleanup
1211 -def test_overlay_modality_mm_kwargs_scoped_video_does_not_leak_to_image():
1212 - """HF-style videos_kwargs must overlay only when modality is video."""
1213 - video_size = {"longest_edge": 469762048, "shortest_edge": 4096}
1214 - kwargs = {"videos_kwargs": {"size": video_size}}
1215 +class _TextProcessorKwargs(TypedDict, total=False):
1216 + padding: bool
1217
1216 - assert overlay_modality_mm_kwargs(kwargs, None) == kwargs
1217 - assert "size" not in overlay_modality_mm_kwargs(kwargs, "image")
1218 - assert overlay_modality_mm_kwargs(kwargs, "video")["size"] == video_size
1218
1219 +class _AudioProcessorKwargs(TypedDict, total=False):
1220 + sampling_rate: int
1221
1221 -@pytest.mark.skip_global_cleanup
1222 -def test_overlay_modality_mm_kwargs_flat_size_stays_shared():
1223 - """A flat size override keeps the current shared-namespace behavior."""
1224 - size = {"longest_edge": 469762048, "shortest_edge": 4096}
1225 - kwargs = {"size": size}
1222
1227 - for modality in (None, "image", "video"):
1228 - assert overlay_modality_mm_kwargs(kwargs, modality)["size"] == size
1223 +class _ProcessorKwargs(TypedDict, total=False):
1224 + text_kwargs: _TextProcessorKwargs
1225 + audio_kwargs: _AudioProcessorKwargs
1226
1227
1231 -@pytest.mark.skip_global_cleanup
1232 -def test_overlay_modality_mm_kwargs_scoped_wins_over_flat_for_modality():
1233 - """A nested videos_kwargs size wins over a flat size for video reads."""
1228 +class _ImageProcessorKwargs(TypedDict, total=False):
1229 + size: dict[str, int]
1230 + min_pixels: int
1231 +
1232 +
1233 +class _VideoProcessorKwargs(TypedDict, total=False):
1234 + size: dict[str, int]
1235 + fps: float
1236 +
1237 +
1238 +@pytest.mark.parametrize(
1239 + ("supported_mm_limits", "processor", "expected"),
1240 + [
1241 + (
1242 + {"image": None, "video": None},
1243 + SimpleNamespace(
1244 + valid_processor_kwargs=_ProcessorKwargs,
1245 + image_processor=SimpleNamespace(valid_kwargs=_ImageProcessorKwargs),
1246 + video_processor=SimpleNamespace(valid_kwargs=_VideoProcessorKwargs),
1247 + ),
1248 + {
1249 + "text_kwargs": {"padding"},
1250 + "images_kwargs": {"size", "min_pixels"},
1251 + "videos_kwargs": {"size", "fps"},
1252 + },
1253 + ),
1254 + (
1255 + {"image": None},
1256 + SimpleNamespace(
1257 + valid_processor_kwargs=_ProcessorKwargs,
1258 + image_processor=SimpleNamespace(valid_kwargs=_ImageProcessorKwargs),
1259 + ),
1260 + {
1261 + "text_kwargs": {"padding"},
1262 + "images_kwargs": {"size", "min_pixels"},
1263 + },
1264 + ),
1265 + (
1266 + {"audio": None},
1267 + SimpleNamespace(valid_processor_kwargs=_ProcessorKwargs),
1268 + {
1269 + "text_kwargs": {"padding"},
1270 + "audio_kwargs": {"sampling_rate"},
1271 + },
1272 + ),
1273 + ],
1274 +)
1275 +def test_get_supported_mm_processor_kwargs_uses_supported_modalities(
1276 + monkeypatch: pytest.MonkeyPatch,
1277 + supported_mm_limits: dict[str, int | None],
1278 + processor: SimpleNamespace,
1279 + expected: dict[str, set[str]],
1280 +) -> None:
1281 + info = BaseProcessingInfo(SimpleNamespace())
1282 + info.__dict__["supported_mm_limits"] = supported_mm_limits
1283 + monkeypatch.setattr(info, "get_hf_processor", lambda **_: processor)
1284 +
1285 + assert info.get_supported_mm_processor_kwargs() == expected
1286 +
1287 +
1288 +def test_supported_mm_processor_kwargs_is_cached(
1289 + monkeypatch: pytest.MonkeyPatch,
1290 +) -> None:
1291 + info = BaseProcessingInfo(SimpleNamespace())
1292 + expected = {"images_kwargs": {"size"}}
1293 + calls = 0
1294 +
1295 + def get_supported_mm_processor_kwargs() -> dict[str, set[str]]:
1296 + nonlocal calls
1297 + calls += 1
1298 + return expected
1299 +
1300 + monkeypatch.setattr(
1301 + info,
1302 + "get_supported_mm_processor_kwargs",
1303 + get_supported_mm_processor_kwargs,
1304 + )
1305 +
1306 + first = info.supported_mm_processor_kwargs
1307 + second = info.supported_mm_processor_kwargs
1308 +
1309 + assert first is expected
1310 + assert second is first
1311 + assert calls == 1
1312 +
1313 +
1314 +def test_resolve_mm_processor_kwargs_routes_supported_flat_kwargs():
1315 + kwargs = {
1316 + "size": {"shortest_edge": 64},
1317 + "padding": True,
1318 + "sampling_rate": 16000,
1319 + "unknown": "keep",
1320 + "images_kwargs": {
1321 + "size": {"longest_edge": 1024},
1322 + "custom_image_kwarg": "keep",
1323 + },
1324 + }
1325 + kwargs_before = deepcopy(kwargs)
1326 +
1327 + resolved = _resolve_mm_processor_kwargs(
1328 + kwargs,
1329 + supported_mm_processor_kwargs={
1330 + "text_kwargs": {"padding"},
1331 + "images_kwargs": {"size"},
1332 + "videos_kwargs": {"size"},
1333 + "audio_kwargs": {"sampling_rate"},
1334 + },
1335 + )
1336 +
1337 + assert resolved == {
1338 + "unknown": "keep",
1339 + "text_kwargs": {"padding": True},
1340 + "images_kwargs": {
1341 + "size": {
1342 + "shortest_edge": 64,
1343 + "longest_edge": 1024,
1344 + },
1345 + "custom_image_kwarg": "keep",
1346 + },
1347 + "videos_kwargs": {
1348 + "size": {"shortest_edge": 64},
1349 + },
1350 + "audio_kwargs": {"sampling_rate": 16000},
1351 + }
1352 + assert kwargs == kwargs_before
1353 +
1354 + video_kwargs = resolved["videos_kwargs"]
1355 + image_kwargs = resolved["images_kwargs"]
1356 + assert isinstance(video_kwargs, dict)
1357 + assert isinstance(image_kwargs, dict)
1358 + video_size = video_kwargs["size"]
1359 + image_size = image_kwargs["size"]
1360 + assert isinstance(video_size, dict)
1361 + assert isinstance(image_size, dict)
1362 + video_size["shortest_edge"] = 999
1363 +
1364 + assert image_size["shortest_edge"] == 64
1365 + assert kwargs == kwargs_before
1366 +
1367 +
1368 +def test_resolve_mm_processor_kwargs_rejects_non_mapping_destination_scope():
1369 + with pytest.raises(TypeError, match="images_kwargs"):
1370 + _resolve_mm_processor_kwargs(
1371 + {
1372 + "size": {"shortest_edge": 64},
1373 + "images_kwargs": 123,
1374 + },
1375 + supported_mm_processor_kwargs={
1376 + "images_kwargs": {"size"},
1377 + },
1378 + )
1379 +
1380 +
1381 +def test_resolve_mm_processor_kwargs_without_schema_deduplicates_existing_scopes():
1382 kwargs = {
1235 - "size": {"longest_edge": 1},
1236 - "videos_kwargs": {"size": {"longest_edge": 2}},
1237 - "images_kwargs": {"size": {"longest_edge": 3}},
1383 + "size": {"shortest_edge": 64},
1384 + "num_frames": 16,
1385 + "fps": 4,
1386 + "images_kwargs": {
1387 + "size": {"longest_edge": 1024},
1388 + },
1389 + "videos_kwargs": {
1390 + "num_frames": 8,
1391 + },
1392 + "audio_kwargs": 123,
1393 + "common_kwargs": {
1394 + "fps": 2,
1395 + },
1396 + }
1397 + kwargs_before = deepcopy(kwargs)
1398 +
1399 + resolved = _resolve_mm_processor_kwargs(kwargs)
1400 +
1401 + assert resolved == {
1402 + "fps": 4,
1403 + "images_kwargs": {
1404 + "size": {"longest_edge": 1024},
1405 + },
1406 + "videos_kwargs": {
1407 + "num_frames": 8,
1408 + },
1409 + "audio_kwargs": 123,
1410 + "common_kwargs": {
1411 + "fps": 2,
1412 + },
1413 }
1414 + assert kwargs == kwargs_before
1415 + assert "text_kwargs" not in resolved
1416 +
1417
1240 - assert overlay_modality_mm_kwargs(kwargs, "video")["size"] == {"longest_edge": 2}
1241 - assert overlay_modality_mm_kwargs(kwargs, "image")["size"] == {"longest_edge": 3}
1242 - assert overlay_modality_mm_kwargs(kwargs, None)["size"] == {"longest_edge": 1}
1418 +@pytest.mark.parametrize(
1419 + "inference_kwargs",
1420 + [
1421 + {"size": {"shortest_edge": 128}},
1422 + {"size": {"shortest_edge": 128}, "images_kwargs": None},
1423 + {"size": {"shortest_edge": 128}, "images_kwargs": {}},
1424 + ],
1425 +)
1426 +def test_get_merged_mm_kwargs_treats_empty_scopes_as_absent_before_routing(
1427 + inference_kwargs: dict[str, object],
1428 +):
1429 + mm_config = MultiModalConfig(
1430 + mm_processor_kwargs={
1431 + "size": {
1432 + "shortest_edge": 64,
1433 + "longest_edge": 512,
1434 + },
1435 + "images_kwargs": {
1436 + "size": {"longest_edge": 1024},
1437 + },
1438 + },
1439 + mm_device_do_normalize=False,
1440 + )
1441 + model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config)
1442 + ctx = InputProcessingContext(model_config, tokenizer=None) # type: ignore[arg-type]
1443 +
1444 + merged = ctx.get_merged_mm_kwargs(
1445 + inference_kwargs,
1446 + supported_mm_processor_kwargs={
1447 + "images_kwargs": {"size"},
1448 + "videos_kwargs": {"size"},
1449 + },
1450 + )
1451
1452 + assert merged == {
1453 + "images_kwargs": {
1454 + "size": {
1455 + "shortest_edge": 128,
1456 + "longest_edge": 1024,
1457 + },
1458 + },
1459 + "videos_kwargs": {
1460 + "size": {
1461 + "shortest_edge": 128,
1462 + "longest_edge": 512,
1463 + },
1464 + },
1465 + }
1466
1245 -@pytest.mark.skip_global_cleanup
1246 -def test_overlay_modality_mm_kwargs_ignores_non_mapping_scoped_value():
1247 - kwargs = {"images_kwargs": "not-a-dict", "size": {"longest_edge": 1}}
1248 - assert overlay_modality_mm_kwargs(kwargs, "image")["size"] == {"longest_edge": 1}
1467
1468 +def test_get_merged_mm_kwargs_merges_before_routing():
1469 + mm_config = MultiModalConfig(
1470 + mm_processor_kwargs={
1471 + "size": {
1472 + "shortest_edge": 64,
1473 + "longest_edge": 512,
1474 + },
1475 + "images_kwargs": {
1476 + "size": {"longest_edge": 1024},
1477 + },
1478 + },
1479 + mm_device_do_normalize=False,
1480 + )
1481 + model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config)
1482 + ctx = InputProcessingContext(model_config, tokenizer=None) # type: ignore[arg-type]
1483
1251 -@pytest.mark.skip_global_cleanup
1252 -def test_mm_processor_kwargs_merge_then_overlay_preserves_scoping():
1253 - """Configured videos_kwargs overlay only for video reads after merge."""
1254 - from vllm.config.multimodal import MultiModalConfig
1484 + merged = ctx.get_merged_mm_kwargs(
1485 + {
1486 + "size": {
1487 + "shortest_edge": 128,
1488 + "longest_edge": 768,
1489 + },
1490 + "images_kwargs": {
1491 + "size": {"longest_edge": 2048},
1492 + },
1493 + },
1494 + supported_mm_processor_kwargs={
1495 + "images_kwargs": {"size"},
1496 + "videos_kwargs": {"size"},
1497 + },
1498 + )
1499
1256 - size = {"longest_edge": 469762048, "shortest_edge": 4096}
1257 - mm_config = MultiModalConfig(mm_processor_kwargs={"videos_kwargs": {"size": size}})
1258 - merged = mm_config.merge_mm_processor_kwargs({})
1259 - assert overlay_modality_mm_kwargs(merged, "video")["size"] == size
1260 - assert "size" not in overlay_modality_mm_kwargs(merged, "image")
1261 - assert "size" not in overlay_modality_mm_kwargs(merged, None)
1500 + assert merged == {
1501 + "images_kwargs": {
1502 + "size": {
1503 + "shortest_edge": 128,
1504 + "longest_edge": 2048,
1505 + },
1506 + },
1507 + "videos_kwargs": {
1508 + "size": {
1509 + "shortest_edge": 128,
1510 + "longest_edge": 768,
1511 + },
1512 + },
1513 + }
1514
1515
1516 def test_processor_inputs_hashes_partial_uuids():
vllm/config/multimodal.py
+187 -1
@@ -27,6 +27,186 @@ logger = init_logger(__name__)
27
28 _T = TypeVar("_T")
29
30 +# HuggingFace processors accept a shared flat namespace in addition to the
31 +# nested processor kwarg scopes ``text_kwargs`` / ``images_kwargs`` /
32 +# ``videos_kwargs`` / ``audio_kwargs``. vLLM recognizes these scopes when
33 +# resolving ``mm_processor_kwargs``.
34 +_HF_PROCESSOR_KWARG_SCOPES = (
35 + "text_kwargs",
36 + "images_kwargs",
37 + "videos_kwargs",
38 + "audio_kwargs",
39 +)
40 +
41 +
42 +def _recursively_merge_mm_processor_kwargs(
43 + defaults: Mapping[str, object],
44 + overrides: Mapping[str, object],
45 +) -> dict[str, object]:
46 + """Merge two mappings recursively, with ``overrides`` taking precedence.
47 +
48 + Nested values are merged only when both values are mappings; otherwise the
49 + override replaces the existing value.
50 + """
51 + merged = dict(defaults)
52 +
53 + for key, value in overrides.items():
54 + default_value = merged.get(key)
55 + if isinstance(default_value, Mapping) and isinstance(value, Mapping):
56 + merged[key] = _recursively_merge_mm_processor_kwargs(default_value, value)
57 + else:
58 + merged[key] = value
59 +
60 + return merged
61 +
62 +
63 +def _merge_mm_processor_scope(
64 + config_flat: Mapping[str, object],
65 + config_scoped: Mapping[str, object],
66 + inference_flat: Mapping[str, object],
67 + inference_scoped: Mapping[str, object],
68 +) -> dict[str, object]:
69 + """Merge kwargs for one existing HuggingFace processor kwarg scope.
70 +
71 + Flat kwargs contribute only to keys present in either the configured or
72 + inference-time scope. For those keys, precedence is:
73 + configured flat < configured scoped < inference flat < inference scoped.
74 +
75 + Nested mappings are merged recursively; otherwise the higher-priority value
76 + replaces the lower-priority value.
77 + """
78 + # Flat kwargs contribute only to keys present in this scope in either input.
79 + scoped_keys = config_scoped.keys() | inference_scoped.keys()
80 + config_flat_for_scope = {
81 + key: value for key, value in config_flat.items() if key in scoped_keys
82 + }
83 + inference_flat_for_scope = {
84 + key: value for key, value in inference_flat.items() if key in scoped_keys
85 + }
86 +
87 + # Merge from lowest to highest priority:
88 + # configured flat -> configured scoped -> inference flat -> inference scoped.
89 + merged = _recursively_merge_mm_processor_kwargs(
90 + config_flat_for_scope, config_scoped
91 + )
92 + merged = _recursively_merge_mm_processor_kwargs(merged, inference_flat_for_scope)
93 + merged = _recursively_merge_mm_processor_kwargs(merged, inference_scoped)
94 + return merged
95 +
96 +
97 +def _merge_mm_processor_kwargs(
98 + config_kwargs: Mapping[str, object],
99 + inference_kwargs: Mapping[str, object],
100 +) -> dict[str, object]:
101 + """Merge configured and inference-time multi-modal processor kwargs.
102 +
103 + The merge is performed in two parts:
104 +
105 + 1. The shared flat kwargs are merged recursively, with inference-time values
106 + taking precedence.
107 + 2. Each existing HuggingFace processor kwarg scope is merged separately.
108 + For keys present in either the configured or inference-time scope,
109 + precedence is:
110 + configured flat < configured scoped < inference flat < inference scoped.
111 +
112 + For example:
113 +
114 + configured = {
115 + "size": {"shortest_edge": 256},
116 + "videos_kwargs": {
117 + "size": {"longest_edge": 1024},
118 + "num_frames": 16,
119 + },
120 + }
121 + inference = {
122 + "size": {"shortest_edge": 512},
123 + "videos_kwargs": {"fps": 4},
124 + }
125 +
126 + produces:
127 +
128 + {
129 + "size": {"shortest_edge": 512},
130 + "videos_kwargs": {
131 + "size": {
132 + "shortest_edge": 512,
133 + "longest_edge": 1024,
134 + },
135 + "num_frames": 16,
136 + "fps": 4,
137 + },
138 + }
139 +
140 + Scoped values do not modify the shared flat kwargs or sibling scopes.
141 + Missing scopes, None-valued scopes, and empty scope mappings are treated as
142 + absent and do not create a scope.
143 + """
144 + # Merge the shared flat kwargs first; nested scopes are handled separately
145 + # below.
146 + config_flat = {
147 + key: value
148 + for key, value in config_kwargs.items()
149 + if key not in _HF_PROCESSOR_KWARG_SCOPES
150 + }
151 + inference_flat = {
152 + key: value
153 + for key, value in inference_kwargs.items()
154 + if key not in _HF_PROCESSOR_KWARG_SCOPES
155 + }
156 + merged = _recursively_merge_mm_processor_kwargs(config_flat, inference_flat)
157 +
158 + # Merge each nested processor kwarg scope present in either input.
159 + for scoped_key in _HF_PROCESSOR_KWARG_SCOPES:
160 + config_scope_value = config_kwargs.get(scoped_key)
161 + inference_scope_value = inference_kwargs.get(scoped_key)
162 +
163 + # Treat missing scopes, None-valued scopes, and empty scope mappings as
164 + # absent.
165 + if isinstance(config_scope_value, Mapping) and not config_scope_value:
166 + config_scope_value = None
167 + if isinstance(inference_scope_value, Mapping) and not inference_scope_value:
168 + inference_scope_value = None
169 +
170 + # Skip adding the scope to the merged result when it is absent from both
171 + # sources.
172 + if config_scope_value is None and inference_scope_value is None:
173 + continue
174 +
175 + # Handle the inference scope first because it has higher precedence; only
176 + # mapping-valued scopes participate in the recursive scoped merge.
177 + inference_scope_kwargs: Mapping[str, object]
178 + if inference_scope_value is None:
179 + inference_scope_kwargs = {}
180 + elif not isinstance(inference_scope_value, Mapping):
181 + # Non-mapping scopes cannot participate in the recursive scoped merge;
182 + # the higher-priority inference value replaces the configured scope.
183 + merged[scoped_key] = inference_scope_value
184 + continue
185 + else:
186 + inference_scope_kwargs = inference_scope_value
187 +
188 + # Handle the configured scope next because it has lower precedence; only
189 + # mapping-valued scopes participate in the recursive scoped merge.
190 + if isinstance(config_scope_value, Mapping):
191 + config_scope_kwargs = config_scope_value
192 + elif inference_scope_value is None:
193 + # Non-mapping scopes cannot participate in the recursive scoped merge;
194 + # preserve the configured value when inference contributes nothing.
195 + merged[scoped_key] = config_scope_value
196 + continue
197 + else:
198 + config_scope_kwargs = {}
199 +
200 + # Merge this scope together with the matching flat kwargs.
201 + merged[scoped_key] = _merge_mm_processor_scope(
202 + config_flat,
203 + config_scope_kwargs,
204 + inference_flat,
205 + inference_scope_kwargs,
206 + )
207 +
208 + return merged
209 +
210
211 @dataclass
212 class BaseDummyOptions:
@@ -533,8 +713,14 @@ class MultiModalConfig:
713 ) -> dict[str, object]:
714 """Get the keyword arguments to pass to the multi-modal processor
715 according to the extra arguments passed during inference.
716 +
717 + Nested mappings are merged recursively, with inference-time values taking
718 + precedence over configured values.
719 """
537 - merged = dict(self.mm_processor_kwargs or {}) | dict(inference_kwargs)
720 + merged = _merge_mm_processor_kwargs(
721 + self.mm_processor_kwargs or {},
722 + inference_kwargs,
723 + )
724 if self.mm_device_do_normalize:
725 # The model expects raw pixels and normalises on device;
726 # overriding these per request would silently double-normalise.
vllm/model_executor/models/mistral3.py
+9 -3
@@ -3,7 +3,7 @@
3
4 import math
5 from collections.abc import Iterable, Mapping, Sequence
6 -from typing import Annotated, Literal
6 +from typing import Annotated, Any, Literal, cast
7
8 import torch
9 import torch.nn as nn
@@ -212,8 +212,14 @@ class Mistral3ProcessingInfo(BaseProcessingInfo):
212 ) -> PixtralHFEncoderInfo:
213 processor = self.get_hf_processor()
214 size = processor.image_processor.size
215 - merged_kwargs = self.ctx.get_merged_mm_kwargs(mm_processor_kwargs or {})
216 - if override_size := merged_kwargs.get("size"):
215 + merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs(
216 + mm_processor_kwargs or {}
217 + )
218 + image_mm_kwargs = cast(
219 + Mapping[str, Any],
220 + merged_mm_kwargs.get("images_kwargs", {}),
221 + )
222 + if override_size := image_mm_kwargs.get("size"):
223 size = size | override_size
224
225 image_size = size["longest_edge"]
vllm/model_executor/models/qwen2_vl.py
+130 -18
@@ -28,7 +28,7 @@
28 import math
29 from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence
30 from functools import partial
31 -from typing import Annotated, Any, Literal, TypeAlias
31 +from typing import Annotated, Any, Literal, TypeAlias, cast
32
33 import numpy as np
34 import torch
@@ -865,6 +865,118 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
865 def get_supported_mm_limits(self) -> Mapping[str, int | None]:
866 return {"image": None, "video": None}
867
868 + @staticmethod
869 + def _complete_mm_processor_size_aliases(
870 + mm_kwargs: Mapping[str, object],
871 + ) -> dict[str, object]:
872 + """Complete missing Qwen-VL size aliases.
873 +
874 + Qwen-VL accepts ``size.shortest_edge`` / ``min_pixels`` and
875 + ``size.longest_edge`` / ``max_pixels`` as equivalent controls. When
876 + only one representation is provided with a non-``None`` value, add the
877 + missing equivalent representation. If both are already present, leave
878 + them unchanged even when their values conflict.
879 +
880 + Apply this independently to the flat kwargs and to each existing
881 + ``images_kwargs`` / ``videos_kwargs`` mapping. Values are not propagated
882 + between these mappings.
883 + """
884 +
885 + def add_missing_size_aliases(
886 + kwargs: Mapping[str, object],
887 + ) -> dict[str, object]:
888 + """Add missing size aliases within a kwargs mapping."""
889 + out_kwargs = dict(kwargs)
890 + # Copy `size` separately because `out_kwargs` is a shallow copy and
891 + # size edges may be added below.
892 + raw_size = cast(
893 + Mapping[str, object] | None,
894 + out_kwargs.get("size"),
895 + )
896 + size_kwargs = {} if raw_size is None else dict(raw_size)
897 +
898 + for edge_key, pixel_key in (
899 + ("shortest_edge", "min_pixels"),
900 + ("longest_edge", "max_pixels"),
901 + ):
902 + # Add the pixel alias from `size` when only the size edge is
903 + # present.
904 + if pixel_key not in out_kwargs:
905 + if edge_key in size_kwargs and size_kwargs[edge_key] is not None:
906 + out_kwargs[pixel_key] = size_kwargs[edge_key]
907 + continue
908 +
909 + pixel_value = out_kwargs[pixel_key]
910 + # HF treats `min_pixels` / `max_pixels` set to None as no size
911 + # override, so do not propagate them to the equivalent size edge.
912 + if pixel_value is None:
913 + continue
914 +
915 + # Add the size edge when only the pixel alias is present.
916 + if edge_key not in size_kwargs:
917 + size_kwargs[edge_key] = pixel_value
918 + out_kwargs["size"] = size_kwargs
919 +
920 + return out_kwargs
921 +
922 + prepared_mm_kwargs = add_missing_size_aliases(mm_kwargs)
923 + # Complete aliases independently inside existing image/video kwargs;
924 + # flat values are not copied into the nested mappings.
925 + for nested_kwargs_key in ("images_kwargs", "videos_kwargs"):
926 + nested_kwargs = prepared_mm_kwargs.get(nested_kwargs_key)
927 + if isinstance(nested_kwargs, Mapping):
928 + prepared_mm_kwargs[nested_kwargs_key] = add_missing_size_aliases(
929 + nested_kwargs
930 + )
931 +
932 + return prepared_mm_kwargs
933 +
934 + def _merge_and_resolve_mm_processor_kwargs(
935 + self,
936 + mm_kwargs: Mapping[str, object],
937 + ) -> dict[str, Any]:
938 + """Merge configured and request Qwen-VL ``mm_processor_kwargs``.
939 +
940 + Request-side ``size.shortest_edge`` / ``min_pixels`` and
941 + ``size.longest_edge`` / ``max_pixels`` aliases are completed before the
942 + merge so request precedence is preserved across equivalent
943 + representations.
944 + """
945 + # Complete request-side Qwen-VL size aliases before merging with
946 + # configured kwargs.
947 + prepared_mm_kwargs = self._complete_mm_processor_size_aliases(mm_kwargs)
948 +
949 + return super()._merge_and_resolve_mm_processor_kwargs(prepared_mm_kwargs)
950 +
951 + @staticmethod
952 + def _get_vision_size(
953 + mm_kwargs: Mapping[str, object],
954 + default_size: Mapping[str, int] | None = None,
955 + ) -> dict[str, int]:
956 + """Resolve Qwen-VL vision size kwargs with optional processor defaults.
957 +
958 + ``size`` overrides any processor defaults, then ``min_pixels`` and
959 + ``max_pixels`` override the corresponding size edges.
960 + """
961 + size = dict(default_size) if default_size is not None else {}
962 +
963 + override_size = cast(
964 + Mapping[str, int] | None,
965 + mm_kwargs.get("size"),
966 + )
967 + if override_size is not None:
968 + size.update(override_size)
969 +
970 + min_pixels = cast(int | None, mm_kwargs.get("min_pixels"))
971 + if min_pixels is not None:
972 + size["shortest_edge"] = min_pixels
973 +
974 + max_pixels = cast(int | None, mm_kwargs.get("max_pixels"))
975 + if max_pixels is not None:
976 + size["longest_edge"] = max_pixels
977 +
978 + return size
979 +
980 def get_mm_max_tokens_per_item(
981 self,
982 seq_len: int,
@@ -883,7 +995,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
995 do_resize: bool = True,
996 image_processor: Qwen2VLImageProcessor,
997 mm_kwargs: Mapping[str, object],
886 - modality: str | None = None,
998 + modality: Literal["image", "video"] | None = None,
999 ) -> tuple[ImageSize, int]:
1000 hf_config = self.get_hf_config()
1001 vision_config = hf_config.vision_config
@@ -891,14 +1003,16 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
1003 merge_size = vision_config.spatial_merge_size
1004 temporal_patch_size = vision_config.temporal_patch_size
1005
894 - mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality)
895 - size = image_processor.size
896 - if override_size := mm_kwargs.get("size"):
897 - size = size | override_size
898 - if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None:
899 - size = size | {"shortest_edge": override_min_pixels}
900 - if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
901 - size = size | {"longest_edge": override_max_pixels}
1006 + if modality is None:
1007 + modality = "image"
1008 +
1009 + merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs(mm_kwargs)
1010 + scope = "images_kwargs" if modality == "image" else "videos_kwargs"
1011 + scoped_mm_kwargs = merged_mm_kwargs.get(scope, {})
1012 + size = self._get_vision_size(
1013 + scoped_mm_kwargs,
1014 + default_size=image_processor.size,
1015 + )
1016
1017 if do_resize:
1018 resized_height, resized_width = smart_resize(
@@ -986,14 +1100,12 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
1100 if max_pixels is None:
1101 image_processor = self.get_image_processor()
1102
989 - mm_kwargs = self.ctx.get_merged_mm_kwargs({}, modality="image")
990 - size = image_processor.size
991 - if override_size := mm_kwargs.get("size"):
992 - size = size | override_size
993 - if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None:
994 - size = size | {"shortest_edge": override_min_pixels}
995 - if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
996 - size = size | {"longest_edge": override_max_pixels}
1103 + merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs({})
1104 + image_mm_kwargs = merged_mm_kwargs.get("images_kwargs", {})
1105 + size = self._get_vision_size(
1106 + image_mm_kwargs,
1107 + default_size=image_processor.size,
1108 + )
1109
1110 max_pixels = size["longest_edge"]
1111
vllm/model_executor/models/qwen3_vl.py
+107 -52
@@ -27,7 +27,7 @@
27 from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence
28 from functools import lru_cache, partial
29 from itertools import islice
30 -from typing import Any
30 +from typing import Any, Literal
31
32 import numpy as np
33 import torch
@@ -944,6 +944,16 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
944 allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
945 )
946
947 + def get_supported_mm_processor_kwargs(self) -> dict[str, set[str]]:
948 + supported = super().get_supported_mm_processor_kwargs()
949 +
950 + # vLLM has historically accepted `min_pixels` / `max_pixels` for
951 + # Qwen3-VL video. Include them in the routing schema even though
952 + # Qwen3VLVideoProcessor only accepts the equivalent `size`; they are
953 + # folded into `size` before calling HF.
954 + supported["videos_kwargs"].update({"min_pixels", "max_pixels"})
955 + return supported
956 +
957 def _get_vision_info(
958 self,
959 *,
@@ -953,7 +963,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
963 do_resize: bool = True,
964 image_processor: Qwen2VLImageProcessor | Qwen3VLVideoProcessor,
965 mm_kwargs: Mapping[str, object],
956 - modality: str | None = None,
966 + modality: Literal["image", "video"] | None = None,
967 ) -> tuple[ImageSize, int]:
968 is_video = isinstance(image_processor, Qwen3VLVideoProcessor)
969
@@ -965,14 +975,14 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
975
976 if modality is None:
977 modality = "video" if is_video else "image"
968 - mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality)
969 - size = image_processor.size
970 - if override_size := mm_kwargs.get("size"):
971 - size = size | override_size
972 - if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None:
973 - size = size | {"shortest_edge": override_min_pixels}
974 - if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
975 - size = size | {"longest_edge": override_max_pixels}
978 +
979 + merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs(mm_kwargs)
980 + scope = "videos_kwargs" if modality == "video" else "images_kwargs"
981 + scoped_mm_kwargs = merged_mm_kwargs.get(scope, {})
982 + size = self._get_vision_size(
983 + scoped_mm_kwargs,
984 + default_size=image_processor.size,
985 + )
986
987 if do_resize:
988 if is_video:
@@ -1030,9 +1040,13 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
1040 ) -> int:
1041 video_processor = self.get_video_processor()
1042
1033 - mm_kwargs = self.ctx.get_merged_mm_kwargs({}, modality="video")
1034 - video_size = mm_kwargs.get("size", video_processor.size)
1035 - temporal_patch_size = mm_kwargs.get(
1043 + merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs({})
1044 + video_mm_kwargs = merged_mm_kwargs.get("videos_kwargs", {})
1045 + video_size = self._get_vision_size(
1046 + video_mm_kwargs,
1047 + default_size=video_processor.size,
1048 + )
1049 + temporal_patch_size = video_mm_kwargs.get(
1050 "temporal_patch_size", video_processor.temporal_patch_size
1051 )
1052
@@ -1162,9 +1176,13 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
1176
1177 video_processor = self.info.get_video_processor()
1178
1165 - mm_kwargs = self.info.ctx.get_merged_mm_kwargs({}, modality="video")
1166 - video_size = mm_kwargs.get("size", video_processor.size)
1167 - temporal_patch_size = mm_kwargs.get(
1179 + merged_mm_kwargs = self.info._merge_and_resolve_mm_processor_kwargs({})
1180 + video_mm_kwargs = merged_mm_kwargs.get("videos_kwargs", {})
1181 + video_size = self.info._get_vision_size(
1182 + video_mm_kwargs,
1183 + default_size=video_processor.size,
1184 + )
1185 + temporal_patch_size = video_mm_kwargs.get(
1186 "temporal_patch_size", video_processor.temporal_patch_size
1187 )
1188
@@ -1180,13 +1198,13 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
1198 # over enough frames that the cap is not binding for the dummy.
1199 # This takes precedence over a shrinking num_frames override, which
1200 # would otherwise reintroduce the under-profiling.
1183 - if mm_kwargs.get("cap_pixels_per_frame"):
1184 - max_video_tokens = mm_kwargs.get(
1201 + if video_mm_kwargs.get("cap_pixels_per_frame"):
1202 + max_video_tokens = video_mm_kwargs.get(
1203 "max_video_tokens",
1204 getattr(video_processor, "max_video_tokens", 768),
1205 )
1188 - patch_size = mm_kwargs.get("patch_size", video_processor.patch_size)
1189 - merge_size = mm_kwargs.get("merge_size", video_processor.merge_size)
1206 + patch_size = video_mm_kwargs.get("patch_size", video_processor.patch_size)
1207 + merge_size = video_mm_kwargs.get("merge_size", video_processor.merge_size)
1208 per_frame_cap = max_video_tokens * (patch_size * merge_size) ** 2
1209 target_num_frames = max(
1210 target_num_frames,
@@ -1348,6 +1366,47 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
1366 if not hf_data:
1367 return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
1368
1369 + # Merge and resolve the multimodal processor kwargs once so every HF
1370 + # call below starts from the same engine/request state and scoped form.
1371 + merged_mm_kwargs = self.info._merge_and_resolve_mm_processor_kwargs(hf_kwargs)
1372 +
1373 + if "images_kwargs" in merged_mm_kwargs:
1374 + image_scoped_kwargs = dict(merged_mm_kwargs["images_kwargs"])
1375 +
1376 + # Qwen2VLImageProcessor requires both `size` edges. If a partial size
1377 + # override is passed, fill missing edges from the processor defaults
1378 + # before any HF call.
1379 + if image_scoped_kwargs.keys() & {"size", "min_pixels", "max_pixels"}:
1380 + default_image_size = self.info.get_image_processor().size
1381 + image_scoped_kwargs["size"] = self.info._get_vision_size(
1382 + image_scoped_kwargs,
1383 + default_size=default_image_size,
1384 + )
1385 +
1386 + merged_mm_kwargs["images_kwargs"] = image_scoped_kwargs
1387 +
1388 + if "videos_kwargs" in merged_mm_kwargs:
1389 + video_scoped_kwargs = dict(merged_mm_kwargs["videos_kwargs"])
1390 +
1391 + # Qwen3VLVideoProcessor requires both `size` edges. If a partial size
1392 + # override is passed, fill missing edges from the processor defaults
1393 + # before any HF call.
1394 + if video_scoped_kwargs.keys() & {"size", "min_pixels", "max_pixels"}:
1395 + default_video_size = self.info.get_video_processor().size
1396 + video_scoped_kwargs["size"] = self.info._get_vision_size(
1397 + video_scoped_kwargs,
1398 + default_size=default_video_size,
1399 + )
1400 +
1401 + # Qwen3VLVideoProcessor does not accept vLLM's min/max pixel
1402 + # compatibility kwargs. They have already been folded into `size`,
1403 + # so remove them before any HF call, including calls that contain
1404 + # only text/images.
1405 + video_scoped_kwargs.pop("min_pixels", None)
1406 + video_scoped_kwargs.pop("max_pixels", None)
1407 +
1408 + merged_mm_kwargs["videos_kwargs"] = video_scoped_kwargs
1409 +
1410 # Separate video processing from image processing. Because the videos
1411 # are processed into several image patches
1412 video_input_ids_lst: list[list[int]] = []
@@ -1373,32 +1432,22 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
1432 # used to calculate the timestamps. Make sure that
1433 # do_sample_frames in mm_kwargs is false for presampled videos.
1434
1376 - # NOTE: a copy of is created to update do_sample_frames,
1377 - # otherwise mm_hash for the object will be incorrect.
1378 - video_mm_kwargs: dict[str, Any] = dict(hf_kwargs)
1379 - merged = self.info.ctx.get_merged_mm_kwargs(hf_kwargs, modality="video")
1380 - if merged.keys() & {"size", "min_pixels", "max_pixels"}:
1381 - video_size = dict(self.info.get_video_processor().size)
1382 - size_override = merged.get("size")
1383 - if size_override is not None:
1384 - video_size = video_size | size_override
1385 - min_pixels = merged.get("min_pixels")
1386 - if min_pixels is not None:
1387 - video_size["shortest_edge"] = min_pixels
1388 - max_pixels = merged.get("max_pixels")
1389 - if max_pixels is not None:
1390 - video_size["longest_edge"] = max_pixels
1391 - video_mm_kwargs["size"] = video_size
1392 - sampled_fps = video_mm_kwargs.get("fps")
1435 + # The root copy is shallow, so give each item its own video
1436 + # scope before modifying per-video values below.
1437 + video_mm_kwargs = dict(merged_mm_kwargs)
1438 + video_scoped_kwargs = dict(video_mm_kwargs.get("videos_kwargs", {}))
1439 + video_mm_kwargs["videos_kwargs"] = video_scoped_kwargs
1440 +
1441 + sampled_fps = video_scoped_kwargs.get("fps")
1442 if is_list_of(sampled_fps, float):
1394 - video_mm_kwargs["fps"] = sampled_fps[item_idx]
1395 - sampled_num_frames = video_mm_kwargs.get("num_frames")
1443 + video_scoped_kwargs["fps"] = sampled_fps[item_idx]
1444 + sampled_num_frames = video_scoped_kwargs.get("num_frames")
1445 if is_list_of(sampled_num_frames, int):
1397 - video_mm_kwargs["num_frames"] = sampled_num_frames[item_idx]
1398 - if "do_sample_frames" not in video_mm_kwargs:
1446 + video_scoped_kwargs["num_frames"] = sampled_num_frames[item_idx]
1447 + if "do_sample_frames" not in video_scoped_kwargs:
1448 # qwen_vl_utils already has "do_sample_frames" in
1449 # mm_kwargs, don't overwrite it.
1401 - video_mm_kwargs["do_sample_frames"] = metadata.get(
1450 + video_scoped_kwargs["do_sample_frames"] = metadata.get(
1451 "do_sample_frames", False
1452 )
1453
@@ -1409,9 +1458,9 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
1458 # Compute timestamps here where we have access to metadata
1459 timestamps = self.info._get_video_second_idx(
1460 metadata=metadata,
1412 - do_sample_frames=video_mm_kwargs["do_sample_frames"],
1413 - sampled_fps=video_mm_kwargs.get("fps"),
1414 - sampled_num_frames=video_mm_kwargs.get("num_frames"),
1461 + do_sample_frames=video_scoped_kwargs["do_sample_frames"],
1462 + sampled_fps=video_scoped_kwargs.get("fps"),
1463 + sampled_num_frames=video_scoped_kwargs.get("num_frames"),
1464 )
1465 timestamps_per_video.append(timestamps)
1466
@@ -1423,8 +1472,11 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
1472 # to prevent HF's BaseVideoProcessor.preprocess() from
1473 # filling in the class default (fps=2) via setdefault(),
1474 # which would conflict with num_frames (mutually exclusive).
1426 - if "num_frames" in video_mm_kwargs and "fps" not in video_mm_kwargs:
1427 - video_mm_kwargs["fps"] = None
1475 + if (
1476 + "num_frames" in video_scoped_kwargs
1477 + and "fps" not in video_scoped_kwargs
1478 + ):
1479 + video_scoped_kwargs["fps"] = None
1480
1481 video_outputs = self.info.ctx.call_hf_processor(
1482 self.info.get_hf_processor(**video_mm_kwargs),
@@ -1485,11 +1537,14 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
1537 else:
1538 video_outputs = dict()
1539
1488 - # fps/num_frames are video-only kwargs already consumed by the loop;
1489 - # exclude them so the text/image processor call below never gets a list.
1490 - non_video_mm_kwargs = {
1491 - k: v for k, v in hf_kwargs.items() if k not in ("fps", "num_frames")
1492 - }
1540 + # The root copy is shallow, so copy the video scope before removing
1541 + # per-video kwargs to avoid mutating `merged_mm_kwargs`.
1542 + non_video_mm_kwargs = dict(merged_mm_kwargs)
1543 + if "videos_kwargs" in non_video_mm_kwargs:
1544 + non_video_scoped_kwargs = dict(non_video_mm_kwargs["videos_kwargs"])
1545 + non_video_scoped_kwargs.pop("fps", None)
1546 + non_video_scoped_kwargs.pop("num_frames", None)
1547 + non_video_mm_kwargs["videos_kwargs"] = non_video_scoped_kwargs
1548 processed_data = self.info.ctx.call_hf_processor(
1549 self.info.get_hf_processor(**non_video_mm_kwargs),
1550 hf_data,
vllm/multimodal/processing/context.py
+176 -42
@@ -5,7 +5,7 @@ from collections.abc import Callable, Mapping
5 from contextlib import contextmanager
6 from dataclasses import dataclass, field
7 from functools import cached_property
8 -from typing import TYPE_CHECKING, Any, overload
8 +from typing import TYPE_CHECKING, Any, get_type_hints, overload
9
10 import torch
11 from typing_extensions import TypeVar
@@ -42,46 +42,129 @@ else:
42
43 logger = init_logger(__name__)
44
45 -# HuggingFace processors accept nested ``images_kwargs`` / ``videos_kwargs`` /
46 -# ``audio_kwargs`` in addition to a shared flat namespace. vLLM's token-budget
47 -# and dummy-input code only reads the flat keys, so scoped overrides have to
48 -# be overlaid per modality.
49 -_HF_MODALITY_PROCESSOR_KWARGS = {
50 - "image": "images_kwargs",
51 - "video": "videos_kwargs",
52 - "audio": "audio_kwargs",
53 -}
54 -
45 +# HuggingFace processors accept a shared flat namespace in addition to the
46 +# nested processor kwarg scopes ``text_kwargs`` / ``images_kwargs`` /
47 +# ``videos_kwargs`` / ``audio_kwargs``. vLLM recognizes these scopes when
48 +# resolving ``mm_processor_kwargs``.
49 +_HF_PROCESSOR_KWARG_SCOPES = (
50 + "text_kwargs",
51 + "images_kwargs",
52 + "videos_kwargs",
53 + "audio_kwargs",
54 +)
55
56 -def overlay_modality_mm_kwargs(
57 - kwargs: Mapping[str, object],
58 - modality: str | None,
59 -) -> dict[str, Any]:
60 - """Overlay HF-style nested processor kwargs onto the flat namespace.
56
62 - Overlay only the requested modality so a video ``size`` bump does not leak
63 - into the image budget. Flat keys keep their current shared-namespace
64 - behavior when no scoped dict is present.
57 +def _merge_scoped_mm_processor_value(
58 + flat_value: object,
59 + scoped_value: object,
60 +) -> object:
61 + """Merge a flat value with an existing, more specific scoped value.
62
66 - Args:
67 - kwargs: Merged multi-modal processor kwargs.
68 - modality: Target modality (``image``, ``video``, or ``audio``).
69 - When ``None``, ``kwargs`` is copied without overlay.
63 + Mappings are merged recursively; otherwise the scoped value replaces the
64 + flat value.
65 + """
66 + if isinstance(flat_value, Mapping) and isinstance(scoped_value, Mapping):
67 + # Merge recursively so scoped leaves override matching flat leaves
68 + # without dropping unrelated flat entries.
69 + merged = dict(flat_value)
70 + for key, value in scoped_value.items():
71 + if key in merged:
72 + merged[key] = _merge_scoped_mm_processor_value(merged[key], value)
73 + else:
74 + merged[key] = value
75 + return merged
76 + return scoped_value
77
71 - Returns:
72 - A new dict with the modality-scoped keys overlaid when present.
78
79 +def _resolve_mm_processor_kwargs(
80 + kwargs: Mapping[str, object],
81 + supported_mm_processor_kwargs: Mapping[str, set[str]] | None = None,
82 +) -> dict[str, object]:
83 + """Resolve flat processor kwargs into HuggingFace processor kwarg scopes.
84 +
85 + With ``supported_mm_processor_kwargs``, each scope lists the flat keys it
86 + supports. Matching flat keys are routed to every scope that supports them,
87 + existing scoped values take precedence, and keys not supported by any scope
88 + remain flat.
89 +
90 + Without ``supported_mm_processor_kwargs``, flat keys already represented in
91 + at least one existing mapping-valued scope are removed from the shared flat
92 + namespace; all other flat keys are left unchanged because there is no
93 + information to determine which scopes should receive them.
94 """
75 - merged = dict(kwargs)
76 - if modality is None:
77 - return merged
78 - scoped_key = _HF_MODALITY_PROCESSOR_KWARGS.get(modality)
79 - if scoped_key is None:
80 - return merged
81 - scoped = merged.get(scoped_key)
82 - if not isinstance(scoped, Mapping):
83 - return merged
84 - return merged | dict(scoped)
95 + resolved = dict(kwargs)
96 +
97 + # Without ``supported_mm_processor_kwargs``, remove flat entries already
98 + # represented in at least one existing mapping-valued scope. There is no
99 + # information to route the remaining flat entries.
100 + if supported_mm_processor_kwargs is None:
101 + # Collect all keys represented in existing mapping-valued scopes.
102 + # Non-mapping scope values do not contain scoped kwargs and are left
103 + # untouched.
104 + represented_scoped_keys: set[str] = set()
105 + for scoped_key in _HF_PROCESSOR_KWARG_SCOPES:
106 + scoped_kwargs = resolved.get(scoped_key)
107 + if isinstance(scoped_kwargs, Mapping):
108 + represented_scoped_keys.update(scoped_kwargs)
109 +
110 + # Remove flat entries for keys already represented in a scope.
111 + for key in represented_scoped_keys:
112 + if key not in _HF_PROCESSOR_KWARG_SCOPES:
113 + resolved.pop(key, None)
114 + return resolved
115 +
116 + # With ``supported_mm_processor_kwargs``, collect the flat kwargs before
117 + # routing each one to the processor kwarg scopes that support it.
118 + flat_kwargs = {
119 + key: value
120 + for key, value in resolved.items()
121 + if key not in _HF_PROCESSOR_KWARG_SCOPES
122 + }
123 +
124 + for key, flat_value in flat_kwargs.items():
125 + # Find every processor kwarg scope that supports this flat key. If none
126 + # supports it, keep the key in the shared flat namespace.
127 + supported_scopes = [
128 + scoped_key
129 + for scoped_key in _HF_PROCESSOR_KWARG_SCOPES
130 + if key in supported_mm_processor_kwargs.get(scoped_key, set())
131 + ]
132 + if not supported_scopes:
133 + continue
134 +
135 + # Add the flat value to each supported destination scope.
136 + for scoped_key in supported_scopes:
137 + # Start from a copy of the existing scoped kwargs, or an empty scope
138 + # if absent.
139 + if scoped_key in resolved:
140 + scoped_kwargs = resolved[scoped_key]
141 + # A scope must be a mapping before keys can be added to it. Do
142 + # not replace an explicit non-mapping value.
143 + if not isinstance(scoped_kwargs, Mapping):
144 + raise TypeError(f"`{scoped_key}` must be a mapping")
145 + scoped_kwargs = dict(scoped_kwargs)
146 + else:
147 + scoped_kwargs = {}
148 +
149 + # Add the flat value to this scope. Existing scoped values take
150 + # precedence.
151 + if key in scoped_kwargs:
152 + scoped_kwargs[key] = _merge_scoped_mm_processor_value(
153 + flat_value, scoped_kwargs[key]
154 + )
155 + else:
156 + # Assign the flat value to this scope. For mappings, assign a
157 + # copy so the same mapping object is not shared between the flat
158 + # value and multiple scopes.
159 + scoped_kwargs[key] = (
160 + dict(flat_value) if isinstance(flat_value, Mapping) else flat_value
161 + )
162 + resolved[scoped_key] = scoped_kwargs
163 +
164 + # After routing to every destination, remove the flat copy.
165 + resolved.pop(key)
166 +
167 + return resolved
168
169
170 @dataclass
@@ -294,19 +377,27 @@ class InputProcessingContext:
377 self,
378 kwargs: Mapping[str, object],
379 *,
297 - modality: str | None = None,
380 + supported_mm_processor_kwargs: Mapping[str, set[str]] | None = None,
381 ) -> dict[str, Any]:
382 """Merge configured and request ``mm_processor_kwargs``.
383
301 - When ``modality`` is set, HF-style nested
302 - ``images_kwargs`` / ``videos_kwargs`` / ``audio_kwargs`` are overlaid
303 - onto the flat namespace for vLLM-side reads (token budgets, dummy
304 - inputs). Processor construction and HF ``__call__`` should omit
305 - ``modality`` so the nested dicts still reach the HF processor.
384 + When ``supported_mm_processor_kwargs`` is provided, flat keys are matched
385 + against the keys supported by each HuggingFace processor kwarg scope.
386 + Matching flat keys are routed to every scope that supports them and
387 + removed from the shared flat namespace; keys not supported by any scope
388 + remain flat.
389 +
390 + Without ``supported_mm_processor_kwargs``, flat keys already represented
391 + in at least one existing mapping-valued scope are removed from the shared
392 + flat namespace; all other flat keys are left unchanged because there is
393 + no information to determine which scopes should receive them.
394 """
395 mm_config = self.model_config.get_multimodal_config()
396 merged = mm_config.merge_mm_processor_kwargs(kwargs)
309 - return overlay_modality_mm_kwargs(merged, modality)
397 + return _resolve_mm_processor_kwargs(
398 + merged,
399 + supported_mm_processor_kwargs=supported_mm_processor_kwargs,
400 + )
401
402 def call_hf_processor(
403 self,
@@ -380,6 +471,49 @@ class BaseProcessingInfo:
471 """
472 return self.ctx.get_hf_processor(**kwargs)
473
474 + def get_supported_mm_processor_kwargs(self) -> dict[str, set[str]]:
475 + """Return supported kwarg names for each HF processor kwargs scope."""
476 + processor = self.get_hf_processor()
477 + processor_kwargs = get_type_hints(processor.valid_processor_kwargs)
478 +
479 + supported = {
480 + "text_kwargs": set(processor_kwargs["text_kwargs"].__annotations__),
481 + }
482 + if "image" in self.supported_mm_limits:
483 + supported["images_kwargs"] = set(
484 + processor.image_processor.valid_kwargs.__annotations__
485 + )
486 + if "video" in self.supported_mm_limits:
487 + supported["videos_kwargs"] = set(
488 + processor.video_processor.valid_kwargs.__annotations__
489 + )
490 + if "audio" in self.supported_mm_limits:
491 + supported["audio_kwargs"] = set(
492 + processor_kwargs["audio_kwargs"].__annotations__
493 + )
494 +
495 + return supported
496 +
497 + @cached_property
498 + def supported_mm_processor_kwargs(self) -> dict[str, set[str]]:
499 + """Supported kwarg names for each HF processor kwargs scope."""
500 + return self.get_supported_mm_processor_kwargs()
501 +
502 + def _merge_and_resolve_mm_processor_kwargs(
503 + self,
504 + mm_kwargs: Mapping[str, object],
505 + ) -> dict[str, Any]:
506 + """Merge configured and request ``mm_processor_kwargs``.
507 +
508 + Flat kwargs are routed into the HuggingFace processor kwarg scopes that
509 + support them after the configured/request merge. When a routed flat value
510 + conflicts with an existing scoped value, the scoped value takes precedence.
511 + """
512 + return self.ctx.get_merged_mm_kwargs(
513 + mm_kwargs,
514 + supported_mm_processor_kwargs=self.supported_mm_processor_kwargs,
515 + )
516 +
517 def get_default_tok_params(self) -> TokenizeParams:
518 """Construct the default parameters for tokenization."""
519 from vllm.renderers import TokenizeParams