[Bugfix][Multimodal] Honor modality-scoped mm_processor_kwargs (#53808)
Signed-off-by: Prudhvivuda <prudhvi12042001@gmail.com>
Prudhvi Vuda committed
Aug 31, 2026 at 02:37 UTC
7292ee2791985ff3afe38850fb34125ad0853933
5 files changed
+172
-9
tests/models/multimodal/processing/test_qwen3_vl.py
+44
@@ -142,6 +142,50 @@ def test_processor_multi_video(
142
)
143
144
145
+# Qwen3-VL / Qwen3.8 "Long Video Understanding" pixel budget from the
146
+# model card. Used to check --mm-processor-kwargs scoping (#52834).
147
+_LONG_VIDEO_SIZE = {"longest_edge": 469762048, "shortest_edge": 4096}
148
+
149
+
150
+def _probe_mm_token_budgets(
151
+ model_id: str, mm_processor_kwargs: dict[str, Any] | None
152
+) -> tuple[int, int]:
153
+ ctx = build_model_context(
154
+ model_id,
155
+ mm_processor_kwargs=mm_processor_kwargs,
156
+ limit_mm_per_prompt={"image": 1, "video": 1},
157
+ )
158
+ info = MULTIMODAL_REGISTRY.create_processor(ctx.model_config).info
159
+ video = info.get_max_video_tokens(
160
+ seq_len=500000, mm_counts={"video": 1, "image": 1}
161
+ )
162
+ return video, info.get_max_image_tokens()
163
+
164
+
165
+@pytest.mark.skip_global_cleanup
166
+@pytest.mark.parametrize("model_id", [MODEL_ID])
167
+def test_processor_kwargs_videos_kwargs_does_not_leak_into_image_budget(
168
+ model_id: str,
169
+) -> None:
170
+ """``videos_kwargs.size`` must raise only the video token budget.
171
+
172
+ A flat ``size`` override still applies to both modalities (the previous
173
+ shared-namespace behavior). Regression for #52834.
174
+ """
175
+ stock_video, stock_image = _probe_mm_token_budgets(model_id, None)
176
+ scoped_video, scoped_image = _probe_mm_token_budgets(
177
+ model_id, {"videos_kwargs": {"size": _LONG_VIDEO_SIZE}}
178
+ )
179
+ flat_video, flat_image = _probe_mm_token_budgets(
180
+ model_id, {"size": _LONG_VIDEO_SIZE}
181
+ )
182
+
183
+ assert scoped_video == flat_video
184
+ assert scoped_video > stock_video
185
+ assert scoped_image == stock_image
186
+ assert flat_image > stock_image
187
+
188
+
189
@pytest.mark.parametrize("model_id", [MODEL_ID])
190
@pytest.mark.parametrize(
191
"hf_mm_kwargs",
tests/multimodal/test_processing.py
+58
-1
@@ -10,7 +10,10 @@ import pytest
10
from vllm.config import ModelConfig
11
from vllm.exceptions import VLLMValidationError
12
from vllm.multimodal import MULTIMODAL_REGISTRY
13
-from vllm.multimodal.processing.context import InputProcessingContext
13
+from vllm.multimodal.processing.context import (
14
+ InputProcessingContext,
15
+ overlay_modality_mm_kwargs,
16
+)
17
from vllm.multimodal.processing.processor import (
18
BaseMultiModalProcessor,
19
PlaceholderFeaturesInfo,
@@ -1205,3 +1208,57 @@ def test_apply_prompt_updates_falls_back_with_index_targets():
1208
1209
assert new_token_ids == [200, 201, ord("d"), 9]
1210
assert [p.tokens for p in placeholders["image"]] == [[200, 201], [9]]
1211
+
1212
+
1213
+@pytest.mark.skip_global_cleanup
1214
+def test_overlay_modality_mm_kwargs_scoped_video_does_not_leak_to_image():
1215
+ """HF-style videos_kwargs must overlay only when modality is video."""
1216
+ video_size = {"longest_edge": 469762048, "shortest_edge": 4096}
1217
+ kwargs = {"videos_kwargs": {"size": video_size}}
1218
+
1219
+ assert overlay_modality_mm_kwargs(kwargs, None) == kwargs
1220
+ assert "size" not in overlay_modality_mm_kwargs(kwargs, "image")
1221
+ assert overlay_modality_mm_kwargs(kwargs, "video")["size"] == video_size
1222
+
1223
+
1224
+@pytest.mark.skip_global_cleanup
1225
+def test_overlay_modality_mm_kwargs_flat_size_stays_shared():
1226
+ """A flat size override keeps the current shared-namespace behavior."""
1227
+ size = {"longest_edge": 469762048, "shortest_edge": 4096}
1228
+ kwargs = {"size": size}
1229
+
1230
+ for modality in (None, "image", "video"):
1231
+ assert overlay_modality_mm_kwargs(kwargs, modality)["size"] == size
1232
+
1233
+
1234
+@pytest.mark.skip_global_cleanup
1235
+def test_overlay_modality_mm_kwargs_scoped_wins_over_flat_for_modality():
1236
+ """A nested videos_kwargs size wins over a flat size for video reads."""
1237
+ kwargs = {
1238
+ "size": {"longest_edge": 1},
1239
+ "videos_kwargs": {"size": {"longest_edge": 2}},
1240
+ "images_kwargs": {"size": {"longest_edge": 3}},
1241
+ }
1242
+
1243
+ assert overlay_modality_mm_kwargs(kwargs, "video")["size"] == {"longest_edge": 2}
1244
+ assert overlay_modality_mm_kwargs(kwargs, "image")["size"] == {"longest_edge": 3}
1245
+ assert overlay_modality_mm_kwargs(kwargs, None)["size"] == {"longest_edge": 1}
1246
+
1247
+
1248
+@pytest.mark.skip_global_cleanup
1249
+def test_overlay_modality_mm_kwargs_ignores_non_mapping_scoped_value():
1250
+ kwargs = {"images_kwargs": "not-a-dict", "size": {"longest_edge": 1}}
1251
+ assert overlay_modality_mm_kwargs(kwargs, "image")["size"] == {"longest_edge": 1}
1252
+
1253
+
1254
+@pytest.mark.skip_global_cleanup
1255
+def test_mm_processor_kwargs_merge_then_overlay_preserves_scoping():
1256
+ """Configured videos_kwargs overlay only for video reads after merge."""
1257
+ from vllm.config.multimodal import MultiModalConfig
1258
+
1259
+ size = {"longest_edge": 469762048, "shortest_edge": 4096}
1260
+ mm_config = MultiModalConfig(mm_processor_kwargs={"videos_kwargs": {"size": size}})
1261
+ merged = mm_config.merge_mm_processor_kwargs({})
1262
+ assert overlay_modality_mm_kwargs(merged, "video")["size"] == size
1263
+ assert "size" not in overlay_modality_mm_kwargs(merged, "image")
1264
+ assert "size" not in overlay_modality_mm_kwargs(merged, None)
vllm/model_executor/models/qwen2_vl.py
+5
-2
@@ -881,6 +881,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
881
do_resize: bool = True,
882
image_processor: Qwen2VLImageProcessor,
883
mm_kwargs: Mapping[str, object],
884
+ modality: str | None = None,
885
) -> tuple[ImageSize, int]:
886
hf_config = self.get_hf_config()
887
vision_config = hf_config.vision_config
@@ -888,7 +889,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
889
merge_size = vision_config.spatial_merge_size
890
temporal_patch_size = vision_config.temporal_patch_size
891
891
- mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs)
892
+ mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality)
893
size = image_processor.size
894
if override_size := mm_kwargs.get("size"):
895
size = size | override_size
@@ -936,6 +937,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
937
num_frames=1,
938
image_processor=image_processor,
939
mm_kwargs=mm_kwargs,
940
+ modality="image",
941
)
942
return num_image_tokens
943
@@ -954,6 +956,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
956
num_frames=num_frames,
957
image_processor=image_processor,
958
mm_kwargs=mm_kwargs,
959
+ modality="video",
960
)
961
return num_video_tokens
962
@@ -981,7 +984,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
984
if max_pixels is None:
985
image_processor = self.get_image_processor()
986
984
- mm_kwargs = self.ctx.get_merged_mm_kwargs({})
987
+ mm_kwargs = self.ctx.get_merged_mm_kwargs({}, modality="image")
988
size = image_processor.size
989
if override_size := mm_kwargs.get("size"):
990
size = size | override_size
vllm/model_executor/models/qwen3_vl.py
+9
-4
@@ -941,6 +941,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
941
do_resize: bool = True,
942
image_processor: Qwen2VLImageProcessor | Qwen3VLVideoProcessor,
943
mm_kwargs: Mapping[str, object],
944
+ modality: str | None = None,
945
) -> tuple[ImageSize, int]:
946
is_video = isinstance(image_processor, Qwen3VLVideoProcessor)
947
@@ -950,7 +951,9 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
951
merge_size = vision_config.spatial_merge_size
952
temporal_patch_size = vision_config.temporal_patch_size
953
953
- mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs)
954
+ if modality is None:
955
+ modality = "video" if is_video else "image"
956
+ mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality)
957
size = image_processor.size
958
if override_size := mm_kwargs.get("size"):
959
size = size | override_size
@@ -1014,7 +1017,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
1017
) -> int:
1018
video_processor = self.get_video_processor()
1019
1017
- mm_kwargs = self.ctx.get_merged_mm_kwargs({})
1020
+ mm_kwargs = self.ctx.get_merged_mm_kwargs({}, modality="video")
1021
video_size = mm_kwargs.get("size", video_processor.size)
1022
temporal_patch_size = mm_kwargs.get(
1023
"temporal_patch_size", video_processor.temporal_patch_size
@@ -1150,7 +1153,7 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
1153
1154
video_processor = self.info.get_video_processor()
1155
1153
- mm_kwargs = self.info.ctx.get_merged_mm_kwargs({})
1156
+ mm_kwargs = self.info.ctx.get_merged_mm_kwargs({}, modality="video")
1157
video_size = mm_kwargs.get("size", video_processor.size)
1158
temporal_patch_size = mm_kwargs.get(
1159
"temporal_patch_size", video_processor.temporal_patch_size
@@ -1365,7 +1368,9 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
1368
# NOTE: a copy of is created to update do_sample_frames,
1369
# otherwise mm_hash for the object will be incorrect.
1370
video_mm_kwargs = dict(**hf_processor_mm_kwargs)
1368
- merged = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
1371
+ merged = self.info.ctx.get_merged_mm_kwargs(
1372
+ hf_processor_mm_kwargs, modality="video"
1373
+ )
1374
if merged.keys() & {"size", "min_pixels", "max_pixels"}:
1375
video_size = dict(self.info.get_video_processor().size)
1376
size_override = merged.get("size")
vllm/multimodal/processing/context.py
+56
-2
@@ -42,6 +42,46 @@ else:
42
43
logger = init_logger(__name__)
44
45
+# HuggingFace processors accept nested ``images_kwargs`` / ``videos_kwargs`` /
46
+# ``audio_kwargs`` in addition to a shared flat namespace. vLLM's token-budget
47
+# and dummy-input code only reads the flat keys, so scoped overrides have to
48
+# be overlaid per modality.
49
+_HF_MODALITY_PROCESSOR_KWARGS = {
50
+ "image": "images_kwargs",
51
+ "video": "videos_kwargs",
52
+ "audio": "audio_kwargs",
53
+}
54
+
55
+
56
+def overlay_modality_mm_kwargs(
57
+ kwargs: Mapping[str, object],
58
+ modality: str | None,
59
+) -> dict[str, Any]:
60
+ """Overlay HF-style nested processor kwargs onto the flat namespace.
61
+
62
+ Overlay only the requested modality so a video ``size`` bump does not leak
63
+ into the image budget. Flat keys keep their current shared-namespace
64
+ behavior when no scoped dict is present.
65
+
66
+ Args:
67
+ kwargs: Merged multi-modal processor kwargs.
68
+ modality: Target modality (``image``, ``video``, or ``audio``).
69
+ When ``None``, ``kwargs`` is copied without overlay.
70
+
71
+ Returns:
72
+ A new dict with the modality-scoped keys overlaid when present.
73
+ """
74
+ merged = dict(kwargs)
75
+ if modality is None:
76
+ return merged
77
+ scoped_key = _HF_MODALITY_PROCESSOR_KWARGS.get(modality)
78
+ if scoped_key is None:
79
+ return merged
80
+ scoped = merged.get(scoped_key)
81
+ if not isinstance(scoped, Mapping):
82
+ return merged
83
+ return merged | dict(scoped)
84
+
85
86
@dataclass
87
class TimingContext:
@@ -253,9 +293,23 @@ class InputProcessingContext:
293
294
return json_map_leaves(_postprocess_one, output)
295
256
- def get_merged_mm_kwargs(self, kwargs: Mapping[str, object]):
296
+ def get_merged_mm_kwargs(
297
+ self,
298
+ kwargs: Mapping[str, object],
299
+ *,
300
+ modality: str | None = None,
301
+ ) -> dict[str, Any]:
302
+ """Merge configured and request ``mm_processor_kwargs``.
303
+
304
+ When ``modality`` is set, HF-style nested
305
+ ``images_kwargs`` / ``videos_kwargs`` / ``audio_kwargs`` are overlaid
306
+ onto the flat namespace for vLLM-side reads (token budgets, dummy
307
+ inputs). Processor construction and HF ``__call__`` should omit
308
+ ``modality`` so the nested dicts still reach the HF processor.
309
+ """
310
mm_config = self.model_config.get_multimodal_config()
258
- return mm_config.merge_mm_processor_kwargs(kwargs)
311
+ merged = mm_config.merge_mm_processor_kwargs(kwargs)
312
+ return overlay_modality_mm_kwargs(merged, modality)
313
314
def call_hf_processor(
315
self,