A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 16:30 UTC
[Misc] Standardize handling of `mm_processor_kwargs.size` (#35284)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Feb 26, 2026 at 21:05 UTC
845ee348ef82d12b5d106384070f7578c843d3cd
9 files changed
+135
-28
tests/lora/test_qwenvl.py
+18
-5
@@ -2,6 +2,9 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from dataclasses import dataclass
4
5
+from packaging.version import Version
6
+from transformers import __version__ as TRANSFORMERS_VERSION
7
+
8
import vllm
9
from vllm.assets.image import ImageAsset
10
from vllm.lora.request import LoRARequest
@@ -18,15 +21,25 @@ class TestConfig:
21
enable_tower_connector_lora: bool = False
22
max_model_len: int = 8192
23
gpu_memory_utilization: float = 0.85
21
- mm_processor_kwargs: dict[str, int] | None = None
24
+ mm_processor_kwargs: dict[str, object] | None = None
25
mm_processor_cache_gb: float = 4
26
27
def __post_init__(self):
28
if self.mm_processor_kwargs is None:
26
- self.mm_processor_kwargs = {
27
- "min_pixels": 28 * 28,
28
- "max_pixels": 1280 * 28 * 28,
29
- }
29
+ # There is a bug in transformers v4 where size is ignored by
30
+ # `Qwen2VLProcessor.__call__`
31
+ if Version(TRANSFORMERS_VERSION) < Version("5.2.0"):
32
+ self.mm_processor_kwargs = {
33
+ "min_pixels": 28 * 28,
34
+ "max_pixels": 1280 * 28 * 28,
35
+ }
36
+ else:
37
+ self.mm_processor_kwargs = {
38
+ "size": {
39
+ "shortest_edge": 28 * 28,
40
+ "longest_edge": 1280 * 28 * 28,
41
+ }
42
+ }
43
44
45
class Qwen2VLTester:
tests/models/multimodal/processing/test_gemma3.py
+7
-5
@@ -150,8 +150,11 @@ class TestGemma3nAudioTensorLogic:
150
151
152
@pytest.mark.parametrize("model_id", [GEMMA3_MODEL_ID])
153
+@pytest.mark.parametrize("mm_processor_kwargs", [{}])
154
def test_get_image_size_with_most_features(
154
- image_assets: ImageTestAssets, model_id: str
155
+ image_assets: ImageTestAssets,
156
+ model_id: str,
157
+ mm_processor_kwargs: dict[str, object],
158
):
159
ctx = build_model_context(
160
model_id,
@@ -160,15 +163,14 @@ def test_get_image_size_with_most_features(
163
)
164
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
165
163
- hf_processor_mm_kwargs: dict[str, object] = {}
164
- hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
166
+ hf_processor = processor.info.get_hf_processor(**mm_processor_kwargs)
167
168
max_image_size = processor.info.get_image_size_with_most_features()
169
max_tokens = processor.info.get_num_image_tokens(
170
image_width=max_image_size.width,
171
image_height=max_image_size.height,
172
processor=hf_processor,
171
- mm_kwargs=hf_processor_mm_kwargs,
173
+ mm_kwargs=mm_processor_kwargs,
174
)
175
176
prompt = "<start_of_image>"
@@ -179,7 +181,7 @@ def test_get_image_size_with_most_features(
181
processed_inputs = processor(
182
prompt,
183
mm_items=processor.info.parse_mm_data(mm_data),
182
- hf_processor_mm_kwargs=hf_processor_mm_kwargs,
184
+ hf_processor_mm_kwargs=mm_processor_kwargs,
185
)
186
mm_kwargs_data = processed_inputs["mm_kwargs"].get_data()
187
num_patches_tensor = mm_kwargs_data["num_patches"]
tests/models/multimodal/processing/test_qwen2_vl.py
+38
-7
@@ -2,6 +2,8 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
import pytest
5
+from packaging.version import Version
6
+from transformers import __version__ as TRANSFORMERS_VERSION
7
8
from vllm.multimodal import MULTIMODAL_REGISTRY
9
@@ -15,6 +17,16 @@ from ...utils import build_model_context
17
[
18
({}, 1426, (5704, 1176)),
19
({"min_pixels": 64**2, "max_pixels": 512**2}, 330, (1320, 1176)),
20
+ (
21
+ {
22
+ "size": {
23
+ "shortest_edge": 64**2,
24
+ "longest_edge": 512**2,
25
+ },
26
+ },
27
+ 330,
28
+ (1320, 1176),
29
+ ),
30
],
31
)
32
@pytest.mark.parametrize("num_imgs", [1, 2])
@@ -29,6 +41,12 @@ def test_processor_override(
41
kwargs_on_init: bool,
42
):
43
"""Ensure Qwen2VLMultiModalProcessor handles min/max pixels properly."""
44
+ if (
45
+ Version(TRANSFORMERS_VERSION) < Version("5.2.0")
46
+ and "size" in mm_processor_kwargs
47
+ ):
48
+ pytest.skip("`size` ignored by `Qwen2VLProcessor.__call__`")
49
+
50
ctx = build_model_context(
51
model_id,
52
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
@@ -60,21 +78,34 @@ def test_processor_override(
78
79
80
@pytest.mark.parametrize("model_id", ["Qwen/Qwen2-VL-2B-Instruct"])
63
-@pytest.mark.parametrize("max_pixels", [1280 * 28 * 28, 1283 * 28 * 28])
81
+@pytest.mark.parametrize(
82
+ "mm_processor_kwargs",
83
+ [
84
+ {"min_pixels": 28 * 28, "max_pixels": 1280 * 28 * 28},
85
+ {"min_pixels": 28 * 28, "max_pixels": 1283 * 28 * 28},
86
+ {"size": {"shortest_edge": 28 * 28, "longest_edge": 1280 * 28 * 28}},
87
+ {"size": {"shortest_edge": 28 * 28, "longest_edge": 1283 * 28 * 28}},
88
+ ],
89
+)
90
def test_get_image_size_with_most_features(
91
image_assets: ImageTestAssets,
92
model_id: str,
67
- max_pixels: int,
93
+ mm_processor_kwargs: dict[str, object],
94
):
95
+ if (
96
+ Version(TRANSFORMERS_VERSION) < Version("5.2.0")
97
+ and "size" in mm_processor_kwargs
98
+ ):
99
+ pytest.skip("`size` ignored by `Qwen2VLProcessor.__call__`")
100
+
101
ctx = build_model_context(
102
model_id,
71
- mm_processor_kwargs={"max_pixels": max_pixels},
103
+ mm_processor_kwargs=mm_processor_kwargs,
104
limit_mm_per_prompt={"image": 1},
105
)
106
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
107
76
- hf_processor_mm_kwargs: dict[str, object] = {}
77
- hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
108
+ hf_processor = processor.info.get_hf_processor(**mm_processor_kwargs)
109
merge_size = processor.info.get_hf_config().vision_config.spatial_merge_size
110
111
max_image_size = processor.info.get_image_size_with_most_features()
@@ -82,7 +113,7 @@ def test_get_image_size_with_most_features(
113
image_width=max_image_size.width,
114
image_height=max_image_size.height,
115
image_processor=hf_processor.image_processor,
85
- mm_kwargs=hf_processor_mm_kwargs,
116
+ mm_kwargs=mm_processor_kwargs,
117
)
118
119
prompt = "<|vision_start|><|image_pad|><|vision_end|>"
@@ -91,7 +122,7 @@ def test_get_image_size_with_most_features(
122
processed_inputs = processor(
123
prompt,
124
mm_items=processor.info.parse_mm_data(mm_data),
94
- hf_processor_mm_kwargs=hf_processor_mm_kwargs,
125
+ hf_processor_mm_kwargs=mm_processor_kwargs,
126
)
127
grid_thw = processed_inputs["mm_kwargs"].get_data()["image_grid_thw"].tolist()
128
t, h, w = grid_thw[0]
vllm/model_executor/models/ernie45_vl.py
+18
-3
@@ -829,16 +829,31 @@ class Ernie4_5_VLProcessingInfo(BaseProcessingInfo):
829
spatial_conv_size = hf_config.spatial_conv_size
830
temporal_conv_size = hf_config.temporal_conv_size
831
832
+ if self.ctx.model_config.trust_remote_code:
833
+ # Defined in HF Hub repo
834
+ min_pixels_key = "min_pixels"
835
+ max_pixels_key = "max_pixels"
836
+ else:
837
+ # Defined in Transformers library (requires v5.0 or above)
838
+ min_pixels_key = "shortest_edge"
839
+ max_pixels_key = "longest_edge"
840
+
841
mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs)
833
- size = mm_kwargs.get("size", image_processor.size)
842
+ size = image_processor.size
843
+ if override_size := mm_kwargs.get("size"):
844
+ size = size | override_size
845
+ if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None:
846
+ size = size | {min_pixels_key: override_min_pixels}
847
+ if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
848
+ size = size | {max_pixels_key: override_max_pixels}
849
850
if do_resize:
851
resized_height, resized_width = smart_resize(
852
height=image_height,
853
width=image_width,
854
factor=patch_size * spatial_conv_size,
840
- min_pixels=size["min_pixels"],
841
- max_pixels=size["max_pixels"],
855
+ min_pixels=size[min_pixels_key],
856
+ max_pixels=size[max_pixels_key],
857
)
858
preprocessed_size = ImageSize(width=resized_width, height=resized_height)
859
else:
vllm/model_executor/models/hunyuan_vision.py
+7
-1
@@ -636,7 +636,13 @@ class HunYuanVLProcessingInfo(BaseProcessingInfo):
636
spatial_merge_size = vision_config.spatial_merge_size
637
638
mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs)
639
- size = mm_kwargs.get("size", image_processor.size)
639
+ size = image_processor.size
640
+ if override_size := mm_kwargs.get("size"):
641
+ size = size | override_size
642
+ if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None:
643
+ size = size | {"shortest_edge": override_min_pixels}
644
+ if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
645
+ size = size | {"longest_edge": override_max_pixels}
646
647
if do_resize:
648
resized_height, resized_width = smart_resize(
vllm/model_executor/models/keye.py
+7
-1
@@ -1021,7 +1021,13 @@ class KeyeProcessingInfo(BaseProcessingInfo):
1021
temporal_patch_size = 1
1022
1023
mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs)
1024
- size = mm_kwargs.get("size", image_processor.size)
1024
+ size = image_processor.size
1025
+ if override_size := mm_kwargs.get("size"):
1026
+ size = size | override_size
1027
+ if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None:
1028
+ size = size | {"min_pixels": override_min_pixels}
1029
+ if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
1030
+ size = size | {"max_pixels": override_max_pixels}
1031
1032
if do_resize:
1033
resized_height, resized_width = smart_resize(
vllm/model_executor/models/paddleocr_vl.py
+18
-3
@@ -155,15 +155,30 @@ class PaddleOCRVLProcessingInfo(BaseProcessingInfo):
155
patch_size = vision_config.patch_size
156
merge_size = vision_config.spatial_merge_size
157
158
+ if self.ctx.model_config.trust_remote_code:
159
+ # Defined in HF Hub repo
160
+ min_pixels_key = "min_pixels"
161
+ max_pixels_key = "max_pixels"
162
+ else:
163
+ # Defined in Transformers library (requires v5.0 or above)
164
+ min_pixels_key = "shortest_edge"
165
+ max_pixels_key = "longest_edge"
166
+
167
mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs)
159
- size = mm_kwargs.get("size", image_processor.size)
168
+ size = image_processor.size
169
+ if override_size := mm_kwargs.get("size"):
170
+ size = size | override_size
171
+ if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None:
172
+ size = size | {min_pixels_key: override_min_pixels}
173
+ if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
174
+ size = size | {max_pixels_key: override_max_pixels}
175
176
resized_height, resized_width = smart_resize(
177
height=image_height,
178
width=image_width,
179
factor=patch_size * merge_size,
165
- min_pixels=size["min_pixels"],
166
- max_pixels=size["max_pixels"],
180
+ min_pixels=size[min_pixels_key],
181
+ max_pixels=size[max_pixels_key],
182
)
183
preprocessed_size = ImageSize(width=resized_width, height=resized_height)
184
vllm/model_executor/models/qwen2_vl.py
+15
-2
@@ -843,7 +843,13 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
843
temporal_patch_size = vision_config.temporal_patch_size
844
845
mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs)
846
- size = mm_kwargs.get("size", image_processor.size)
846
+ size = image_processor.size
847
+ if override_size := mm_kwargs.get("size"):
848
+ size = size | override_size
849
+ if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None:
850
+ size = size | {"shortest_edge": override_min_pixels}
851
+ if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
852
+ size = size | {"longest_edge": override_max_pixels}
853
854
if do_resize:
855
resized_height, resized_width = smart_resize(
@@ -930,7 +936,14 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
936
image_processor = self.get_image_processor()
937
938
mm_kwargs = self.ctx.get_merged_mm_kwargs({})
933
- size = mm_kwargs.get("size", image_processor.size)
939
+ size = image_processor.size
940
+ if override_size := mm_kwargs.get("size"):
941
+ size = size | override_size
942
+ if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None:
943
+ size = size | {"shortest_edge": override_min_pixels}
944
+ if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
945
+ size = size | {"longest_edge": override_max_pixels}
946
+
947
max_pixels = size["longest_edge"]
948
949
unit = patch_size * merge_size
vllm/model_executor/models/qwen3_vl.py
+7
-1
@@ -647,7 +647,13 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
647
temporal_patch_size = vision_config.temporal_patch_size
648
649
mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs)
650
- size = mm_kwargs.get("size", image_processor.size)
650
+ size = image_processor.size
651
+ if override_size := mm_kwargs.get("size"):
652
+ size = size | override_size
653
+ if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None:
654
+ size = size | {"shortest_edge": override_min_pixels}
655
+ if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
656
+ size = size | {"longest_edge": override_max_pixels}
657
658
if do_resize:
659
if is_video: