A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 15:00 UTC
[Bugfix] Schedule failure due to wrong get_image_size_with_most_features (#29692)
Jaehwang Jung committed
Dec 12, 2025 at 19:27 UTC
f90319d5d14266769b65f0de28ff60b002a65fcc
4 files changed
+117
-9
tests/models/multimodal/processing/test_gemma3.py
new
+42
@@ -0,0 +1,42 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import pytest
5
+
6
+from vllm.multimodal import MULTIMODAL_REGISTRY
7
+
8
+from ....conftest import ImageTestAssets
9
+from ...utils import build_model_context
10
+
11
+
12
+@pytest.mark.parametrize("model_id", ["google/gemma-3-4b-it"])
13
+def test_get_image_size_with_most_features(
14
+ image_assets: ImageTestAssets, model_id: str
15
+):
16
+ ctx = build_model_context(
17
+ model_id,
18
+ mm_processor_kwargs={"do_pan_and_scan": True},
19
+ limit_mm_per_prompt={"image": 1},
20
+ )
21
+ processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
22
+
23
+ hf_processor_mm_kwargs: dict[str, object] = {}
24
+ hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
25
+
26
+ max_image_size = processor.info.get_image_size_with_most_features()
27
+ max_tokens = processor.info.get_num_image_tokens(
28
+ image_width=max_image_size.width,
29
+ image_height=max_image_size.height,
30
+ processor=hf_processor,
31
+ )
32
+
33
+ prompt = "<start_of_image>"
34
+ image_seq_length = hf_processor.image_seq_length
35
+
36
+ for asset in image_assets:
37
+ mm_data = {"image": [asset.pil_image]}
38
+ processed_inputs = processor.apply(prompt, mm_data, hf_processor_mm_kwargs)
39
+ mm_kwargs_data = processed_inputs["mm_kwargs"].get_data()
40
+ num_patches_tensor = mm_kwargs_data["num_patches"]
41
+ tokens = int(num_patches_tensor.item()) * image_seq_length
42
+ assert tokens <= max_tokens
tests/models/multimodal/processing/test_qwen2_vl.py
+35
@@ -53,3 +53,38 @@ def test_processor_override(
53
assert img_tok_count == expected_toks_per_img * num_imgs
54
assert pixel_shape[0] == expected_pixels_shape[0] * num_imgs
55
assert pixel_shape[1] == expected_pixels_shape[1]
56
+
57
+
58
+@pytest.mark.parametrize("model_id", ["Qwen/Qwen2-VL-2B-Instruct"])
59
+@pytest.mark.parametrize("max_pixels", [1280 * 28 * 28, 1283 * 28 * 28])
60
+def test_get_image_size_with_most_features(
61
+ image_assets: ImageTestAssets,
62
+ model_id: str,
63
+ max_pixels: int,
64
+):
65
+ ctx = build_model_context(
66
+ model_id,
67
+ mm_processor_kwargs={"max_pixels": max_pixels},
68
+ limit_mm_per_prompt={"image": 1},
69
+ )
70
+ processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
71
+
72
+ hf_processor_mm_kwargs: dict[str, object] = {}
73
+ hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
74
+ merge_size = processor.info.get_hf_config().vision_config.spatial_merge_size
75
+
76
+ max_image_size = processor.info.get_image_size_with_most_features()
77
+ max_tokens = processor.info.get_num_image_tokens(
78
+ image_width=max_image_size.width,
79
+ image_height=max_image_size.height,
80
+ image_processor=hf_processor.image_processor,
81
+ )
82
+
83
+ prompt = "<|vision_start|><|image_pad|><|vision_end|>"
84
+ for asset in image_assets:
85
+ mm_data = {"image": [asset.pil_image]}
86
+ processed_inputs = processor.apply(prompt, mm_data, hf_processor_mm_kwargs)
87
+ grid_thw = processed_inputs["mm_kwargs"].get_data()["image_grid_thw"].tolist()
88
+ t, h, w = grid_thw[0]
89
+ tokens = (t * h * w) // (merge_size**2)
90
+ assert tokens < max_tokens
vllm/model_executor/models/gemma3_mm.py
+3
-2
@@ -237,8 +237,9 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):
237
)
238
max_num_crops = images_kwargs["pan_and_scan_max_num_crops"]
239
240
- # Result in the max possible feature size (h:w = max_num_crops:1)
241
- return ImageSize(height=50 * max_num_crops, width=50)
240
+ vision_config = self.get_hf_config().vision_config
241
+ native_size = vision_config.image_size
242
+ return ImageSize(height=native_size * max_num_crops, width=native_size)
243
244
245
class Gemma3DummyInputsBuilder(BaseDummyInputsBuilder[Gemma3ProcessingInfo]):
vllm/model_executor/models/qwen2_vl.py
+37
-7
@@ -25,6 +25,7 @@
25
# limitations under the License.
26
"""Inference-only Qwen2-VL model compatible with HuggingFace weights."""
27
28
+import math
29
from collections.abc import Callable, Iterable, Mapping, Sequence
30
from functools import partial
31
from typing import Annotated, Any, Literal, TypeAlias
@@ -959,13 +960,42 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
960
return num_video_tokens
961
962
def get_image_size_with_most_features(self) -> ImageSize:
962
- max_image_size, _ = self._get_vision_info(
963
- image_width=9999999,
964
- image_height=9999999,
965
- num_frames=1,
966
- image_processor=None,
967
- )
968
- return max_image_size
963
+ # NOTE: Simply processing a huge size with _get_vision_info might not give a
964
+ # size that maximizes the number of featrues, i.e., the number of (merged)
965
+ # patches. This is because the number of patches limits the allowed aspect
966
+ # ratios. For example, suppose the maximum number of patches is 1280. A square
967
+ # image cannot be broken down into 1280 patches, so feeding a giant square image
968
+ # into _get_vision_info will not yield a size that maximizes the number of
969
+ # patches. Therefore, we directly factorize the maximum number of patches into
970
+ # height and width. The tricky part is to avoid extreme aspect ratios (>200 for
971
+ # qwen2-vl). If we can't find a suitable aspect ratio, we decrease the number of
972
+ # patches and retry. This is safe because the processor does not accept extreme
973
+ # aspect ratios, so there is no valid post-resize image with the number of
974
+ # patches that yields extreme aspect ratios.
975
+
976
+ hf_config = self.get_hf_config()
977
+ vision_config = hf_config.vision_config
978
+ patch_size = vision_config.patch_size
979
+ merge_size = vision_config.spatial_merge_size
980
+ image_processor = self.get_image_processor()
981
+ max_pixels = image_processor.max_pixels or image_processor.size["longest_edge"]
982
+ unit = patch_size * merge_size
983
+ max_seq_len = max_pixels // (unit * unit)
984
+
985
+ def closest_factor_pair(n: int) -> tuple[int, int]:
986
+ # left <= right
987
+ for d in range(math.isqrt(n), 0, -1):
988
+ if n % d == 0:
989
+ return d, n // d
990
+ return 1, n
991
+
992
+ height_factor, width_factor = 1, max_seq_len
993
+ for seq_len in range(max_seq_len, 0, -1):
994
+ height_factor, width_factor = closest_factor_pair(seq_len)
995
+ if width_factor / height_factor <= 200:
996
+ break
997
+
998
+ return ImageSize(width=unit * width_factor, height=unit * height_factor)
999
1000
def get_max_image_tokens(self) -> int:
1001
target_width, target_height = self.get_image_size_with_most_features()