A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 08:45 UTC
[Bugfix] Standardize getting number of image patches/tokens (#34358)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Feb 13, 2026 at 12:47 UTC
372b2e762aeeb040e57a690f0aa0428775a1e239
29 files changed
+320
-332
tests/kernels/core/test_mrope.py
+2
-22
@@ -4,8 +4,6 @@ from typing import NamedTuple
4
5
import pytest
6
import torch
7
-from packaging.version import Version
8
-from transformers import __version__ as TRANSFORMERS_VERSION
7
8
from vllm.model_executor.layers.rotary_embedding import get_rope
9
from vllm.platforms import current_platform
@@ -46,31 +44,13 @@ class MRoPETestInfo(NamedTuple):
44
marks: list[pytest.MarkDecorator] = []
45
46
49
-TRANSFORMERS_BASE_VERSION = Version(TRANSFORMERS_VERSION).base_version
50
-
47
MODELS_TO_TEST = [
48
MRoPETestInfo(model_name="zai-org/GLM-4.1V-9B-Thinking"),
49
MRoPETestInfo(model_name="Qwen/Qwen2-VL-7B-Instruct"),
50
MRoPETestInfo(model_name="Qwen/Qwen2-VL-72B-Instruct"),
51
MRoPETestInfo(model_name="Qwen/Qwen2.5-VL-72B-Instruct"),
56
- MRoPETestInfo(
57
- model_name="Qwen/Qwen3-VL-4B-Instruct",
58
- marks=[
59
- pytest.mark.skipif(
60
- Version(TRANSFORMERS_BASE_VERSION) < Version("4.57.0"),
61
- reason="Qwen3-VL only available after Transformers v4.57",
62
- )
63
- ],
64
- ),
65
- MRoPETestInfo(
66
- model_name="Qwen/Qwen3-VL-30B-A3B-Instruct",
67
- marks=[
68
- pytest.mark.skipif(
69
- Version(TRANSFORMERS_BASE_VERSION) < Version("4.57.0"),
70
- reason="Qwen3-VL only available after Transformers v4.57",
71
- )
72
- ],
73
- ),
52
+ MRoPETestInfo(model_name="Qwen/Qwen3-VL-4B-Instruct"),
53
+ MRoPETestInfo(model_name="Qwen/Qwen3-VL-30B-A3B-Instruct"),
54
]
55
56
num_tokens_list = [11, 8192]
tests/models/multimodal/generation/test_common.py
-6
@@ -961,12 +961,6 @@ VLM_TEST_SETTINGS = {
961
limit_mm_per_prompt={"image": 4},
962
)
963
],
964
- marks=[
965
- pytest.mark.skipif(
966
- Version(TRANSFORMERS_VERSION) == Version("4.57.1"),
967
- reason="This model is broken in Transformers v4.57.1",
968
- )
969
- ],
964
),
965
# regression test for https://github.com/vllm-project/vllm/issues/15122
966
"qwen2_5_vl-windows-attention": VLMTestInfo(
tests/models/multimodal/processing/test_gemma3.py
+1
@@ -168,6 +168,7 @@ def test_get_image_size_with_most_features(
168
image_width=max_image_size.width,
169
image_height=max_image_size.height,
170
processor=hf_processor,
171
+ mm_kwargs=hf_processor_mm_kwargs,
172
)
173
174
prompt = "<start_of_image>"
tests/models/multimodal/processing/test_idefics3.py
+11
-1
@@ -3,7 +3,9 @@
3
"""Tests for Idefics3's multimodal preprocessing kwargs."""
4
5
import pytest
6
+from packaging.version import Version
7
from transformers import Idefics3Config
8
+from transformers import __version__ as TRANSFORMERS_VERSION
9
10
from vllm.multimodal import MULTIMODAL_REGISTRY
11
@@ -11,6 +13,10 @@ from ....conftest import ImageTestAssets
13
from ...utils import build_model_context
14
15
16
+@pytest.mark.skipif(
17
+ Version(TRANSFORMERS_VERSION) < Version("5.2.0"),
18
+ reason="See https://github.com/huggingface/transformers/pull/43948",
19
+)
20
@pytest.mark.parametrize("model_id", ["HuggingFaceM4/Idefics3-8B-Llama3"])
21
@pytest.mark.parametrize(
22
("mm_processor_kwargs", "expected_toks_per_img"),
@@ -63,7 +69,11 @@ def test_processor_override(
69
70
# Ensure the placeholders format are correct
71
hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
66
- hf_processed_inputs = hf_processor(text=prompt, images=mm_data["image"])
72
+ hf_processed_inputs = hf_processor(
73
+ text=prompt,
74
+ images=mm_data["image"],
75
+ **processor.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs),
76
+ )
77
assert processed_inputs["prompt_token_ids"] == hf_processed_inputs["input_ids"][0]
78
79
# Ensure we have the right number of placeholders per num_crops size
tests/models/multimodal/processing/test_qwen2_vl.py
+1
@@ -82,6 +82,7 @@ def test_get_image_size_with_most_features(
82
image_width=max_image_size.width,
83
image_height=max_image_size.height,
84
image_processor=hf_processor.image_processor,
85
+ mm_kwargs=hf_processor_mm_kwargs,
86
)
87
88
prompt = "<|vision_start|><|image_pad|><|vision_end|>"
tests/models/multimodal/processing/test_smolvlm.py
+11
-1
@@ -3,7 +3,9 @@
3
"""Tests for smolvlm's multimodal preprocessing kwargs."""
4
5
import pytest
6
+from packaging.version import Version
7
from transformers import SmolVLMConfig
8
+from transformers import __version__ as TRANSFORMERS_VERSION
9
10
from vllm.multimodal import MULTIMODAL_REGISTRY
11
@@ -11,6 +13,10 @@ from ....conftest import ImageTestAssets
13
from ...utils import build_model_context
14
15
16
+@pytest.mark.skipif(
17
+ Version(TRANSFORMERS_VERSION) < Version("5.2.0"),
18
+ reason="See https://github.com/huggingface/transformers/pull/43948",
19
+)
20
@pytest.mark.parametrize("model_id", ["HuggingFaceTB/SmolVLM2-2.2B-Instruct"])
21
@pytest.mark.parametrize(
22
("mm_processor_kwargs", "expected_toks_per_img"),
@@ -63,7 +69,11 @@ def test_processor_override(
69
70
# Ensure the placeholders format are correct
71
hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
66
- hf_processed_inputs = hf_processor(text=prompt, images=mm_data["image"])
72
+ hf_processed_inputs = hf_processor(
73
+ text=prompt,
74
+ images=mm_data["image"],
75
+ **processor.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs),
76
+ )
77
assert processed_inputs["prompt_token_ids"] == hf_processed_inputs["input_ids"][0]
78
79
# Ensure we have the right number of placeholders per num_crops size
vllm/model_executor/models/cohere2_vision.py
+10
-31
@@ -11,7 +11,7 @@ from torch import nn
11
from transformers import BatchFeature, PretrainedConfig
12
from transformers.models.cohere2_vision import Cohere2VisionConfig
13
from transformers.models.cohere2_vision.image_processing_cohere2_vision_fast import ( # noqa: E501
14
- get_optimal_tiled_canvas,
14
+ Cohere2VisionImageProcessorFast,
15
)
16
from transformers.models.cohere2_vision.processing_cohere2_vision import (
17
Cohere2VisionProcessor,
@@ -166,43 +166,20 @@ class Cohere2VisionProcessingInfo(BaseProcessingInfo):
166
*,
167
image_width: int,
168
image_height: int,
169
- processor: Cohere2VisionProcessor | None,
169
+ processor: Cohere2VisionProcessor,
170
+ mm_kwargs: Mapping[str, object],
171
) -> int:
172
"""
173
Calculate the number of image patches for a given image.
174
Uses the HF processor to determine the actual number of patches.
175
"""
175
- if processor is None:
176
- processor = self.get_hf_processor()
177
-
178
- image_processor = processor.image_processor
176
+ image_processor: Cohere2VisionImageProcessorFast = processor.image_processor
177
180
- # The current implementation of get_number_of_image_patches
181
- # is incorrect, so we patch it here.
182
- # TODO: Revert once
183
- # https://github.com/huggingface/transformers/pull/40312 is released.
184
- # return image_processor.get_number_of_image_patches(image_height,
185
- # image_width, {})
186
-
187
- min_patches = image_processor.min_patches
188
- max_patches = image_processor.max_patches
189
- patch_size = image_processor.size
190
- crop_to_patches = image_processor.crop_to_patches
191
-
192
- if not crop_to_patches:
193
- return 1
194
-
195
- num_columns, num_rows = get_optimal_tiled_canvas(
196
- (image_height, image_width),
197
- (patch_size["height"], patch_size["width"]),
198
- min_patches,
199
- max_patches,
178
+ return image_processor.get_number_of_image_patches(
179
+ image_height,
180
+ image_width,
181
+ self.ctx.get_merged_mm_kwargs(mm_kwargs),
182
)
201
- num_patches = num_columns * num_rows
202
- if num_patches > 1:
203
- num_patches += 1 # Thumbnail image
204
-
205
- return num_patches
183
184
185
class Cohere2VisionDummyInputsBuilder(
@@ -271,6 +248,7 @@ class Cohere2VisionMultiModalProcessor(
248
image_width=parsed_images.get_image_size(i).width,
249
image_height=parsed_images.get_image_size(i).height,
250
processor=hf_processor,
251
+ mm_kwargs=mm_kwargs,
252
)
253
for i in range(len(parsed_images))
254
]
@@ -311,6 +289,7 @@ class Cohere2VisionMultiModalProcessor(
289
image_width=image_size.width,
290
image_height=image_size.height,
291
processor=hf_processor,
292
+ mm_kwargs=hf_processor_mm_kwargs,
293
)
294
patch_tokens = image_token * img_tokens_per_tile + img_line_break_token
295
repl = f"{boi_token}{patch_tokens * num_patches}{eoi_token}"
vllm/model_executor/models/ernie45_vl.py
+27
-12
@@ -34,7 +34,7 @@ import torch
34
import torch.nn as nn
35
import torch.nn.functional as F
36
from einops import rearrange
37
-from transformers import BatchFeature
37
+from transformers import BaseImageProcessor, BatchFeature
38
39
from vllm.config import VllmConfig
40
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
@@ -818,10 +818,9 @@ class Ernie4_5_VLProcessingInfo(BaseProcessingInfo):
818
image_height: int,
819
num_frames: int = 1,
820
do_resize: bool = True,
821
- image_processor: Any | None,
821
+ image_processor: BaseImageProcessor,
822
+ mm_kwargs: Mapping[str, object],
823
) -> tuple[ImageSize, int]:
823
- if image_processor is None:
824
- image_processor = self.get_image_processor()
824
hf_config = self.get_hf_config()
825
vision_config = hf_config.vision_config
826
@@ -829,13 +828,16 @@ class Ernie4_5_VLProcessingInfo(BaseProcessingInfo):
828
spatial_conv_size = hf_config.spatial_conv_size
829
temporal_conv_size = hf_config.temporal_conv_size
830
831
+ mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs)
832
+ size = mm_kwargs.get("size", image_processor.size)
833
+
834
if do_resize:
835
resized_height, resized_width = smart_resize(
836
height=image_height,
837
width=image_width,
838
factor=patch_size * spatial_conv_size,
837
- min_pixels=image_processor.min_pixels,
838
- max_pixels=image_processor.max_pixels,
839
+ min_pixels=size["min_pixels"],
840
+ max_pixels=size["max_pixels"],
841
)
842
preprocessed_size = ImageSize(width=resized_width, height=resized_height)
843
else:
@@ -855,12 +857,14 @@ class Ernie4_5_VLProcessingInfo(BaseProcessingInfo):
857
*,
858
image_width: int,
859
image_height: int,
858
- image_processor: Any | None,
860
+ image_processor: BaseImageProcessor,
861
+ mm_kwargs: Mapping[str, object],
862
) -> int:
863
_, num_image_tokens = self._get_vision_info(
864
image_width=image_width,
865
image_height=image_height,
866
image_processor=image_processor,
867
+ mm_kwargs=mm_kwargs,
868
)
869
return num_image_tokens
870
@@ -870,35 +874,43 @@ class Ernie4_5_VLProcessingInfo(BaseProcessingInfo):
874
image_width: int,
875
image_height: int,
876
num_frames: int,
873
- image_processor: Any | None,
877
+ image_processor: BaseImageProcessor,
878
+ mm_kwargs: Mapping[str, object],
879
) -> int:
880
_, num_video_tokens = self._get_vision_info(
881
image_width=image_width,
882
image_height=image_height,
883
num_frames=num_frames,
884
image_processor=image_processor,
885
+ mm_kwargs=mm_kwargs,
886
)
887
return num_video_tokens
888
889
def get_image_size_with_most_features(self) -> ImageSize:
890
+ image_processor = self.get_image_processor()
891
+
892
max_image_size, _ = self._get_vision_info(
893
image_width=9999999,
894
image_height=9999999,
887
- image_processor=None,
895
+ image_processor=image_processor,
896
+ mm_kwargs={},
897
)
898
return max_image_size
899
900
def get_max_image_tokens(self) -> int:
901
+ image_processor = self.get_image_processor()
902
target_width, target_height = self.get_image_size_with_most_features()
903
904
num_image_tokens = self.get_num_image_tokens(
905
image_width=target_width,
906
image_height=target_height,
897
- image_processor=None,
907
+ image_processor=image_processor,
908
+ mm_kwargs={},
909
)
910
return num_image_tokens
911
912
def _get_max_video_frames(self, max_tokens: int) -> int:
913
+ image_processor = self.get_image_processor()
914
target_width, target_height = self.get_image_size_with_most_features()
915
916
num_frames = 0
@@ -909,7 +921,8 @@ class Ernie4_5_VLProcessingInfo(BaseProcessingInfo):
921
image_width=target_width,
922
image_height=target_height,
923
num_frames=next_num_frames,
912
- image_processor=None,
924
+ image_processor=image_processor,
925
+ mm_kwargs={},
926
)
927
928
if next_max_tokens > max_tokens:
@@ -942,13 +955,15 @@ class Ernie4_5_VLProcessingInfo(BaseProcessingInfo):
955
seq_len: int,
956
mm_counts: Mapping[str, int],
957
) -> int:
958
+ image_processor = self.get_image_processor()
959
target_width, target_height = self.get_image_size_with_most_features()
960
961
return self.get_num_video_tokens(
962
image_width=target_width,
963
image_height=target_height,
964
num_frames=self.get_num_frames_with_most_features(seq_len, mm_counts),
951
- image_processor=None,
965
+ image_processor=image_processor,
966
+ mm_kwargs={},
967
)
968
969
vllm/model_executor/models/gemma3_mm.py
+40
-52
@@ -7,6 +7,7 @@ from typing import Annotated, Any, Literal
7
import torch
8
from torch import nn
9
from transformers import BatchFeature, Gemma3Config, Gemma3Processor
10
+from transformers.models.gemma3.image_processing_gemma3 import Gemma3ImageProcessor
11
from transformers.models.gemma3.processing_gemma3 import Gemma3ProcessorKwargs
12
13
from vllm.config import VllmConfig
@@ -84,54 +85,35 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):
85
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
86
return {"image": None}
87
87
- def _resolve_image_kwargs(
88
- self,
89
- processor: Gemma3Processor,
90
- keys: set[str],
91
- ) -> dict[str, Any]:
92
- image_processor = processor.image_processor
93
- kwargs = processor._merge_kwargs(
94
- Gemma3ProcessorKwargs,
95
- tokenizer_init_kwargs=processor.tokenizer.init_kwargs,
96
- )
97
-
98
- images_kwargs = kwargs["images_kwargs"]
99
-
100
- def _resolve_kw(key: str):
101
- val = getattr(image_processor, key)
102
- if val is None:
103
- val = images_kwargs[key]
104
-
105
- return val
106
-
107
- return {k: _resolve_kw(k) for k in keys}
108
-
88
def get_num_crops(
89
self,
90
*,
91
image_width: int,
92
image_height: int,
114
- processor: Gemma3Processor | None,
93
+ processor: Gemma3Processor,
94
+ mm_kwargs: Mapping[str, object],
95
) -> int:
116
- if processor is None:
117
- processor = self.get_hf_processor()
118
-
119
- images_kwargs = self._resolve_image_kwargs(
120
- processor,
121
- {
122
- "do_pan_and_scan",
123
- "pan_and_scan_min_crop_size",
124
- "pan_and_scan_max_num_crops",
125
- "pan_and_scan_min_ratio_to_activate",
126
- },
127
- )
96
+ image_processor: Gemma3ImageProcessor = processor.image_processor
97
129
- do_pan_and_scan = images_kwargs["do_pan_and_scan"]
130
- pan_and_scan_min_crop_size = images_kwargs["pan_and_scan_min_crop_size"]
131
- pan_and_scan_max_num_crops = images_kwargs["pan_and_scan_max_num_crops"]
132
- pan_and_scan_min_ratio_to_activate = images_kwargs[
133
- "pan_and_scan_min_ratio_to_activate"
134
- ]
98
+ images_kwargs = processor._merge_kwargs(
99
+ Gemma3ProcessorKwargs,
100
+ tokenizer_init_kwargs=processor.tokenizer.init_kwargs,
101
+ **self.ctx.get_merged_mm_kwargs(mm_kwargs),
102
+ )["images_kwargs"]
103
+
104
+ do_pan_and_scan = images_kwargs.get(
105
+ "do_pan_and_scan", image_processor.do_pan_and_scan
106
+ )
107
+ pan_and_scan_min_crop_size = images_kwargs.get(
108
+ "pan_and_scan_min_crop_size", image_processor.pan_and_scan_min_crop_size
109
+ )
110
+ pan_and_scan_max_num_crops = images_kwargs.get(
111
+ "pan_and_scan_max_num_crops", image_processor.pan_and_scan_max_num_crops
112
+ )
113
+ pan_and_scan_min_ratio_to_activate = images_kwargs.get(
114
+ "pan_and_scan_min_ratio_to_activate",
115
+ image_processor.pan_and_scan_min_ratio_to_activate,
116
+ )
117
118
if not do_pan_and_scan:
119
return 0
@@ -180,17 +162,16 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):
162
*,
163
image_width: int,
164
image_height: int,
183
- processor: Gemma3Processor | None,
165
+ processor: Gemma3Processor,
166
+ mm_kwargs: Mapping[str, object],
167
) -> PromptUpdateDetails[str]:
185
- if processor is None:
186
- processor = self.get_hf_processor()
187
-
168
boi_token = processor.boi_token
169
170
num_crops = self.get_num_crops(
171
image_width=image_width,
172
image_height=image_height,
173
processor=processor,
174
+ mm_kwargs=mm_kwargs,
175
)
176
177
if num_crops == 0:
@@ -215,15 +196,14 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):
196
*,
197
image_width: int,
198
image_height: int,
218
- processor: Gemma3Processor | None,
199
+ processor: Gemma3Processor,
200
+ mm_kwargs: Mapping[str, object],
201
) -> int:
220
- if processor is None:
221
- processor = self.get_hf_processor()
222
-
202
num_crops = self.get_num_crops(
203
image_width=image_width,
204
image_height=image_height,
205
processor=processor,
206
+ mm_kwargs=mm_kwargs,
207
)
208
image_seq_len = processor.image_seq_length
209
@@ -231,11 +211,17 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):
211
212
def get_image_size_with_most_features(self) -> ImageSize:
213
processor = self.get_hf_processor()
214
+ image_processor: Gemma3ImageProcessor = processor.image_processor
215
+
216
+ images_kwargs = processor._merge_kwargs(
217
+ Gemma3ProcessorKwargs,
218
+ tokenizer_init_kwargs=processor.tokenizer.init_kwargs,
219
+ **self.ctx.get_merged_mm_kwargs({}),
220
+ )["images_kwargs"]
221
235
- images_kwargs = self._resolve_image_kwargs(
236
- processor, {"pan_and_scan_max_num_crops"}
222
+ max_num_crops = images_kwargs.get(
223
+ "pan_and_scan_max_num_crops", image_processor.pan_and_scan_max_num_crops
224
)
238
- max_num_crops = images_kwargs["pan_and_scan_max_num_crops"]
225
226
vision_config = self.get_hf_config().vision_config
227
native_size = vision_config.image_size
@@ -303,6 +289,7 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
289
image_width=size.width,
290
image_height=size.height,
291
processor=hf_processor,
292
+ mm_kwargs=mm_kwargs,
293
)
294
for size in image_sizes
295
]
@@ -339,6 +326,7 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
326
image_width=image_size.width,
327
image_height=image_size.height,
328
processor=hf_processor,
329
+ mm_kwargs=hf_processor_mm_kwargs,
330
)
331
332
return [
vllm/model_executor/models/gemma3n_mm.py
+2
-8
@@ -131,7 +131,7 @@ class Gemma3nProcessingInfo(BaseProcessingInfo):
131
*,
132
image_width: int,
133
image_height: int,
134
- processor: Gemma3nProcessor | None,
134
+ processor: Gemma3nProcessor,
135
) -> str:
136
"""
137
Get the replacement text for image tokens.
@@ -139,9 +139,6 @@ class Gemma3nProcessingInfo(BaseProcessingInfo):
139
For Gemma3n, this should return the full_image_sequence which includes
140
BOI token, repeated image tokens, and EOI token.
141
"""
142
- if processor is None:
143
- processor = self.get_hf_processor()
144
-
142
return PromptUpdateDetails.select_token_id(
143
processor.full_image_sequence, processor.image_token_id
144
)
@@ -149,7 +146,7 @@ class Gemma3nProcessingInfo(BaseProcessingInfo):
146
def get_audio_repl(
147
self,
148
*,
152
- processor: Gemma3nProcessor | None,
149
+ processor: Gemma3nProcessor,
150
) -> str:
151
"""
152
Get the replacement text for audio tokens.
@@ -157,9 +154,6 @@ class Gemma3nProcessingInfo(BaseProcessingInfo):
154
For Gemma3n, this should return the full_audio_sequence which includes
155
BOA token, repeated audio tokens, and EOA token.
156
"""
160
- if processor is None:
161
- processor = self.get_hf_processor()
162
-
157
# Return the full audio sequence as defined by the processor
158
return PromptUpdateDetails.select_token_id(
159
processor.full_audio_sequence, processor.audio_token_id
vllm/model_executor/models/h2ovl.py
+1
-4
@@ -424,12 +424,9 @@ class H2OVLProcessingInfo(BaseInternVLProcessingInfo):
424
*,
425
image_width: int,
426
image_height: int,
427
- processor: H2OVLProcessor | None,
427
+ processor: H2OVLProcessor,
428
use_msac: bool | None = None,
429
) -> int:
430
- if processor is None:
431
- processor = self.get_hf_processor()
432
-
430
return processor.get_num_image_tokens(
431
image_width=image_width,
432
image_height=image_height,
vllm/model_executor/models/hunyuan_vision.py
+23
-11
@@ -78,7 +78,10 @@ from vllm.transformers_utils.configs.hunyuan_vl import (
78
HunYuanVLVisionConfig,
79
)
80
from vllm.transformers_utils.processors.hunyuan_vl import HunYuanVLProcessor
81
-from vllm.transformers_utils.processors.hunyuan_vl_image import smart_resize
81
+from vllm.transformers_utils.processors.hunyuan_vl_image import (
82
+ HunYuanVLImageProcessor,
83
+ smart_resize,
84
+)
85
from vllm.utils.tensor_schema import TensorSchema, TensorShape
86
87
from .interfaces import (
@@ -596,7 +599,7 @@ class HunYuanVLProcessingInfo(BaseProcessingInfo):
599
def get_image_processor(
600
self,
601
**kwargs: object,
599
- ) -> HunYuanVLProcessor:
602
+ ) -> HunYuanVLImageProcessor:
603
return self.get_hf_processor(**kwargs).image_processor
604
605
def get_data_parser(self):
@@ -624,23 +627,24 @@ class HunYuanVLProcessingInfo(BaseProcessingInfo):
627
image_height: int,
628
num_frames: int = 1,
629
do_resize: bool = True,
627
- image_processor: HunYuanVLProcessor | None,
630
+ image_processor: HunYuanVLImageProcessor,
631
+ mm_kwargs: Mapping[str, object],
632
) -> tuple[ImageSize, int]:
629
- if image_processor is None:
630
- image_processor = self.get_image_processor()
631
-
633
hf_config = self.get_hf_config()
634
vision_config = hf_config.vision_config
635
patch_size = vision_config.patch_size
636
spatial_merge_size = vision_config.spatial_merge_size
637
638
+ mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs)
639
+ size = mm_kwargs.get("size", image_processor.size)
640
+
641
if do_resize:
642
resized_height, resized_width = smart_resize(
643
height=image_height,
644
width=image_width,
645
factor=patch_size * spatial_merge_size,
642
- min_pixels=image_processor.min_pixels,
643
- max_pixels=image_processor.max_pixels,
646
+ min_pixels=size["shortest_edge"],
647
+ max_pixels=size["longest_edge"],
648
)
649
preprocessed_size = ImageSize(width=resized_width, height=resized_height)
650
else:
@@ -662,29 +666,37 @@ class HunYuanVLProcessingInfo(BaseProcessingInfo):
666
*,
667
image_width: int,
668
image_height: int,
665
- image_processor: HunYuanVLProcessor | None,
669
+ image_processor: HunYuanVLImageProcessor,
670
+ mm_kwargs: Mapping[str, object],
671
) -> int:
672
_, num_image_tokens = self._get_vision_info(
673
image_width=image_width,
674
image_height=image_height,
675
image_processor=image_processor,
676
+ mm_kwargs=mm_kwargs,
677
)
678
return num_image_tokens
679
680
def get_image_size_with_most_features(self) -> ImageSize:
681
+ image_processor = self.get_image_processor()
682
+
683
max_image_size, _ = self._get_vision_info(
684
image_width=512,
685
image_height=8192,
678
- image_processor=None,
686
+ image_processor=image_processor,
687
+ mm_kwargs={},
688
)
689
return max_image_size
690
691
def get_max_image_tokens(self) -> int:
692
+ image_processor = self.get_image_processor()
693
target_width, target_height = self.get_image_size_with_most_features()
694
+
695
return self.get_num_image_tokens(
696
image_width=target_width,
697
image_height=target_height,
687
- image_processor=None,
698
+ image_processor=image_processor,
699
+ mm_kwargs={},
700
)
701
702
vllm/model_executor/models/idefics3.py
+22
-42
@@ -16,7 +16,6 @@
16
# limitations under the License.
17
"""Inference-only Idefics3 model compatible with HuggingFace weights."""
18
19
-import math
19
from collections.abc import Iterable, Mapping, Sequence
20
from typing import Annotated, Literal, TypeAlias
21
@@ -168,54 +167,35 @@ class Idefics3ProcessingInfo(BaseProcessingInfo):
167
*,
168
image_width: int,
169
image_height: int,
171
- processor: Idefics3Processor | None,
172
- ) -> tuple[int, int]:
173
- if processor is None:
174
- processor = self.get_hf_processor()
175
-
170
+ processor: Idefics3Processor,
171
+ mm_kwargs: Mapping[str, object],
172
+ ) -> tuple[int, int, int]:
173
image_processor: Idefics3ImageProcessor = processor.image_processor
174
178
- max_image_size = image_processor.max_image_size["longest_edge"]
179
- size = image_processor.size["longest_edge"]
180
- assert size % max_image_size == 0, (
181
- "`longest_edge` in image_processor's `size` must be divisible by "
182
- "`longest_edge` in `max_image_size`, this may be caused by "
183
- "incorrect mm_kwargs override."
184
- )
185
-
186
- resized_height, resized_width = self._get_resize_output_image_size(
187
- image_width=image_width,
188
- image_height=image_height,
189
- resolution_max_side=size,
175
+ return image_processor.get_number_of_image_patches(
176
+ image_height,
177
+ image_width,
178
+ self.ctx.get_merged_mm_kwargs(mm_kwargs),
179
)
191
- if resized_height > max_image_size or resized_width > max_image_size:
192
- grid_h = math.ceil(resized_height / max_image_size)
193
- grid_w = math.ceil(resized_width / max_image_size)
194
- else:
195
- grid_h = grid_w = 0
196
- return grid_w, grid_h
180
181
def get_num_patches(
182
self,
183
*,
184
image_width: int,
185
image_height: int,
203
- processor: Idefics3Processor | None,
186
+ processor: Idefics3Processor,
187
+ mm_kwargs: Mapping[str, object],
188
) -> int:
205
- grid_w, grid_h = self._get_image_feature_grid_size(
189
+ num_patches, _, _ = self._get_image_feature_grid_size(
190
image_width=image_width,
191
image_height=image_height,
192
processor=processor,
193
+ mm_kwargs=mm_kwargs,
194
)
195
211
- return grid_w * grid_h + 1
212
-
213
- def _get_image_token(
214
- self, processor: Idefics3Processor | None
215
- ) -> tuple[str, str, str]:
216
- if processor is None:
217
- processor = self.get_hf_processor()
196
+ return num_patches
197
198
+ def _get_image_token(self, processor: Idefics3Processor) -> tuple[str, str, str]:
199
image_token = processor.image_token
200
fake_image_token = processor.fake_image_token
201
global_image_token = processor.global_image_tag
@@ -226,11 +206,9 @@ class Idefics3ProcessingInfo(BaseProcessingInfo):
206
*,
207
image_width: int,
208
image_height: int,
229
- processor: Idefics3Processor | None,
209
+ processor: Idefics3Processor,
210
+ mm_kwargs: Mapping[str, object],
211
) -> str:
231
- if processor is None:
232
- processor = self.get_hf_processor()
233
-
212
image_token, fake_image_token, global_img_token = self._get_image_token(
213
processor
214
)
@@ -241,10 +219,11 @@ class Idefics3ProcessingInfo(BaseProcessingInfo):
219
global_img_placeholder = fake_image_token + global_img_token + p_img
220
tile_img_placeholder = fake_image_token + grid_placeholder + p_img
221
244
- grid_w, grid_h = self._get_image_feature_grid_size(
222
+ _, grid_h, grid_w = self._get_image_feature_grid_size(
223
image_width=image_width,
224
image_height=image_height,
225
processor=processor,
226
+ mm_kwargs=mm_kwargs,
227
)
228
if grid_w == 0 and grid_h == 0:
229
return global_img_placeholder + fake_image_token
@@ -272,15 +251,14 @@ class Idefics3ProcessingInfo(BaseProcessingInfo):
251
*,
252
image_width: int,
253
image_height: int,
275
- processor: Idefics3Processor | None,
254
+ processor: Idefics3Processor,
255
+ mm_kwargs: Mapping[str, object],
256
) -> int:
277
- if processor is None:
278
- processor = self.get_hf_processor()
279
-
257
num_patches = self.get_num_patches(
258
image_width=image_width,
259
image_height=image_height,
260
processor=processor,
261
+ mm_kwargs=mm_kwargs,
262
)
263
264
return num_patches * processor.image_seq_len
@@ -353,6 +331,7 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo
331
image_width=size.width,
332
image_height=size.height,
333
processor=hf_processor,
334
+ mm_kwargs=mm_kwargs,
335
)
336
for size in image_sizes
337
]
@@ -398,6 +377,7 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo
377
image_width=image_size.width,
378
image_height=image_size.height,
379
processor=hf_processor,
380
+ mm_kwargs=hf_processor_mm_kwargs,
381
)
382
383
return PromptUpdateDetails.select_text(
vllm/model_executor/models/interns1.py
+13
-13
@@ -197,20 +197,18 @@ class InternS1ProcessingInfo(BaseProcessingInfo):
197
*,
198
image_width: int,
199
image_height: int,
200
- processor: GotOcr2ImageProcessorFast | None = None,
200
+ processor: InternVLProcessor,
201
+ mm_kwargs: Mapping[str, object],
202
) -> int:
202
- if processor is None:
203
- processor = self.get_hf_processor().image_processor
203
+ image_processor: GotOcr2ImageProcessorFast = processor.image_processor
204
205
- if not isinstance(processor, GotOcr2ImageProcessorFast):
206
- raise ValueError(
207
- f"GotOcr2ImageProcessorFast is expected but got {type(processor)}"
208
- )
209
- num_image_patches = processor.get_number_of_image_patches(
210
- image_height, image_width, images_kwargs=dict()
205
+ num_image_patches = image_processor.get_number_of_image_patches(
206
+ image_height,
207
+ image_width,
208
+ self.ctx.get_merged_mm_kwargs(mm_kwargs),
209
)
212
- num_image_tokens = self.get_hf_processor().image_seq_length * num_image_patches
213
- return num_image_tokens
210
+
211
+ return processor.image_seq_length * num_image_patches
212
213
def resolve_target_ratios(self, use_thumbnail: bool | None = None):
214
image_processor = self.get_hf_processor().image_processor
@@ -243,7 +241,8 @@ class InternS1ProcessingInfo(BaseProcessingInfo):
241
feat_size = self.get_num_image_tokens(
242
image_width=width,
243
image_height=height,
246
- processor=processor.image_processor,
244
+ processor=processor,
245
+ mm_kwargs={},
246
)
247
if feat_size > largest_feature_size:
248
largest_feature_size = feat_size
@@ -262,7 +261,8 @@ class InternS1ProcessingInfo(BaseProcessingInfo):
261
return self.get_num_image_tokens(
262
image_width=target_width,
263
image_height=target_height,
265
- processor=processor.image_processor,
264
+ processor=processor,
265
+ mm_kwargs={},
266
)
267
268
def get_num_frames_with_most_features(
vllm/model_executor/models/internvl.py
+1
-4
@@ -705,11 +705,8 @@ class BaseInternVLProcessingInfo(BaseProcessingInfo):
705
*,
706
image_width: int,
707
image_height: int,
708
- processor: BaseInternVLProcessor | None,
708
+ processor: BaseInternVLProcessor,
709
) -> int:
710
- if processor is None:
711
- processor = self.get_hf_processor()
712
-
710
return processor.get_num_image_tokens(
711
image_width=image_width,
712
image_height=image_height,
vllm/model_executor/models/keye.py
+28
-16
@@ -10,7 +10,7 @@ import numpy as np
10
import torch
11
import torch.nn as nn
12
from einops import rearrange
13
-from transformers import PretrainedConfig
13
+from transformers import BaseImageProcessor, PretrainedConfig
14
from transformers.activations import GELUActivation
15
from transformers.feature_extraction_utils import BatchFeature
16
from transformers.modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling
@@ -1011,24 +1011,25 @@ class KeyeProcessingInfo(BaseProcessingInfo):
1011
image_height: int,
1012
num_frames: int = 1,
1013
do_resize: bool = True,
1014
- image_processor,
1014
+ image_processor: BaseImageProcessor,
1015
+ mm_kwargs: Mapping[str, object],
1016
) -> tuple[ImageSize, int]:
1016
- if image_processor is None:
1017
- image_processor = self.get_image_processor()
1018
-
1017
hf_config = self.get_hf_config()
1018
vision_config = hf_config.vision_config
1019
patch_size = vision_config.patch_size
1020
merge_size = vision_config.spatial_merge_size
1021
temporal_patch_size = 1
1022
1023
+ mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs)
1024
+ size = mm_kwargs.get("size", image_processor.size)
1025
+
1026
if do_resize:
1027
resized_height, resized_width = smart_resize(
1028
height=image_height,
1029
width=image_width,
1030
factor=patch_size * merge_size,
1030
- min_pixels=image_processor.min_pixels,
1031
- max_pixels=image_processor.max_pixels,
1031
+ min_pixels=size["min_pixels"],
1032
+ max_pixels=size["max_pixels"],
1033
)
1034
preprocessed_size = ImageSize(width=resized_width, height=resized_height)
1035
else:
@@ -1050,12 +1051,14 @@ class KeyeProcessingInfo(BaseProcessingInfo):
1051
*,
1052
image_width: int,
1053
image_height: int,
1053
- image_processor,
1054
+ image_processor: BaseImageProcessor,
1055
+ mm_kwargs: Mapping[str, object],
1056
) -> int:
1057
_, num_image_tokens = self._get_vision_info(
1058
image_width=image_width,
1059
image_height=image_height,
1060
image_processor=image_processor,
1061
+ mm_kwargs=mm_kwargs,
1062
)
1063
return num_image_tokens
1064
@@ -1065,36 +1068,42 @@ class KeyeProcessingInfo(BaseProcessingInfo):
1068
image_width: int,
1069
image_height: int,
1070
num_frames: int,
1068
- image_processor,
1071
+ image_processor: BaseImageProcessor,
1072
+ mm_kwargs: Mapping[str, object],
1073
) -> int:
1074
_, num_video_tokens = self._get_vision_info(
1075
image_width=image_width,
1076
image_height=image_height,
1077
num_frames=num_frames,
1078
image_processor=image_processor,
1079
+ mm_kwargs=mm_kwargs,
1080
)
1081
return num_video_tokens
1082
1078
- def get_image_size_with_most_features(
1079
- self,
1080
- ) -> ImageSize:
1083
+ def get_image_size_with_most_features(self) -> ImageSize:
1084
+ image_processor = self.get_image_processor()
1085
+
1086
max_image_size, _ = self._get_vision_info(
1087
image_width=self.get_max_image_size(),
1088
image_height=self.get_max_image_size(),
1084
- image_processor=None,
1089
+ image_processor=image_processor,
1090
+ mm_kwargs={},
1091
)
1092
return max_image_size
1093
1094
def get_max_image_tokens(self) -> int:
1095
+ image_processor = self.get_image_processor()
1096
target_width, target_height = self.get_image_size_with_most_features()
1097
1098
return self.get_num_image_tokens(
1099
image_width=target_width,
1100
image_height=target_height,
1094
- image_processor=None,
1101
+ image_processor=image_processor,
1102
+ mm_kwargs={},
1103
)
1104
1105
def _get_max_video_frames(self, max_tokens: int) -> int:
1106
+ image_processor = self.get_image_processor()
1107
target_width, target_height = self.get_image_size_with_most_features()
1108
1109
num_frames = 0
@@ -1105,7 +1114,8 @@ class KeyeProcessingInfo(BaseProcessingInfo):
1114
image_width=target_width,
1115
image_height=target_height,
1116
num_frames=next_num_frames,
1108
- image_processor=None,
1117
+ image_processor=image_processor,
1118
+ mm_kwargs={},
1119
)
1120
1121
if next_max_tokens > max_tokens:
@@ -1130,13 +1140,15 @@ class KeyeProcessingInfo(BaseProcessingInfo):
1140
return max(max_frames_per_video, 1)
1141
1142
def get_max_video_tokens(self, seq_len: int) -> int:
1143
+ image_processor = self.get_image_processor()
1144
target_width, target_height = self.get_image_size_with_most_features()
1145
1146
return self.get_num_video_tokens(
1147
image_width=target_width,
1148
image_height=target_height,
1149
num_frames=self.get_num_frames_with_most_features(seq_len),
1139
- image_processor=None,
1150
+ image_processor=image_processor,
1151
+ mm_kwargs={},
1152
)
1153
1154
vllm/model_executor/models/lfm2_vl.py
+44
-21
@@ -176,7 +176,7 @@ class Lfm2VLProcessingInfo(BaseProcessingInfo):
176
min_tiles: int,
177
max_tiles: int,
178
tile_size: int,
179
- ) -> tuple[int, int]:
179
+ ) -> tuple[int, int, int]:
180
aspect_ratio = width / height
181
target_ratios = self._target_ratios(min_tiles, max_tiles)
182
# find best matching grid configuration
@@ -190,18 +190,27 @@ class Lfm2VLProcessingInfo(BaseProcessingInfo):
190
self,
191
image_width: int,
192
image_height: int,
193
- processor: Lfm2VlProcessor | None,
194
- ) -> tuple[int, int]:
195
- if processor is None:
196
- processor = self.get_image_processor()
193
+ processor: Lfm2VlProcessor,
194
+ mm_kwargs: Mapping[str, object],
195
+ ) -> tuple[int, int, int]:
196
+ image_processor: Lfm2VlImageProcessorFast = processor.image_processor
197
198
- downsample_factor = processor.image_processor.downsample_factor
199
- encoder_patch_size = processor.image_processor.encoder_patch_size
200
- max_pixels_tolerance = processor.image_processor.max_pixels_tolerance
201
- min_tiles = processor.image_processor.min_tiles
202
- max_tiles = processor.image_processor.max_tiles
203
- max_image_tokens = processor.image_processor.max_image_tokens
204
- tile_size = processor.image_processor.tile_size
198
+ mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs)
199
+ downsample_factor = mm_kwargs.get(
200
+ "downsample_factor", image_processor.downsample_factor
201
+ )
202
+ encoder_patch_size = mm_kwargs.get(
203
+ "encoder_patch_size", image_processor.encoder_patch_size
204
+ )
205
+ max_pixels_tolerance = mm_kwargs.get(
206
+ "max_pixels_tolerance", image_processor.max_pixels_tolerance
207
+ )
208
+ min_tiles = mm_kwargs.get("min_tiles", image_processor.min_tiles)
209
+ max_tiles = mm_kwargs.get("max_tiles", image_processor.max_tiles)
210
+ max_image_tokens = mm_kwargs.get(
211
+ "max_image_tokens", image_processor.max_image_tokens
212
+ )
213
+ tile_size = mm_kwargs.get("tile_size", image_processor.tile_size)
214
215
do_image_splitting = not min_tiles == max_tiles == 1
216
is_image_large = self._is_image_too_large(
@@ -235,12 +244,14 @@ class Lfm2VLProcessingInfo(BaseProcessingInfo):
244
*,
245
image_width: int,
246
image_height: int,
238
- processor: Lfm2VlProcessor | None,
247
+ processor: Lfm2VlProcessor,
248
+ mm_kwargs: Mapping[str, object],
249
) -> int:
250
_, _, total_patches = self._get_image_feature_grid_size(
251
image_width=image_width,
252
image_height=image_height,
253
processor=processor,
254
+ mm_kwargs=mm_kwargs,
255
)
256
return total_patches
257
@@ -249,11 +260,9 @@ class Lfm2VLProcessingInfo(BaseProcessingInfo):
260
image_width: int,
261
image_height: int,
262
spatial_shapes: torch.Tensor,
252
- processor: Lfm2VlProcessor | None,
263
+ processor: Lfm2VlProcessor,
264
+ mm_kwargs: Mapping[str, object],
265
) -> str:
254
- if processor is None:
255
- processor = self.get_hf_processor()
256
-
266
grid_placeholder = "<|img_row_{n_h}_col_{n_w}|>"
267
image_token = processor.image_token
268
image_start_token = processor.image_start_token
@@ -263,6 +272,7 @@ class Lfm2VLProcessingInfo(BaseProcessingInfo):
272
num_thumbnail_tokens, num_tokens_per_tile = self.get_num_image_tokens(
273
spatial_shapes=spatial_shapes,
274
processor=processor,
275
+ mm_kwargs=mm_kwargs,
276
)
277
tile_img_placeholder = grid_placeholder + (image_token * num_tokens_per_tile)
278
@@ -270,6 +280,7 @@ class Lfm2VLProcessingInfo(BaseProcessingInfo):
280
image_width=image_width,
281
image_height=image_height,
282
processor=processor,
283
+ mm_kwargs=mm_kwargs,
284
)
285
286
if grid_w > 1 or grid_h > 1:
@@ -295,15 +306,25 @@ class Lfm2VLProcessingInfo(BaseProcessingInfo):
306
self,
307
*,
308
spatial_shapes: torch.Tensor,
298
- processor: Lfm2VlProcessor | None,
309
+ processor: Lfm2VlProcessor,
310
+ mm_kwargs: Mapping[str, object],
311
) -> tuple[int, int]:
300
- tile_size = processor.image_processor.tile_size
301
- downsample_factor = processor.image_processor.downsample_factor
302
- encoder_patch_size = processor.image_processor.encoder_patch_size
312
+ image_processor: Lfm2VlImageProcessorFast = processor.image_processor
313
+
314
+ mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs)
315
+ downsample_factor = mm_kwargs.get(
316
+ "downsample_factor", image_processor.downsample_factor
317
+ )
318
+ encoder_patch_size = mm_kwargs.get(
319
+ "encoder_patch_size", image_processor.encoder_patch_size
320
+ )
321
+ tile_size = mm_kwargs.get("tile_size", image_processor.tile_size)
322
+
323
num_thumbnail_tokens = spatial_shapes[-1].prod() // (downsample_factor**2)
324
num_patches_tile = tile_size // encoder_patch_size
325
dwn_num_patches_tile = math.ceil(num_patches_tile / downsample_factor)
326
num_tiles_tokens = dwn_num_patches_tile * dwn_num_patches_tile
327
+
328
return num_thumbnail_tokens, num_tiles_tokens
329
330
@@ -372,6 +393,7 @@ class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):
393
image_width=size.width,
394
image_height=size.height,
395
processor=hf_processor,
396
+ mm_kwargs=mm_kwargs,
397
)
398
for size in image_sizes
399
]
@@ -414,6 +436,7 @@ class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):
436
image_height=image_size.height,
437
spatial_shapes=spatial_shapes,
438
processor=hf_processor,
439
+ mm_kwargs=hf_processor_mm_kwargs,
440
)
441
return PromptUpdateDetails.select_text(
442
image_repl,
vllm/model_executor/models/molmo.py
+1
-4
@@ -1224,11 +1224,8 @@ class MolmoProcessingInfo(BaseProcessingInfo):
1224
*,
1225
image_width: int,
1226
image_height: int,
1227
- processor: MolmoProcessorWrapper | None,
1227
+ processor: MolmoProcessorWrapper,
1228
) -> int:
1229
- if processor is None:
1230
- processor = self.get_hf_processor()
1231
-
1229
ncols, nrows = processor.get_patches_grid_size(
1230
image_width=image_width,
1231
image_height=image_height,
vllm/model_executor/models/molmo2.py
+18
-14
@@ -1869,12 +1869,9 @@ class Molmo2ProcessingInfo(BaseProcessingInfo):
1869
*,
1870
image_height: int,
1871
image_width: int,
1872
- processor: Molmo2ProcessorWrapper | None = None,
1872
+ processor: Molmo2ProcessorWrapper,
1873
) -> int:
1874
- if processor is None:
1875
- processor = self.get_hf_processor()
1876
-
1877
- hf_processor = processor.processor # type: ignore
1874
+ hf_processor = processor.processor
1875
1876
resize_nrows, resize_cols = processor.get_base_grid_size(is_video=False)
1877
# start/end tokens + image patch token + col tokens
@@ -1897,11 +1894,8 @@ class Molmo2ProcessingInfo(BaseProcessingInfo):
1894
self,
1895
*,
1896
num_frames: int,
1900
- processor: Molmo2ProcessorWrapper | None = None,
1897
+ processor: Molmo2ProcessorWrapper,
1898
) -> int:
1902
- if processor is None:
1903
- processor = self.get_hf_processor()
1904
-
1899
resize_nrows, resize_cols = processor.get_base_grid_size(is_video=True)
1900
# start/end tokens
1901
extra = 2 + resize_nrows * (
@@ -1929,7 +1923,9 @@ class Molmo2ProcessingInfo(BaseProcessingInfo):
1923
width = wr * crop_window_size + total_margin_pixels
1924
1925
feat_size = self.get_num_image_tokens(
1932
- image_height=height, image_width=width, processor=processor
1926
+ image_height=height,
1927
+ image_width=width,
1928
+ processor=processor,
1929
)
1930
if feat_size > largest_feature_size:
1931
largest_feature_size = feat_size
@@ -1940,8 +1936,15 @@ class Molmo2ProcessingInfo(BaseProcessingInfo):
1936
1937
return largest_feature_pinpoint
1938
1943
- def _get_max_video_frames(self, max_tokens: int) -> int:
1944
- num_tokens_per_frame = self.get_num_video_tokens(num_frames=1)
1939
+ def _get_max_video_frames(
1940
+ self,
1941
+ max_tokens: int,
1942
+ processor: Molmo2ProcessorWrapper,
1943
+ ) -> int:
1944
+ num_tokens_per_frame = self.get_num_video_tokens(
1945
+ num_frames=1,
1946
+ processor=processor,
1947
+ )
1948
max_frames = max_tokens // num_tokens_per_frame
1949
return max(max_frames, 1)
1950
@@ -1950,10 +1953,11 @@ class Molmo2ProcessingInfo(BaseProcessingInfo):
1953
seq_len: int,
1954
mm_counts: Mapping[str, int],
1955
) -> int:
1953
- video_processor = self.get_hf_processor().processor.video_processor
1956
+ processor = self.get_hf_processor()
1957
+ video_processor = processor.processor.video_processor
1958
num_frames = video_processor.num_frames
1959
max_videos = mm_counts.get("video", 0)
1956
- max_total_frames = self._get_max_video_frames(seq_len)
1960
+ max_total_frames = self._get_max_video_frames(seq_len, processor)
1961
max_frames_per_video = min(
1962
max_total_frames // max(max_videos, 1),
1963
num_frames,
vllm/model_executor/models/ovis2_5.py
+1
-4
@@ -215,7 +215,7 @@ class Ovis2_5ProcessingInfo(BaseProcessingInfo):
215
image_width: int,
216
image_height: int,
217
num_frames: int = 1,
218
- ) -> tuple[ImageSize, int]:
218
+ ) -> int:
219
hf_config = self.get_hf_config()
220
vit_config = hf_config.vit_config
221
patch_size = vit_config.patch_size
@@ -245,7 +245,6 @@ class Ovis2_5ProcessingInfo(BaseProcessingInfo):
245
image_width=target_width,
246
image_height=target_height,
247
num_frames=next_num_frames,
248
- image_processor=None,
248
)
249
if next_max_tokens > max_tokens:
250
break
@@ -270,7 +269,6 @@ class Ovis2_5ProcessingInfo(BaseProcessingInfo):
269
image_width: int,
270
image_height: int,
271
num_frames: int,
273
- image_processor: BaseImageProcessor | None,
272
) -> int:
273
num_video_tokens = self.get_num_image_tokens(
274
image_width=image_width, image_height=image_height, num_frames=num_frames
@@ -287,7 +285,6 @@ class Ovis2_5ProcessingInfo(BaseProcessingInfo):
285
image_width=target_width,
286
image_height=target_height,
287
num_frames=self.get_num_frames_with_most_features(seq_len, mm_counts),
290
- image_processor=None,
288
)
289
290
vllm/model_executor/models/paddleocr_vl.py
+12
-8
@@ -23,7 +23,7 @@ import numpy as np
23
import torch
24
import torch.nn as nn
25
from einops import rearrange
26
-from transformers import BatchFeature, PretrainedConfig
26
+from transformers import BaseImageProcessor, BatchFeature, PretrainedConfig
27
from transformers.activations import GELUActivation
28
from transformers.modeling_outputs import (
29
BaseModelOutputWithPooling,
@@ -147,21 +147,23 @@ class PaddleOCRVLProcessingInfo(BaseProcessingInfo):
147
*,
148
image_width: int,
149
image_height: int,
150
- image_processor,
150
+ image_processor: BaseImageProcessor,
151
+ mm_kwargs: Mapping[str, object],
152
) -> int:
152
- if image_processor is None:
153
- image_processor = self.get_image_processor()
154
-
153
hf_config = self.get_hf_config()
154
vision_config = hf_config.vision_config
155
patch_size = vision_config.patch_size
156
merge_size = vision_config.spatial_merge_size
157
+
158
+ mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs)
159
+ size = mm_kwargs.get("size", image_processor.size)
160
+
161
resized_height, resized_width = smart_resize(
162
height=image_height,
163
width=image_width,
164
factor=patch_size * merge_size,
163
- min_pixels=image_processor.min_pixels,
164
- max_pixels=image_processor.max_pixels,
165
+ min_pixels=size["min_pixels"],
166
+ max_pixels=size["max_pixels"],
167
)
168
preprocessed_size = ImageSize(width=resized_width, height=resized_height)
169
@@ -176,12 +178,13 @@ class PaddleOCRVLProcessingInfo(BaseProcessingInfo):
178
179
def get_image_size_with_most_features(self) -> ImageSize:
180
hf_config = self.get_hf_config()
181
+ image_processor = self.get_image_processor()
182
183
# See `smart_resize` for the calculation of the image size.
184
merge_size = hf_config.vision_config.spatial_merge_size
185
patch_size = hf_config.vision_config.patch_size
186
factor = merge_size * patch_size
184
- max_num_tokens = self.get_image_processor().max_pixels // (factor**2)
187
+ max_num_tokens = image_processor.max_pixels // (factor**2)
188
# Find factors of max_num_tokens close to its square root
189
# to create a dummy image with a reasonable aspect ratio.
190
h_patches = int(math.sqrt(max_num_tokens))
@@ -276,6 +279,7 @@ class PaddleOCRVLMultiModalProcessor(
279
image_width=image_size.width,
280
image_height=image_size.height,
281
image_processor=image_processor,
282
+ mm_kwargs=hf_processor_mm_kwargs,
283
)
284
285
return [image_token_id] * num_image_tokens
vllm/model_executor/models/phi3v.py
+1
-4
@@ -351,11 +351,8 @@ class Phi3VProcessingInfo(BaseProcessingInfo):
351
*,
352
image_width: int,
353
image_height: int,
354
- processor: ProcessorMixin | None = None,
354
+ processor: ProcessorMixin,
355
) -> int:
356
- if processor is None:
357
- processor = self.get_hf_processor()
358
-
356
return processor.calc_num_image_tokens_from_image_size( # type: ignore
357
width=image_width,
358
height=image_height,
vllm/model_executor/models/phi4mm.py
+9
-9
@@ -558,10 +558,8 @@ class Phi4MMProcessingInfo(BaseProcessingInfo):
558
559
def get_dynamic_hd(
560
self,
561
- processor: ProcessorMixin | None = None,
561
+ processor: ProcessorMixin,
562
) -> int:
563
- if processor is None:
564
- processor = self.get_hf_processor()
563
image_processor = processor.image_processor
564
return image_processor.dynamic_hd
565
@@ -715,7 +713,7 @@ class Phi4MMProcessingInfo(BaseProcessingInfo):
713
*,
714
image_width: int,
715
image_height: int,
718
- processor: ProcessorMixin | None = None,
716
+ processor: ProcessorMixin,
717
) -> int:
718
hf_config = self.get_hf_config()
719
vision_encoder_name = hf_config.img_processor
@@ -739,10 +737,9 @@ class Phi4MMProcessingInfo(BaseProcessingInfo):
737
738
return image_num_tokens
739
742
- def get_image_size_with_most_features(
743
- self,
744
- processor: ProcessorMixin | None = None,
745
- ) -> ImageSize:
740
+ def get_image_size_with_most_features(self) -> ImageSize:
741
+ processor = self.get_hf_processor()
742
+
743
hf_config = self.get_hf_config()
744
vision_encoder_name = hf_config.img_processor
745
if vision_encoder_name is None:
@@ -874,9 +871,12 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
871
prompt, mm_data, mm_kwargs, tok_kwargs
872
)
873
874
+ hf_processor = self.info.get_hf_processor(**mm_kwargs)
875
num_img_tokens = [
876
self.info.get_num_image_tokens(
879
- image_width=img_size[0], image_height=img_size[1]
877
+ image_width=img_size[0],
878
+ image_height=img_size[1],
879
+ processor=hf_processor,
880
)
881
for img_size in processed_outputs["image_sizes"]
882
]
vllm/model_executor/models/pixtral.py
+1
-16
@@ -217,28 +217,13 @@ class PixtralProcessingInfo(BaseProcessingInfo):
217
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
218
return {"image": None}
219
220
- def get_vision_config(
221
- self,
222
- processor: PixtralProcessorAdapter | None = None,
223
- ):
224
- if processor is None:
225
- processor = self.get_hf_processor()
226
-
227
- return PixtralVisionConfig(
228
- image_size=processor.image_size,
229
- patch_size=processor.patch_size,
230
- )
231
-
220
def get_num_image_tokens(
221
self,
222
*,
223
image_width: int,
224
image_height: int,
237
- processor: PixtralProcessorAdapter | None = None,
225
+ processor: PixtralProcessorAdapter,
226
) -> int:
239
- if processor is None:
240
- processor = self.get_hf_processor()
241
-
227
ncols, nrows = processor.image_processor._image_to_num_tokens(
228
Image.new("RGB", (image_width, image_height))
229
)
vllm/model_executor/models/qwen2_vl.py
+22
-11
@@ -832,24 +832,25 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
832
image_height: int,
833
num_frames: int = 1,
834
do_resize: bool = True,
835
- image_processor: Qwen2VLImageProcessor | None,
835
+ image_processor: Qwen2VLImageProcessor,
836
+ mm_kwargs: Mapping[str, object],
837
) -> tuple[ImageSize, int]:
837
- if image_processor is None:
838
- image_processor = self.get_image_processor()
839
-
838
hf_config = self.get_hf_config()
839
vision_config = hf_config.vision_config
840
patch_size = vision_config.patch_size
841
merge_size = vision_config.spatial_merge_size
842
temporal_patch_size = vision_config.temporal_patch_size
843
844
+ mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs)
845
+ size = mm_kwargs.get("size", image_processor.size)
846
+
847
if do_resize:
848
resized_height, resized_width = smart_resize(
849
height=image_height,
850
width=image_width,
851
factor=patch_size * merge_size,
851
- min_pixels=image_processor.size["shortest_edge"],
852
- max_pixels=image_processor.size["longest_edge"],
852
+ min_pixels=size["shortest_edge"],
853
+ max_pixels=size["longest_edge"],
854
)
855
preprocessed_size = ImageSize(width=resized_width, height=resized_height)
856
else:
@@ -873,13 +874,15 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
874
*,
875
image_width: int,
876
image_height: int,
876
- image_processor: Qwen2VLImageProcessor | None,
877
+ image_processor: Qwen2VLImageProcessor,
878
+ mm_kwargs: Mapping[str, object],
879
) -> int:
880
_, num_image_tokens = self._get_vision_info(
881
image_width=image_width,
882
image_height=image_height,
883
num_frames=1,
884
image_processor=image_processor,
885
+ mm_kwargs=mm_kwargs,
886
)
887
return num_image_tokens
888
@@ -889,13 +892,15 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
892
image_width: int,
893
image_height: int,
894
num_frames: int,
892
- image_processor: Qwen2VLImageProcessor | None,
895
+ image_processor: Qwen2VLImageProcessor,
896
+ mm_kwargs: Mapping[str, object],
897
) -> int:
898
_, num_video_tokens = self._get_vision_info(
899
image_width=image_width,
900
image_height=image_height,
901
num_frames=num_frames,
902
image_processor=image_processor,
903
+ mm_kwargs=mm_kwargs,
904
)
905
return num_video_tokens
906
@@ -941,15 +946,18 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
946
return ImageSize(width=unit * width_factor, height=unit * height_factor)
947
948
def get_max_image_tokens(self) -> int:
949
+ image_processor = self.get_image_processor()
950
target_width, target_height = self.get_image_size_with_most_features()
951
952
return self.get_num_image_tokens(
953
image_width=target_width,
954
image_height=target_height,
949
- image_processor=None,
955
+ image_processor=image_processor,
956
+ mm_kwargs={},
957
)
958
959
def _get_max_video_frames(self, max_tokens: int, start_num_frames: int = 1) -> int:
960
+ image_processor = self.get_image_processor()
961
target_width, target_height = self.get_image_size_with_most_features()
962
963
num_frames = start_num_frames
@@ -960,7 +968,8 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
968
image_width=target_width,
969
image_height=target_height,
970
num_frames=next_num_frames,
963
- image_processor=None,
971
+ image_processor=image_processor,
972
+ mm_kwargs={},
973
)
974
975
if next_max_tokens > max_tokens:
@@ -990,13 +999,15 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
999
seq_len: int,
1000
mm_counts: Mapping[str, int],
1001
) -> int:
1002
+ image_processor = self.get_image_processor()
1003
target_width, target_height = self.get_image_size_with_most_features()
1004
1005
return self.get_num_video_tokens(
1006
image_width=target_width,
1007
image_height=target_height,
1008
num_frames=self.get_num_frames_with_most_features(seq_len, mm_counts),
999
- image_processor=None,
1009
+ image_processor=image_processor,
1010
+ mm_kwargs={},
1011
)
1012
1013
vllm/model_executor/models/qwen3_vl.py
+11
-9
@@ -642,13 +642,9 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
642
image_height: int,
643
num_frames: int = 2,
644
do_resize: bool = True,
645
- image_processor: Qwen2VLImageProcessorFast | Qwen3VLVideoProcessor | None,
645
+ image_processor: Qwen2VLImageProcessorFast | Qwen3VLVideoProcessor,
646
+ mm_kwargs: Mapping[str, object],
647
) -> tuple[ImageSize, int]:
647
- if image_processor is None and num_frames > 1:
648
- image_processor = self.get_video_processor()
649
- elif image_processor is None:
650
- image_processor = self.get_image_processor()
651
-
648
is_video = isinstance(image_processor, Qwen3VLVideoProcessor)
649
650
hf_config = self.get_hf_config()
@@ -657,6 +653,9 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
653
merge_size = vision_config.spatial_merge_size
654
temporal_patch_size = vision_config.temporal_patch_size
655
656
+ mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs)
657
+ size = mm_kwargs.get("size", image_processor.size)
658
+
659
if do_resize:
660
if is_video:
661
smart_resize = video_smart_resize
@@ -667,12 +666,13 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
666
else:
667
smart_resize = image_smart_resize
668
extra_kwargs = {}
669
+
670
resized_height, resized_width = smart_resize(
671
height=image_height,
672
width=image_width,
673
factor=patch_size * merge_size,
674
- min_pixels=image_processor.size["shortest_edge"],
675
- max_pixels=image_processor.size["longest_edge"],
674
+ min_pixels=size["shortest_edge"],
675
+ max_pixels=size["longest_edge"],
676
**extra_kwargs,
677
)
678
preprocessed_size = ImageSize(width=resized_width, height=resized_height)
@@ -720,7 +720,8 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
720
image_width=target_width,
721
image_height=target_height,
722
num_frames=2,
723
- image_processor=None,
723
+ image_processor=video_processor,
724
+ mm_kwargs={},
725
)
726
return num_video_soft_tokens
727
@@ -846,6 +847,7 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
847
image_height=target_video_height,
848
num_frames=target_num_frames,
849
image_processor=video_processor,
850
+ mm_kwargs={},
851
)
852
# NOTE: we need to do this check here since Qwen3-VL resizes video
853
# frames depending on how many frames there are.
vllm/model_executor/models/skyworkr1v.py
+1
-4
@@ -487,11 +487,8 @@ class SkyworkR1VProcessingInfo(BaseProcessingInfo):
487
*,
488
image_width: int,
489
image_height: int,
490
- processor: SkyworkR1VProcessor | None,
490
+ processor: SkyworkR1VProcessor,
491
) -> int:
492
- if processor is None:
493
- processor = self.get_hf_processor()
494
-
492
return processor.get_num_image_tokens(
493
image_width=image_width,
494
image_height=image_height,
vllm/model_executor/models/smolvlm.py
+1
-3
@@ -16,9 +16,7 @@ class SmolVLMProcessingInfo(Idefics3ProcessingInfo):
16
def get_hf_processor(self, **kwargs: object) -> SmolVLMProcessor:
17
return self.ctx.get_hf_processor(SmolVLMProcessor, **kwargs)
18
19
- def _get_image_token(self, processor: SmolVLMProcessor | None) -> tuple[str, str]:
20
- if processor is None:
21
- processor = self.get_hf_processor()
19
+ def _get_image_token(self, processor: SmolVLMProcessor) -> tuple[str, str, str]:
20
image_token = processor.image_token
21
fake_image_token = processor.fake_image_token
22
global_image_token = processor.global_image_token
vllm/multimodal/processing/context.py
+5
-2
@@ -409,6 +409,10 @@ class InputProcessingContext:
409
410
return json_map_leaves(_postprocess_one, output)
411
412
+ def get_merged_mm_kwargs(self, kwargs: Mapping[str, object]):
413
+ mm_config = self.model_config.get_multimodal_config()
414
+ return mm_config.merge_mm_processor_kwargs(kwargs)
415
+
416
def call_hf_processor(
417
self,
418
hf_processor: ProcessorMixin,
@@ -424,8 +428,7 @@ class InputProcessingContext:
428
"""
429
assert callable(hf_processor)
430
427
- mm_config = self.model_config.get_multimodal_config()
428
- merged_kwargs = mm_config.merge_mm_processor_kwargs(kwargs)
431
+ merged_kwargs = self.get_merged_mm_kwargs(kwargs)
432
433
allowed_kwargs = get_allowed_kwarg_only_overrides(
434
hf_processor,