A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
[VLM] Support pan-and-scan for Gemma3 multi-modal processor (#14672)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu> Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: Roger Wang <ywang@roblox.com> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu> Co-authored-by: Roger Wang <ywang@roblox.com>
Cyrus Leung committed
Mar 13, 2025 at 17:23 UTC
382403921f18c22f6c4b6c8391fd19b9d21a84ee
9 files changed
+315
-81
docs/source/models/supported_models.md
+25
-30
@@ -763,7 +763,7 @@ See [this page](#generative-models) for more information on how to use generativ
763
* `google/gemma-3-4b-it`, `google/gemma-3-27b-it`, etc.
764
* ✅︎
765
* ✅︎
766
- * ✅︎\*
766
+ * ⚠️
767
- * `GLM4VForCausalLM`<sup>^</sup>
768
* GLM-4V
769
* T + I
@@ -856,12 +856,12 @@ See [this page](#generative-models) for more information on how to use generativ
856
* ✅︎
857
* ✅︎
858
- * `PaliGemmaForConditionalGeneration`
859
- * PaliGemma ⚠️, PaliGemma 2 ⚠️
859
+ * PaliGemma, PaliGemma 2
860
* T + I<sup>E</sup>
861
* `google/paligemma-3b-pt-224`, `google/paligemma-3b-mix-224`, `google/paligemma2-3b-ft-docci-448`, etc.
862
*
863
* ✅︎
864
- * ✅︎
864
+ * ⚠️
865
- * `Phi3VForCausalLM`
866
* Phi-3-Vision, Phi-3.5-Vision
867
* T + I<sup>E+</sup>
@@ -926,34 +926,15 @@ See [this page](#generative-models) for more information on how to use generativ
926
<sup>E</sup> Pre-computed embeddings can be inputted for this modality.
927
<sup>+</sup> Multiple items can be inputted per text prompt for this modality.
928
929
-:::{warning}
930
-vLLM does not currently support PrefixLM attention mask, so our PaliGemma implementation uses regular causal attention, which causes the model output to be unstable.
931
-
932
-We may deprecate this model series in a future release.
933
-:::
934
-
935
-:::{note}
936
-`h2oai/h2ovl-mississippi-2b` will be available in V1 once we support backends other than FlashAttention.
937
-:::
938
-
939
-:::{note}
940
-To use `TIGER-Lab/Mantis-8B-siglip-llama3`, you have to pass `--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'` when running vLLM.
941
-:::
942
-
943
-:::{note}
944
-The official `openbmb/MiniCPM-V-2` doesn't work yet, so we need to use a fork (`HwwwH/MiniCPM-V-2`) for now.
945
-For more details, please see: <gh-pr:4087#issuecomment-2250397630>
946
-:::
947
-
948
-:::{note}
949
-To use Qwen2.5-VL series models, you have to install Hugging Face Transformers library from source via `pip install git+https://github.com/huggingface/transformers`.
950
-:::
951
-
952
-:::{note}
929
+:::{important}
930
To use Gemma3 series models, you have to install Hugging Face Transformers library from source via
931
`pip install git+https://github.com/huggingface/transformers`.
955
-The earliest commit that supports this is [`50d3530aa04e7a7d003e6b255a98f79fd0447357`](https://github.com/huggingface/transformers/commit/50d3530aa04e7a7d003e6b255a98f79fd0447357).
932
933
+Pan-and-scan image pre-processing is currently supported on V0 (but not V1).
934
+You can enable it by passing `--mm-processor-kwargs '{"do_pan_and_scan": True}'`.
935
+:::
936
+
937
+:::{warning}
938
Both V0 and V1 support `Gemma3ForConditionalGeneration` for text-only inputs.
939
However, there are differences in how they handle text + image inputs:
940
@@ -969,9 +950,23 @@ V1 currently uses a simplified attention pattern:
950
- Will be updated in the future to support the correct behavior
951
952
This limitation exists because the model's mixed attention pattern (bidirectional for images, causal otherwise) is not yet supported by vLLM's attention backends.
953
+:::
954
+
955
+:::{note}
956
+`h2oai/h2ovl-mississippi-2b` will be available in V1 once we support backends other than FlashAttention.
957
+:::
958
973
-Additionally, vLLM's current Gemma 3 implementation does not support the pan-and-scan image pre-processing algorithm, which helps handle images with skewed aspect ratios by intelligently cropping them into multiple views.
974
-Without this feature, model performance may degrade when processing images that deviate significantly from square dimensions.
959
+:::{note}
960
+To use `TIGER-Lab/Mantis-8B-siglip-llama3`, you have to pass `--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'` when running vLLM.
961
+:::
962
+
963
+:::{note}
964
+The official `openbmb/MiniCPM-V-2` doesn't work yet, so we need to use a fork (`HwwwH/MiniCPM-V-2`) for now.
965
+For more details, please see: <gh-pr:4087#issuecomment-2250397630>
966
+:::
967
+
968
+:::{warning}
969
+Our PaliGemma implementations have the same problem as Gemma 3 (see above) for both V0 and V1.
970
:::
971
972
### Pooling Models
examples/offline_inference/vision_language.py
+8
-4
@@ -123,10 +123,14 @@ def run_gemma3(questions: list[str], modality: str):
123
assert modality == "image"
124
model_name = "google/gemma-3-4b-it"
125
126
- llm = LLM(model=model_name,
127
- max_model_len=2048,
128
- max_num_seqs=2,
129
- disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
126
+ llm = LLM(
127
+ model=model_name,
128
+ max_model_len=2048,
129
+ max_num_seqs=2,
130
+ # Default is False; setting it to True is not supported in V1 yet
131
+ mm_processor_kwargs={"do_pan_and_scan": True},
132
+ disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
133
+ )
134
135
prompts = [("<bos><start_of_turn>user\n"
136
f"<start_of_image>{question}<end_of_turn>\n"
examples/offline_inference/vision_language_multi_image.py
+8
-4
@@ -83,10 +83,14 @@ def load_deepseek_vl2(question: str, image_urls: list[str]):
83
def load_gemma3(question, image_urls: list[str]) -> ModelRequestData:
84
model_name = "google/gemma-3-4b-it"
85
86
- llm = LLM(model=model_name,
87
- max_model_len=8192,
88
- max_num_seqs=2,
89
- limit_mm_per_prompt={"image": len(image_urls)})
86
+ llm = LLM(
87
+ model=model_name,
88
+ max_model_len=8192,
89
+ max_num_seqs=2,
90
+ # Default is False; setting it to True is not supported in V1 yet
91
+ mm_processor_kwargs={"do_pan_and_scan": True},
92
+ limit_mm_per_prompt={"image": len(image_urls)},
93
+ )
94
95
placeholders = [{"type": "image", "image": url} for url in image_urls]
96
messages = [{
tests/models/decoder_only/vision_language/test_models.py
+18
-1
@@ -9,7 +9,7 @@ from pathlib import PosixPath
9
10
import pytest
11
from packaging.version import Version
12
-from transformers import AutoModelForVision2Seq
12
+from transformers import AutoModelForPreTraining, AutoModelForVision2Seq
13
from transformers import __version__ as TRANSFORMERS_VERSION
14
15
from vllm.platforms import current_platform
@@ -234,6 +234,23 @@ VLM_TEST_SETTINGS = {
234
num_logprobs=10,
235
image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
236
),
237
+ "gemma3": VLMTestInfo(
238
+ models=["google/gemma-3-4b-it"],
239
+ test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
240
+ prompt_formatter=lambda img_prompt: f"<bos><start_of_turn>user\n{img_prompt}<end_of_turn>\n<start_of_turn>model\n", # noqa: E501
241
+ single_image_prompts=IMAGE_ASSETS.prompts({
242
+ "stop_sign": "<start_of_image>What's the content in the center of the image?", # noqa: E501
243
+ "cherry_blossom": "<start_of_image>What is the season?", # noqa: E501
244
+ }),
245
+ multi_image_prompt="<start_of_image><start_of_image>Describe the two images in detail.", # noqa: E501
246
+ max_model_len=4096,
247
+ max_num_seqs=2,
248
+ # TODO: Use AutoModelForVision2Seq once transformers supports this
249
+ auto_cls=AutoModelForPreTraining,
250
+ dtype="bfloat16",
251
+ vllm_runner_kwargs={"mm_processor_kwargs": {"do_pan_and_scan": True}},
252
+ patch_hf_runner=model_utils.gemma3_patch_hf_runner,
253
+ ),
254
"glm4v": VLMTestInfo(
255
models=["THUDM/glm-4v-9b"],
256
test_type=VLMTestType.IMAGE,
tests/models/decoder_only/vision_language/vlm_utils/model_utils.py
+12
@@ -304,6 +304,18 @@ def deepseekvl2_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
304
return hf_model
305
306
307
+def gemma3_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
308
+ """Patches and returns an instance of the HfRunner to use for Gemma 3."""
309
+ hf_processor = hf_model.processor
310
+
311
+ def processor(*args, **kwargs):
312
+ return hf_processor(*args, do_pan_and_scan=True, **kwargs)
313
+
314
+ hf_model.processor = processor
315
+
316
+ return hf_model
317
+
318
+
319
def glm_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
320
"""Patches and returns an instance of the HfRunner to use for GLM4."""
321
hf_processor = hf_model.processor
vllm/inputs/registry.py
+8
-1
@@ -348,7 +348,11 @@ class InputRegistry:
348
dummy_factory = self._get_dummy_data_factory(model_cls)
349
mm_counts = mm_registry.get_mm_limits_per_prompt(model_config)
350
mm_processor_kwargs = get_allowed_kwarg_only_overrides(
351
- dummy_factory, overrides=model_config.mm_processor_kwargs)
351
+ dummy_factory,
352
+ overrides=model_config.mm_processor_kwargs,
353
+ requires_kw_only=False,
354
+ allow_var_kwargs=True,
355
+ )
356
357
dummy_data = dummy_factory(InputContext(model_config), seq_len,
358
_MultiModalCounts(mm_counts),
@@ -381,6 +385,7 @@ class InputRegistry:
385
self,
386
ctx: InputContext,
387
inputs: ProcessorInputs,
388
+ **kwargs: object,
389
) -> ProcessorInputs:
390
"""The default input processor is a no-op."""
391
return inputs
@@ -447,6 +452,8 @@ class InputRegistry:
452
model_config.mm_processor_kwargs,
453
inputs.get("mm_processor_kwargs", {}), # type: ignore
454
processor,
455
+ requires_kw_only=False,
456
+ allow_var_kwargs=True,
457
)
458
459
processed_inputs = processor(
vllm/model_executor/models/gemma3_mm.py
+229
-38
@@ -1,10 +1,12 @@
1
# SPDX-License-Identifier: Apache-2.0
2
+import math
3
from typing import (Any, Iterable, Literal, Mapping, Optional, Sequence, Set,
4
Tuple, TypedDict, Union)
5
6
import torch
7
from torch import nn
7
-from transformers import BatchFeature, Gemma3Config, ProcessorMixin
8
+from transformers import BatchFeature, Gemma3Config, Gemma3Processor
9
+from transformers.models.gemma3.processing_gemma3 import Gemma3ProcessorKwargs
10
11
from vllm.config import VllmConfig
12
from vllm.logger import init_logger
@@ -14,10 +16,11 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
16
from vllm.multimodal import MULTIMODAL_REGISTRY
17
from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
18
NestedTensors)
17
-from vllm.multimodal.parse import ImageSize, MultiModalDataItems
19
+from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
20
+ MultiModalDataItems)
21
from vllm.multimodal.processing import (BaseMultiModalProcessor,
22
BaseProcessingInfo, PromptReplacement,
20
- PromptUpdate, PromptUpdateDetails)
23
+ PromptUpdate, encode_tokens)
24
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
25
from vllm.sequence import IntermediateTensors
26
@@ -31,8 +34,15 @@ logger = init_logger(__name__)
34
35
class Gemma3ImagePixelInputs(TypedDict):
36
type: Literal["pixel_values"]
34
- data: torch.Tensor
35
- """Shape: `(batch_size * num_images, num_channels, height, width)`"""
37
+ pixel_values: torch.Tensor
38
+ """
39
+ Shape: `(num_crops_total, num_channels, height, width)`
40
+
41
+ `num_crops_total` is the total number of crops
42
+ over each image over each prompt in the batch.
43
+ """
44
+ num_crops: torch.Tensor
45
+ """Shape: `(batch_size * num_images,)`"""
46
47
48
Gemma3ImageInputs = Gemma3ImagePixelInputs
@@ -40,6 +50,9 @@ Gemma3ImageInputs = Gemma3ImagePixelInputs
50
51
class Gemma3ProcessingInfo(BaseProcessingInfo):
52
53
+ def get_hf_processor(self, **kwargs: object):
54
+ return self.ctx.get_hf_processor(Gemma3Processor, **kwargs)
55
+
56
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
57
return {"image": None}
58
@@ -48,22 +61,160 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):
61
seq_len: int,
62
mm_counts: Mapping[str, int],
63
) -> Mapping[str, int]:
51
- hf_config = self.ctx.get_hf_config()
52
- return {"image": hf_config.mm_tokens_per_image}
64
+ return {"image": self.get_max_image_tokens()}
65
+
66
+ def _resolve_image_kwargs(
67
+ self,
68
+ processor: Gemma3Processor,
69
+ keys: set[str],
70
+ ) -> dict[str, Any]:
71
+ image_processor = processor.image_processor
72
+ kwargs = processor._merge_kwargs(
73
+ Gemma3ProcessorKwargs,
74
+ tokenizer_init_kwargs=processor.tokenizer.init_kwargs,
75
+ )
76
+
77
+ images_kwargs = kwargs["images_kwargs"]
78
+
79
+ def _resolve_kw(key: str):
80
+ val = getattr(image_processor, key)
81
+ if val is None:
82
+ val = images_kwargs[key]
83
+
84
+ return val
85
+
86
+ return {k: _resolve_kw(k) for k in keys}
87
+
88
+ def get_num_crops(
89
+ self,
90
+ *,
91
+ image_width: int,
92
+ image_height: int,
93
+ processor: Optional[Gemma3Processor],
94
+ ) -> int:
95
+ if processor is None:
96
+ processor = self.get_hf_processor()
97
+
98
+ images_kwargs = self._resolve_image_kwargs(
99
+ processor, {
100
+ "do_pan_and_scan", "pan_and_scan_min_crop_size",
101
+ "pan_and_scan_max_num_crops",
102
+ "pan_and_scan_min_ratio_to_activate"
103
+ })
104
+
105
+ do_pan_and_scan = images_kwargs["do_pan_and_scan"]
106
+ pan_and_scan_min_crop_size = images_kwargs[
107
+ "pan_and_scan_min_crop_size"]
108
+ pan_and_scan_max_num_crops = images_kwargs[
109
+ "pan_and_scan_max_num_crops"]
110
+ pan_and_scan_min_ratio_to_activate = images_kwargs[
111
+ "pan_and_scan_min_ratio_to_activate"]
112
+
113
+ if not do_pan_and_scan:
114
+ return 0
115
+
116
+ # Based on Gemma3ImageProcessor.pan_and_scan
117
+ if image_width >= image_height:
118
+ if image_width / image_height < pan_and_scan_min_ratio_to_activate:
119
+ return 0
120
+
121
+ num_crops_w = min(
122
+ int(math.floor(image_width / pan_and_scan_min_crop_size)),
123
+ int(math.floor(image_width / image_height + 0.5)),
124
+ )
125
+
126
+ num_crops_w = max(2, num_crops_w)
127
+ num_crops_w = min(pan_and_scan_max_num_crops, num_crops_w)
128
+ num_crops_h = 1
129
+ else:
130
+ if image_height / image_width < pan_and_scan_min_ratio_to_activate:
131
+ return 0
132
+
133
+ num_crops_h = min(
134
+ int(math.floor(image_height / pan_and_scan_min_crop_size)),
135
+ int(math.floor(image_height / image_width + 0.5)),
136
+ )
137
+
138
+ num_crops_h = max(2, num_crops_h)
139
+ num_crops_h = min(pan_and_scan_max_num_crops, num_crops_h)
140
+ num_crops_w = 1
141
+
142
+ crop_size_w = int(math.ceil(image_width / num_crops_w))
143
+ crop_size_h = int(math.ceil(image_height / num_crops_h))
144
+
145
+ if min(crop_size_w, crop_size_h) < pan_and_scan_min_crop_size:
146
+ return 0
147
+
148
+ return num_crops_w * num_crops_h
149
+
150
+ def get_image_repl(
151
+ self,
152
+ *,
153
+ image_width: int,
154
+ image_height: int,
155
+ processor: Optional[Gemma3Processor],
156
+ ) -> str:
157
+ if processor is None:
158
+ processor = self.get_hf_processor()
159
+
160
+ image_token = processor.boi_token
161
+
162
+ num_crops = self.get_num_crops(
163
+ image_width=image_width,
164
+ image_height=image_height,
165
+ processor=processor,
166
+ )
167
+
168
+ if num_crops == 0:
169
+ image_text = image_token
170
+ else:
171
+ crops_image_tokens = " ".join(image_token
172
+ for _ in range(num_crops))
173
+ image_text = (
174
+ f"Here is the original image {image_token} and here are some "
175
+ f"crops to help you see better {crops_image_tokens}")
176
+
177
+ return image_text.replace(image_token, processor.full_image_sequence)
178
179
def get_num_image_tokens(
180
self,
181
*,
182
image_width: int,
183
image_height: int,
59
- processor: Optional[ProcessorMixin],
184
+ processor: Optional[Gemma3Processor],
185
) -> int:
61
- hf_config = self.ctx.get_hf_config()
62
- return hf_config.mm_tokens_per_image
186
+ tokenizer = self.get_tokenizer()
187
+ image_repl = self.get_image_repl(
188
+ image_width=image_width,
189
+ image_height=image_height,
190
+ processor=processor,
191
+ )
192
+
193
+ image_repl_tokens = encode_tokens(
194
+ tokenizer,
195
+ image_repl,
196
+ add_special_tokens=False,
197
+ )
198
+ return len(image_repl_tokens)
199
200
def get_image_size_with_most_features(self) -> ImageSize:
65
- # Result in the max possible feature size (h:w = 16:1)
66
- return ImageSize(height=8000, width=50)
201
+ processor = self.get_hf_processor()
202
+
203
+ images_kwargs = self._resolve_image_kwargs(
204
+ processor, {"pan_and_scan_max_num_crops"})
205
+ max_num_crops = images_kwargs["pan_and_scan_max_num_crops"]
206
+
207
+ # Result in the max possible feature size (h:w = max_num_crops:1)
208
+ return ImageSize(height=50 * max_num_crops, width=50)
209
+
210
+ def get_max_image_tokens(self) -> int:
211
+ target_width, target_height = self.get_image_size_with_most_features()
212
+
213
+ return self.get_num_image_tokens(
214
+ image_width=target_width,
215
+ image_height=target_height,
216
+ processor=None,
217
+ )
218
219
220
class Gemma3DummyInputsBuilder(BaseDummyInputsBuilder[Gemma3ProcessingInfo]):
@@ -73,10 +224,11 @@ class Gemma3DummyInputsBuilder(BaseDummyInputsBuilder[Gemma3ProcessingInfo]):
224
seq_len: int,
225
mm_counts: Mapping[str, int],
226
) -> ProcessorInputs:
76
- tokenizer = self.info.get_tokenizer()
77
- boi_token = tokenizer.boi_token
227
+ processor = self.info.get_hf_processor()
228
+ image_token = processor.boi_token
229
230
num_images = mm_counts.get("image", 0)
231
+
232
target_width, target_height = \
233
self.info.get_image_size_with_most_features()
234
@@ -86,8 +238,13 @@ class Gemma3DummyInputsBuilder(BaseDummyInputsBuilder[Gemma3ProcessingInfo]):
238
height=target_height,
239
num_images=num_images)
240
}
241
+
242
+ # NOTE: We need to separate the image tokens here because
243
+ # encode("\n\n\n\n") != encode("\n\n") * 2, which interferes
244
+ # with the detection of prompt updates when the image tokens are
245
+ # right next to each other
246
return ProcessorInputs(
90
- prompt_text=" ".join([boi_token] * num_images),
247
+ prompt_text=" ".join([image_token] * num_images),
248
mm_data=mm_data,
249
)
250
@@ -100,22 +257,49 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
257
mm_data: Mapping[str, object],
258
mm_kwargs: Mapping[str, object],
259
) -> BatchFeature:
103
- # TODO(woosuk): Support pan-and-scan.
104
- img_kwargs = mm_kwargs.get("images_kwargs", {})
105
- img_kwargs["do_pan_and_scan"] = False
106
- mm_kwargs["images_kwargs"] = img_kwargs
107
- return super()._call_hf_processor(
108
- prompt=prompt,
109
- mm_data=mm_data,
110
- mm_kwargs=mm_kwargs,
260
+ processed_outputs = super()._call_hf_processor(
261
+ prompt,
262
+ mm_data,
263
+ mm_kwargs,
264
)
265
266
+ # HF processor pops the `num_crops` kwarg, which is needed by vLLM
267
+ if (images := mm_data.get("images")) is not None:
268
+ assert isinstance(images, list)
269
+
270
+ parsed_images = (self._get_data_parser().parse_mm_data({
271
+ "image":
272
+ images
273
+ }).get_items("image", ImageProcessorItems))
274
+ image_sizes = [
275
+ parsed_images.get_image_size(i)
276
+ for i in range(len(parsed_images))
277
+ ]
278
+ hf_processor = self.info.get_hf_processor(**mm_kwargs)
279
+
280
+ num_crops = [
281
+ self.info.get_num_crops(image_width=size.width,
282
+ image_height=size.height,
283
+ processor=hf_processor)
284
+ for size in image_sizes
285
+ ]
286
+
287
+ processed_outputs["num_crops"] = torch.tensor(num_crops)
288
+
289
+ return processed_outputs
290
+
291
def _get_mm_fields_config(
292
self,
293
hf_inputs: BatchFeature,
294
hf_processor_mm_kwargs: Mapping[str, object],
295
) -> Mapping[str, MultiModalFieldConfig]:
118
- return dict(pixel_values=MultiModalFieldConfig.batched("image"))
296
+ num_crops = hf_inputs.get("num_crops", torch.empty(0))
297
+
298
+ return dict(
299
+ pixel_values=MultiModalFieldConfig.flat_from_sizes(
300
+ "image", num_crops + 1),
301
+ num_crops=MultiModalFieldConfig.batched("image"),
302
+ )
303
304
def _get_prompt_updates(
305
self,
@@ -123,25 +307,23 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
307
hf_processor_mm_kwargs: Mapping[str, Any],
308
out_mm_kwargs: MultiModalKwargs,
309
) -> Sequence[PromptUpdate]:
126
- tokenizer = self.info.get_tokenizer()
310
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
128
- hf_config = self.info.get_hf_config()
129
-
130
- boi_token = tokenizer.boi_token
131
- image_token = tokenizer.image_token
132
- mm_tokens_per_image = hf_config.mm_tokens_per_image
133
- image_tokens_expanded = "".join([image_token] * mm_tokens_per_image)
311
+ image_token = hf_processor.boi_token
312
313
def get_replacement_gemma3(item_idx: int):
136
- return PromptUpdateDetails(
137
- full=hf_processor.full_image_sequence,
138
- features=image_tokens_expanded,
314
+ images = mm_items.get_items("image", ImageProcessorItems)
315
+
316
+ image_size = images.get_image_size(item_idx)
317
+ return self.info.get_image_repl(
318
+ image_width=image_size.width,
319
+ image_height=image_size.height,
320
+ processor=hf_processor,
321
)
322
323
return [
324
PromptReplacement(
325
modality="image",
144
- target=boi_token,
326
+ target=image_token,
327
replacement=get_replacement_gemma3,
328
)
329
]
@@ -254,19 +436,27 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal,
436
def _parse_and_validate_image_input(
437
self, **kwargs: object) -> Optional[Gemma3ImageInputs]:
438
pixel_values = kwargs.pop("pixel_values", None)
439
+ num_crops = kwargs.pop("num_crops", None)
440
image_embeds = kwargs.pop("image_embeds", None)
441
assert image_embeds is None, "Gemma3 does not support image_embeds."
442
if pixel_values is None:
443
return None
444
262
- if not isinstance(pixel_values, (torch.Tensor, list[torch.Tensor])):
445
+ if not isinstance(pixel_values, (torch.Tensor, list)):
446
raise ValueError("Incorrect type of pixel values. "
447
f"Got type: {type(pixel_values)}")
448
449
+ if not isinstance(num_crops, (torch.Tensor, list)):
450
+ raise ValueError("Incorrect type of num_crops values. "
451
+ f"Got type: {type(num_crops)}")
452
+
453
pixel_values = flatten_bn(pixel_values, concat=True)
454
+ num_crops = flatten_bn(num_crops, concat=True)
455
+
456
return Gemma3ImagePixelInputs(
457
type="pixel_values",
269
- data=self._validate_pixel_values(pixel_values),
458
+ pixel_values=self._validate_pixel_values(pixel_values),
459
+ num_crops=num_crops,
460
)
461
462
def _image_pixels_to_features(
@@ -283,7 +473,8 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal,
473
image_input: Gemma3ImageInputs,
474
) -> torch.Tensor:
475
assert self.vision_tower is not None
286
- pixel_values = image_input["data"]
476
+
477
+ pixel_values = image_input["pixel_values"]
478
vision_outputs = self._image_pixels_to_features(
479
self.vision_tower,
480
pixel_values,
vllm/multimodal/base.py
+5
-1
@@ -226,7 +226,11 @@ class MultiModalPlugin(ABC):
226
227
if callable(max_mm_tokens):
228
mm_processor_kwargs = get_allowed_kwarg_only_overrides(
229
- max_mm_tokens, overrides=model_config.mm_processor_kwargs)
229
+ max_mm_tokens,
230
+ overrides=model_config.mm_processor_kwargs,
231
+ requires_kw_only=False,
232
+ allow_var_kwargs=True,
233
+ )
234
max_mm_tokens = max_mm_tokens(InputContext(model_config),
235
**mm_processor_kwargs)
236
vllm/utils.py
+2
-2
@@ -1488,11 +1488,11 @@ def get_allowed_kwarg_only_overrides(
1488
if requires_kw_only:
1489
logger.warning(
1490
"The following intended overrides are not keyword-only args "
1491
- "and and will be dropped: %s", dropped_keys)
1491
+ "and will be dropped: %s", dropped_keys)
1492
else:
1493
logger.warning(
1494
"The following intended overrides are not keyword args "
1495
- "and and will be dropped: %s", dropped_keys)
1495
+ "and will be dropped: %s", dropped_keys)
1496
1497
return filtered_overrides
1498