A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
[Bugfix] Make Gemma3 MM V0 only for now (#14971)
Signed-off-by: Roger Wang <ywang@roblox.com>
Roger Wang committed
Mar 17, 2025 at 10:04 UTC
37e380613220f607cb465fc15f72a4a033a98b23
2 files changed
+6
-3
docs/source/models/supported_models.md
+4
-1
@@ -763,7 +763,7 @@ See [this page](#generative-models) for more information on how to use generativ
763
* `google/gemma-3-4b-it`, `google/gemma-3-27b-it`, etc.
764
* ✅︎
765
* ✅︎
766
- * ⚠️
766
+ *
767
- * `GLM4VForCausalLM`<sup>^</sup>
768
* GLM-4V
769
* T + I
@@ -948,8 +948,11 @@ V1 currently uses a simplified attention pattern:
948
- Uses causal attention for all tokens, including image tokens
949
- Generates reasonable outputs but does not match the original model's attention for text + image inputs
950
- Will be updated in the future to support the correct behavior
951
+- Does not support `"do_pan_and_scan": True`
952
953
This limitation exists because the model's mixed attention pattern (bidirectional for images, causal otherwise) is not yet supported by vLLM's attention backends.
954
+
955
+For these reasons, `Gemma3ForConditionalGeneration` is supported only on V0 at the moment.
956
:::
957
958
:::{note}
vllm/model_executor/models/gemma3_mm.py
+2
-2
@@ -25,7 +25,7 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
25
from vllm.sequence import IntermediateTensors
26
27
from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
28
- SupportsMultiModal, SupportsPP)
28
+ SupportsMultiModal, SupportsPP, SupportsV0Only)
29
from .siglip import SiglipVisionModel
30
from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
31
maybe_prefix, merge_multimodal_embeddings)
@@ -374,7 +374,7 @@ class Gemma3MultiModalProjector(nn.Module):
374
info=Gemma3ProcessingInfo,
375
dummy_inputs=Gemma3DummyInputsBuilder)
376
class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,
377
- SupportsLoRA):
377
+ SupportsLoRA, SupportsV0Only):
378
packed_modules_mapping = {
379
"qkv_proj": [
380
"q_proj",