A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 14:45 UTC
[Feature] Add LoRA support for Gemma3 vision components (#32764)
VihaanThat committed
Jan 26, 2026 at 19:26 UTC
208c56256f74f0eb80d3bd1c592014b1e475b6f6
1 file changed
+38
vllm/model_executor/models/gemma3_mm.py
+38
@@ -656,3 +656,41 @@ class Gemma3ForConditionalGeneration(
656
connector="multi_modal_projector",
657
tower_model="vision_tower",
658
)
659
+
660
+ def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
661
+ """
662
+ Calculate the number of tokens output by the vision encoder.
663
+
664
+ The vision encoder processes images into patch embeddings. For Gemma3,
665
+ the relationship between prompt placeholder tokens and actual vision
666
+ encoder output tokens depends on the patch grid size.
667
+
668
+ Args:
669
+ num_image_tokens: Number of image placeholder tokens in the prompt
670
+ (typically mm_tokens_per_image per image)
671
+
672
+ Returns:
673
+ Number of tokens output by the vision encoder
674
+ """
675
+ # For Gemma3, the vision encoder outputs tokens_per_side x tokens_per_side
676
+ # tokens per image. Since num_image_tokens represents the number of
677
+ # connector output tokens (mm_tokens_per_image = 256), and tokens_per_side
678
+ # is sqrt(256) = 16, we need to account for the token expansion.
679
+ # Based on empirical testing, the multiplier of 16 works correctly.
680
+ return num_image_tokens * 16
681
+
682
+ def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
683
+ """
684
+ Calculate the number of tokens output by the multimodal connector.
685
+
686
+ The connector applies projection and normalization but maintains the
687
+ token count for Gemma3.
688
+
689
+ Args:
690
+ num_vision_tokens: Number of tokens from vision encoder
691
+
692
+ Returns:
693
+ Number of tokens after connector processing
694
+ """
695
+ # The Gemma3 connector maintains a 1:1 token mapping
696
+ return num_vision_tokens