A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 20:15 UTC
[Feature] Add LoRA tower/connector support for Llama 4 Vision (mllama4) (#35147)
Signed-off-by: dorhuri123 <dor.huri1@live.biu.ac.il> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Dor Huri committed
Feb 24, 2026 at 14:10 UTC
012dee92331c7f9477c90c3f3944600c1f03f38d
1 file changed
+23
-1
vllm/model_executor/models/mllama4.py
+23
-1
@@ -1151,6 +1151,28 @@ class Llama4ForConditionalGeneration(
1151
"""
1152
return MultiModelKeys.from_string_field(
1153
language_model="language_model",
1154
- connector="multi_modal_projector.",
1154
+ connector=[
1155
+ "multi_modal_projector.",
1156
+ "vision_model.vision_adapter.",
1157
+ ],
1158
tower_model="vision_model.",
1159
)
1160
+
1161
+ def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
1162
+ vision_config = self.config.vision_config
1163
+ patches_per_chunk = Mllama4ProcessingInfo.get_patch_per_chunk(vision_config)
1164
+ if num_image_tokens <= 0 or patches_per_chunk <= 0:
1165
+ return 0
1166
+ raw_patches = (vision_config.image_size // vision_config.patch_size) ** 2
1167
+ num_chunks = num_image_tokens // patches_per_chunk
1168
+ # Encoder processes raw_patches + 1 (CLS) per chunk
1169
+ return num_chunks * (raw_patches + 1)
1170
+
1171
+ def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
1172
+ vision_config = self.config.vision_config
1173
+ raw_patches = (vision_config.image_size // vision_config.patch_size) ** 2
1174
+ if num_vision_tokens <= 0:
1175
+ return 0
1176
+ num_chunks = num_vision_tokens // (raw_patches + 1)
1177
+ patches_per_chunk = Mllama4ProcessingInfo.get_patch_per_chunk(vision_config)
1178
+ return num_chunks * patches_per_chunk