A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[Misc] Move Llama 4 projector call into encoder execution (#16201)
Roger Wang committed
Apr 7, 2025 at 14:02 UTC
ed636d99caa00ebfff66410baedbd059121807a0
1 file changed
+4
-3
vllm/model_executor/models/mllama4.py
+4
-3
@@ -760,6 +760,8 @@ class Llama4ForConditionalGeneration(nn.Module, SupportsMultiModal,
760
flat_data = image_input["flat_data"]
761
patches_per_image = image_input["patches_per_image"].tolist()
762
vision_embeddings_flat = self.vision_model(flat_data)
763
+ vision_embeddings_flat = self.multi_modal_projector(
764
+ vision_embeddings_flat)
765
return vision_embeddings_flat.split(patches_per_image, dim=0)
766
767
def get_multimodal_embeddings(self,
@@ -791,10 +793,9 @@ class Llama4ForConditionalGeneration(nn.Module, SupportsMultiModal,
793
inputs_embeds = self.language_model.get_input_embeddings(input_ids)
794
795
if multimodal_embeddings is not None:
794
- multimodal_embeddings = torch.cat(multimodal_embeddings)
795
- mm_embeddings = self.multi_modal_projector(multimodal_embeddings)
796
inputs_embeds = merge_multimodal_embeddings(
797
- input_ids, inputs_embeds, select_patch_features(mm_embeddings),
797
+ input_ids, inputs_embeds,
798
+ select_patch_features(multimodal_embeddings),
799
self.config.image_token_index)
800
801
return inputs_embeds