A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[V1] Update interface for idefics3 (#10680)
Signed-off-by: Roger Wang <ywang@roblox.com>
Roger Wang committed
Nov 26, 2024 at 18:04 UTC
0a4d96850013eb2c295b25df53177ad2302110ca
1 file changed
+46
-27
vllm/model_executor/models/idefics3.py
+46
-27
@@ -39,6 +39,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
39
from vllm.model_executor.sampling_metadata import SamplingMetadata
40
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
41
from vllm.multimodal.image import cached_get_image_processor
42
+from vllm.multimodal.inputs import NestedTensors
43
from vllm.sequence import IntermediateTensors, SequenceData
44
from vllm.transformers_utils.processor import cached_get_processor
45
from vllm.utils import is_list_of
@@ -597,6 +598,12 @@ class Idefics3Model(nn.Module):
598
image_features = self._process_image_pixels(image_input)
599
return self.connector(image_features)
600
601
+ def get_input_embeddings(
602
+ self,
603
+ input_ids: torch.Tensor,
604
+ ) -> torch.Tensor:
605
+ return self.text_model.get_input_embeddings(input_ids)
606
+
607
def forward(
608
self,
609
input_ids: torch.Tensor,
@@ -604,26 +611,8 @@ class Idefics3Model(nn.Module):
611
kv_caches: List[torch.Tensor],
612
attn_metadata: AttentionMetadata,
613
intermediate_tensors: Optional[IntermediateTensors] = None,
607
- **kwargs: object,
614
+ inputs_embeds: Optional[torch.Tensor] = None,
615
) -> Union[torch.Tensor, IntermediateTensors]:
609
- if intermediate_tensors is not None:
610
- input_ids = None
611
- inputs_embeds = None
612
- else:
613
- # always pass the input via `inputs_embeds`
614
- # to make sure the computation graph is consistent
615
- image_input = self._parse_and_validate_image_input(**kwargs)
616
-
617
- if image_input is not None:
618
- vision_embeddings = self._process_image_input(image_input)
619
- inputs_embeds = self.text_model.get_input_embeddings(input_ids)
620
-
621
- inputs_embeds = merge_multimodal_embeddings(
622
- input_ids, inputs_embeds, vision_embeddings,
623
- self.image_token_id)
624
- else:
625
- inputs_embeds = self.text_model.get_input_embeddings(input_ids)
626
- input_ids = None
616
617
hidden_states = self.text_model(
618
input_ids,
@@ -718,6 +707,25 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
707
self.logits_processor = LogitsProcessor(config.text_config.vocab_size)
708
self.sampler = Sampler()
709
710
+ def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
711
+ image_input = self.model._parse_and_validate_image_input(**kwargs)
712
+ if image_input is None:
713
+ return None
714
+ vision_embeddings = self.model._process_image_input(image_input)
715
+ return vision_embeddings
716
+
717
+ def get_input_embeddings(
718
+ self,
719
+ input_ids: torch.Tensor,
720
+ multimodal_embeddings: Optional[NestedTensors] = None,
721
+ ) -> torch.Tensor:
722
+ inputs_embeds = self.model.get_input_embeddings(input_ids)
723
+ if multimodal_embeddings is not None:
724
+ inputs_embeds = merge_multimodal_embeddings(
725
+ input_ids, inputs_embeds, multimodal_embeddings,
726
+ self.config.image_token_id)
727
+ return inputs_embeds
728
+
729
def forward(
730
self,
731
input_ids: torch.Tensor,
@@ -725,16 +733,27 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
733
kv_caches: List[torch.Tensor],
734
attn_metadata: AttentionMetadata,
735
intermediate_tensors: Optional[IntermediateTensors] = None,
736
+ inputs_embeds: Optional[torch.Tensor] = None,
737
**kwargs: object,
738
) -> Union[torch.Tensor, IntermediateTensors]:
730
- hidden_states = self.model(
731
- input_ids,
732
- positions,
733
- kv_caches,
734
- attn_metadata,
735
- intermediate_tensors,
736
- **kwargs,
737
- )
739
+ if intermediate_tensors is not None:
740
+ inputs_embeds = None
741
+
742
+ # NOTE: In v1, inputs_embeds is always generated at model runner, this
743
+ # condition is for v0 compatibility.
744
+ elif inputs_embeds is None:
745
+ vision_embeddings = self.get_multimodal_embeddings(**kwargs)
746
+ inputs_embeds = self.get_input_embeddings(input_ids,
747
+ vision_embeddings)
748
+ input_ids = None
749
+
750
+ hidden_states = self.model.text_model(input_ids,
751
+ positions,
752
+ kv_caches,
753
+ attn_metadata,
754
+ intermediate_tensors,
755
+ inputs_embeds=inputs_embeds)
756
+
757
return hidden_states
758
759
def compute_logits(self, hidden_states: torch.Tensor,