@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0a4d968500
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[V1] Update interface for idefics3 (#10680)

Signed-off-by: Roger Wang <ywang@roblox.com>

Roger Wang committed Nov 26, 2024 at 18:04 UTC 0a4d96850013eb2c295b25df53177ad2302110ca
1 file changed +46 -27
vllm/model_executor/models/idefics3.py
+46 -27
@@ -39,6 +39,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
39 from vllm.model_executor.sampling_metadata import SamplingMetadata
40 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
41 from vllm.multimodal.image import cached_get_image_processor
42 +from vllm.multimodal.inputs import NestedTensors
43 from vllm.sequence import IntermediateTensors, SequenceData
44 from vllm.transformers_utils.processor import cached_get_processor
45 from vllm.utils import is_list_of
@@ -597,6 +598,12 @@ class Idefics3Model(nn.Module):
598 image_features = self._process_image_pixels(image_input)
599 return self.connector(image_features)
600
601 + def get_input_embeddings(
602 + self,
603 + input_ids: torch.Tensor,
604 + ) -> torch.Tensor:
605 + return self.text_model.get_input_embeddings(input_ids)
606 +
607 def forward(
608 self,
609 input_ids: torch.Tensor,
@@ -604,26 +611,8 @@ class Idefics3Model(nn.Module):
611 kv_caches: List[torch.Tensor],
612 attn_metadata: AttentionMetadata,
613 intermediate_tensors: Optional[IntermediateTensors] = None,
607 - **kwargs: object,
614 + inputs_embeds: Optional[torch.Tensor] = None,
615 ) -> Union[torch.Tensor, IntermediateTensors]:
609 - if intermediate_tensors is not None:
610 - input_ids = None
611 - inputs_embeds = None
612 - else:
613 - # always pass the input via `inputs_embeds`
614 - # to make sure the computation graph is consistent
615 - image_input = self._parse_and_validate_image_input(**kwargs)
616 -
617 - if image_input is not None:
618 - vision_embeddings = self._process_image_input(image_input)
619 - inputs_embeds = self.text_model.get_input_embeddings(input_ids)
620 -
621 - inputs_embeds = merge_multimodal_embeddings(
622 - input_ids, inputs_embeds, vision_embeddings,
623 - self.image_token_id)
624 - else:
625 - inputs_embeds = self.text_model.get_input_embeddings(input_ids)
626 - input_ids = None
616
617 hidden_states = self.text_model(
618 input_ids,
@@ -718,6 +707,25 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
707 self.logits_processor = LogitsProcessor(config.text_config.vocab_size)
708 self.sampler = Sampler()
709
710 + def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
711 + image_input = self.model._parse_and_validate_image_input(**kwargs)
712 + if image_input is None:
713 + return None
714 + vision_embeddings = self.model._process_image_input(image_input)
715 + return vision_embeddings
716 +
717 + def get_input_embeddings(
718 + self,
719 + input_ids: torch.Tensor,
720 + multimodal_embeddings: Optional[NestedTensors] = None,
721 + ) -> torch.Tensor:
722 + inputs_embeds = self.model.get_input_embeddings(input_ids)
723 + if multimodal_embeddings is not None:
724 + inputs_embeds = merge_multimodal_embeddings(
725 + input_ids, inputs_embeds, multimodal_embeddings,
726 + self.config.image_token_id)
727 + return inputs_embeds
728 +
729 def forward(
730 self,
731 input_ids: torch.Tensor,
@@ -725,16 +733,27 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
733 kv_caches: List[torch.Tensor],
734 attn_metadata: AttentionMetadata,
735 intermediate_tensors: Optional[IntermediateTensors] = None,
736 + inputs_embeds: Optional[torch.Tensor] = None,
737 **kwargs: object,
738 ) -> Union[torch.Tensor, IntermediateTensors]:
730 - hidden_states = self.model(
731 - input_ids,
732 - positions,
733 - kv_caches,
734 - attn_metadata,
735 - intermediate_tensors,
736 - **kwargs,
737 - )
739 + if intermediate_tensors is not None:
740 + inputs_embeds = None
741 +
742 + # NOTE: In v1, inputs_embeds is always generated at model runner, this
743 + # condition is for v0 compatibility.
744 + elif inputs_embeds is None:
745 + vision_embeddings = self.get_multimodal_embeddings(**kwargs)
746 + inputs_embeds = self.get_input_embeddings(input_ids,
747 + vision_embeddings)
748 + input_ids = None
749 +
750 + hidden_states = self.model.text_model(input_ids,
751 + positions,
752 + kv_caches,
753 + attn_metadata,
754 + intermediate_tensors,
755 + inputs_embeds=inputs_embeds)
756 +
757 return hidden_states
758
759 def compute_logits(self, hidden_states: torch.Tensor,