@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ae5279a163
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[torch.compile] Adding torch compile to vision-language models (#9946)

Yongzao committed Nov 3, 2024 at 03:56 UTC ae5279a16385e15c07ab2bcadcbcab44367595e9
3 files changed +21 -8
vllm/model_executor/models/llava_next.py
+7 -3
@@ -606,7 +606,6 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
606 :class:`LlavaNextImageInputs`
607 """
608 if intermediate_tensors is not None:
609 - input_ids = None
609 inputs_embeds = None
610 else:
611 image_input = self._parse_and_validate_image_input(**kwargs)
@@ -618,9 +617,14 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
617 self.language_model.model.get_input_embeddings,
618 lambda _: self._process_image_input(image_input),
619 )
621 - input_ids = None
620 else:
623 - inputs_embeds = None
621 + inputs_embeds = self.language_model.model.get_input_embeddings(
622 + input_ids)
623 +
624 + # always pass the input via `inputs_embeds`
625 + # to make sure the computation graph is consistent
626 + # for `torch.compile` integration
627 + input_ids = None
628
629 hidden_states = self.language_model.model(input_ids,
630 positions,
vllm/model_executor/models/minicpmv.py
+6 -1
@@ -564,8 +564,13 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
564
565 vlm_embeddings, _ = self.get_embedding(input_ids, image_inputs)
566
567 + # always pass the input via `inputs_embeds`
568 + # to make sure the computation graph is consistent
569 + # for `torch.compile` integration
570 + input_ids = None
571 +
572 output = self.llm(
568 - input_ids=None,
573 + input_ids=input_ids,
574 positions=positions,
575 kv_caches=kv_caches,
576 attn_metadata=attn_metadata,
vllm/model_executor/models/molmo.py
+8 -4
@@ -15,6 +15,7 @@ from transformers import PretrainedConfig
15
16 from vllm.attention import Attention, AttentionMetadata
17 from vllm.attention.selector import _Backend
18 +from vllm.compilation.decorators import support_torch_compile
19 from vllm.config import CacheConfig, MultiModalConfig
20 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
21 get_tensor_model_parallel_world_size,
@@ -713,6 +714,7 @@ class MolmoVisionBackbone(nn.Module):
714 return image_features
715
716
717 +@support_torch_compile
718 class MolmoModel(nn.Module):
719
720 def __init__(
@@ -1141,7 +1143,6 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1143 **kwargs: object,
1144 ) -> SamplerOutput:
1145 if intermediate_tensors is not None:
1144 - input_ids = None
1146 inputs_embeds = None
1147 else:
1148 image_input = self._parse_and_validate_image_input(**kwargs)
@@ -1156,10 +1157,13 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1157 image_input["image_input_idx"],
1158 image_input["seq_len"],
1159 )
1159 -
1160 - input_ids = None
1160 else:
1162 - inputs_embeds = None
1161 + inputs_embeds = self.model.embed_tokens(input_ids)
1162 +
1163 + # always pass the input via `inputs_embeds`
1164 + # to make sure the computation graph is consistent
1165 + # for `torch.compile` integration
1166 + input_ids = None
1167
1168 hidden_states = self.model(
1169 input_ids=input_ids,