A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[torch.compile] Adding torch compile to vision-language models (#9946)
Yongzao committed
Nov 3, 2024 at 03:56 UTC
ae5279a16385e15c07ab2bcadcbcab44367595e9
3 files changed
+21
-8
vllm/model_executor/models/llava_next.py
+7
-3
@@ -606,7 +606,6 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
606
:class:`LlavaNextImageInputs`
607
"""
608
if intermediate_tensors is not None:
609
- input_ids = None
609
inputs_embeds = None
610
else:
611
image_input = self._parse_and_validate_image_input(**kwargs)
@@ -618,9 +617,14 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
617
self.language_model.model.get_input_embeddings,
618
lambda _: self._process_image_input(image_input),
619
)
621
- input_ids = None
620
else:
623
- inputs_embeds = None
621
+ inputs_embeds = self.language_model.model.get_input_embeddings(
622
+ input_ids)
623
+
624
+ # always pass the input via `inputs_embeds`
625
+ # to make sure the computation graph is consistent
626
+ # for `torch.compile` integration
627
+ input_ids = None
628
629
hidden_states = self.language_model.model(input_ids,
630
positions,
vllm/model_executor/models/minicpmv.py
+6
-1
@@ -564,8 +564,13 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
564
565
vlm_embeddings, _ = self.get_embedding(input_ids, image_inputs)
566
567
+ # always pass the input via `inputs_embeds`
568
+ # to make sure the computation graph is consistent
569
+ # for `torch.compile` integration
570
+ input_ids = None
571
+
572
output = self.llm(
568
- input_ids=None,
573
+ input_ids=input_ids,
574
positions=positions,
575
kv_caches=kv_caches,
576
attn_metadata=attn_metadata,
vllm/model_executor/models/molmo.py
+8
-4
@@ -15,6 +15,7 @@ from transformers import PretrainedConfig
15
16
from vllm.attention import Attention, AttentionMetadata
17
from vllm.attention.selector import _Backend
18
+from vllm.compilation.decorators import support_torch_compile
19
from vllm.config import CacheConfig, MultiModalConfig
20
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
21
get_tensor_model_parallel_world_size,
@@ -713,6 +714,7 @@ class MolmoVisionBackbone(nn.Module):
714
return image_features
715
716
717
+@support_torch_compile
718
class MolmoModel(nn.Module):
719
720
def __init__(
@@ -1141,7 +1143,6 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1143
**kwargs: object,
1144
) -> SamplerOutput:
1145
if intermediate_tensors is not None:
1144
- input_ids = None
1146
inputs_embeds = None
1147
else:
1148
image_input = self._parse_and_validate_image_input(**kwargs)
@@ -1156,10 +1157,13 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1157
image_input["image_input_idx"],
1158
image_input["seq_len"],
1159
)
1159
-
1160
- input_ids = None
1160
else:
1162
- inputs_embeds = None
1161
+ inputs_embeds = self.model.embed_tokens(input_ids)
1162
+
1163
+ # always pass the input via `inputs_embeds`
1164
+ # to make sure the computation graph is consistent
1165
+ # for `torch.compile` integration
1166
+ input_ids = None
1167
1168
hidden_states = self.model(
1169
input_ids=input_ids,