A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Bugfix][Model] Add base class for vision-language models (#4809)
Cyrus Leung committed
May 19, 2024 at 15:13 UTC
f68470e803df575f294e67167b4b83adfe004cfa
4 files changed
+53
-29
tests/models/test_registry.py
new
+9
@@ -0,0 +1,9 @@
1
+import pytest
2
+
3
+from vllm.model_executor.models import _MODELS, ModelRegistry
4
+
5
+
6
+@pytest.mark.parametrize("model_cls", _MODELS)
7
+def test_registry_imports(model_cls):
8
+ # Ensure all model classes can be imported successfully
9
+ ModelRegistry.load_model_cls(model_cls)
vllm/model_executor/model_loader/loader.py
+7
-6
@@ -26,11 +26,7 @@ from vllm.model_executor.model_loader.weight_utils import (
26
download_weights_from_hf, filter_files_not_needed_for_inference,
27
get_quant_config, initialize_dummy_weights, np_cache_weights_iterator,
28
pt_weights_iterator, safetensors_weights_iterator)
29
-from vllm.model_executor.models.llava import LlavaForConditionalGeneration
30
-
31
-_VISION_MODEL_CLASSES = [
32
- LlavaForConditionalGeneration,
33
-]
29
+from vllm.model_executor.models.vlm_base import VisionLanguageModelBase
30
31
logger = init_logger(__name__)
32
@@ -73,7 +69,12 @@ def _get_model_initialization_kwargs(
69
"but LoRA is enabled. Support for this model may "
70
"be added in the future. If this is important to you, "
71
"please open an issue on github.")
76
- elif model_class in _VISION_MODEL_CLASSES:
72
+ elif issubclass(model_class, VisionLanguageModelBase):
73
+ if vision_language_config is None:
74
+ raise ValueError("Provide `image_input_type` and other vision "
75
+ "related configurations through LLM entrypoint "
76
+ "or engine arguments.")
77
+
78
extra_kwargs["vision_language_config"] = vision_language_config
79
return extra_kwargs
80
vllm/model_executor/models/llava.py
+25
-23
@@ -19,6 +19,8 @@ from vllm.model_executor.models.llama import LlamaModel
19
from vllm.model_executor.sampling_metadata import SamplingMetadata
20
from vllm.sequence import SamplerOutput
21
22
+from .vlm_base import VisionLanguageModelBase
23
+
24
_KEYS_TO_MODIFY_MAPPING = {
25
"language_model.lm_head": "lm_head",
26
"language_model.model": "language_model",
@@ -40,7 +42,7 @@ class LlavaMultiModalProjector(nn.Module):
42
text_hidden_size,
43
bias=True)
44
43
- def forward(self, image_features):
45
+ def forward(self, image_features: torch.Tensor) -> torch.Tensor:
46
hidden_states = self.linear_1(image_features)
47
hidden_states = self.act(hidden_states)
48
hidden_states = self.linear_2(hidden_states)
@@ -50,29 +52,31 @@ class LlavaMultiModalProjector(nn.Module):
52
def _merge_vision_embeddings(input_ids: torch.Tensor,
53
inputs_embeds: torch.Tensor,
54
vision_embeddings: torch.Tensor,
53
- image_token_id: int):
55
+ image_token_id: int) -> torch.Tensor:
56
"""In place merges in vision_embeddings with inputs_embeds."""
57
mask = (input_ids == image_token_id)
56
- inputs_embeds[mask] = vision_embeddings.view(-1,
58
+
59
+ image_feature_size = vision_embeddings.shape[0] * vision_embeddings.shape[1]
60
+ if mask.sum() != image_feature_size:
61
+ raise ValueError(f"image_feature_size should be {image_feature_size}, "
62
+ f"but found: {mask.sum()}")
63
+
64
+ inputs_embeds[mask] = vision_embeddings.view(image_feature_size,
65
vision_embeddings.shape[-1])
66
67
+ return inputs_embeds
68
60
-class LlavaForConditionalGeneration(nn.Module):
69
+
70
+class LlavaForConditionalGeneration(VisionLanguageModelBase):
71
72
def __init__(self,
63
- config: "LlavaConfig",
73
+ config: LlavaConfig,
74
vision_language_config: VisionLanguageConfig,
75
cache_config: Optional[CacheConfig] = None,
66
- quant_config: Optional["QuantizationConfig"] = None) -> None:
67
- super().__init__()
68
- self.config = config
69
-
70
- self.vision_language_config = vision_language_config
76
+ quant_config: Optional[QuantizationConfig] = None) -> None:
77
+ super().__init__(vision_language_config)
78
72
- assert self.vision_language_config, (
73
- "Provide `image_input_type` and other vision "
74
- "related configurations through LLM entrypoint "
75
- "or engine arguments.")
79
+ self.config = config
80
81
if self.vision_language_config.image_input_type == (
82
VisionLanguageConfig.ImageInputType.PIXEL_VALUES):
@@ -98,14 +102,12 @@ class LlavaForConditionalGeneration(nn.Module):
102
config.vocab_size, logit_scale)
103
self.sampler = Sampler()
104
101
- def forward(
102
- self,
103
- input_ids: torch.Tensor,
104
- positions: torch.Tensor,
105
- kv_caches: List[torch.Tensor],
106
- attn_metadata: AttentionMetadata,
107
- image_input: Optional[torch.Tensor] = None
108
- ) -> SamplerOutput: # noqa: E501
105
+ def forward(self,
106
+ input_ids: torch.Tensor,
107
+ positions: torch.Tensor,
108
+ kv_caches: List[torch.Tensor],
109
+ attn_metadata: AttentionMetadata,
110
+ image_input: Optional[torch.Tensor] = None) -> SamplerOutput:
111
"""Run forward pass for Llava 1.5.
112
113
One key thing to understand is the `input_ids` already accounts for the
@@ -172,7 +174,7 @@ class LlavaForConditionalGeneration(nn.Module):
174
image_features = image_input
175
vision_embeddings = self.multi_modal_projector(image_features)
176
inputs_embeds = self.language_model.get_input_embeddings(input_ids)
175
- _merge_vision_embeddings(
177
+ inputs_embeds = _merge_vision_embeddings(
178
input_ids, inputs_embeds, vision_embeddings,
179
self.vision_language_config.image_token_id)
180
input_ids = None
vllm/model_executor/models/vlm_base.py
new
+12
@@ -0,0 +1,12 @@
1
+from torch import nn
2
+
3
+from vllm.config import VisionLanguageConfig
4
+
5
+
6
+class VisionLanguageModelBase(nn.Module):
7
+ """Base class for all vision language models (VLMs)."""
8
+
9
+ def __init__(self, vision_language_config: VisionLanguageConfig) -> None:
10
+ super().__init__()
11
+
12
+ self.vision_language_config = vision_language_config