@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f68470e803
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Bugfix][Model] Add base class for vision-language models (#4809)

Cyrus Leung committed May 19, 2024 at 15:13 UTC f68470e803df575f294e67167b4b83adfe004cfa
4 files changed +53 -29
tests/models/test_registry.py new
+9
@@ -0,0 +1,9 @@
1 +import pytest
2 +
3 +from vllm.model_executor.models import _MODELS, ModelRegistry
4 +
5 +
6 +@pytest.mark.parametrize("model_cls", _MODELS)
7 +def test_registry_imports(model_cls):
8 + # Ensure all model classes can be imported successfully
9 + ModelRegistry.load_model_cls(model_cls)
vllm/model_executor/model_loader/loader.py
+7 -6
@@ -26,11 +26,7 @@ from vllm.model_executor.model_loader.weight_utils import (
26 download_weights_from_hf, filter_files_not_needed_for_inference,
27 get_quant_config, initialize_dummy_weights, np_cache_weights_iterator,
28 pt_weights_iterator, safetensors_weights_iterator)
29 -from vllm.model_executor.models.llava import LlavaForConditionalGeneration
30 -
31 -_VISION_MODEL_CLASSES = [
32 - LlavaForConditionalGeneration,
33 -]
29 +from vllm.model_executor.models.vlm_base import VisionLanguageModelBase
30
31 logger = init_logger(__name__)
32
@@ -73,7 +69,12 @@ def _get_model_initialization_kwargs(
69 "but LoRA is enabled. Support for this model may "
70 "be added in the future. If this is important to you, "
71 "please open an issue on github.")
76 - elif model_class in _VISION_MODEL_CLASSES:
72 + elif issubclass(model_class, VisionLanguageModelBase):
73 + if vision_language_config is None:
74 + raise ValueError("Provide `image_input_type` and other vision "
75 + "related configurations through LLM entrypoint "
76 + "or engine arguments.")
77 +
78 extra_kwargs["vision_language_config"] = vision_language_config
79 return extra_kwargs
80
vllm/model_executor/models/llava.py
+25 -23
@@ -19,6 +19,8 @@ from vllm.model_executor.models.llama import LlamaModel
19 from vllm.model_executor.sampling_metadata import SamplingMetadata
20 from vllm.sequence import SamplerOutput
21
22 +from .vlm_base import VisionLanguageModelBase
23 +
24 _KEYS_TO_MODIFY_MAPPING = {
25 "language_model.lm_head": "lm_head",
26 "language_model.model": "language_model",
@@ -40,7 +42,7 @@ class LlavaMultiModalProjector(nn.Module):
42 text_hidden_size,
43 bias=True)
44
43 - def forward(self, image_features):
45 + def forward(self, image_features: torch.Tensor) -> torch.Tensor:
46 hidden_states = self.linear_1(image_features)
47 hidden_states = self.act(hidden_states)
48 hidden_states = self.linear_2(hidden_states)
@@ -50,29 +52,31 @@ class LlavaMultiModalProjector(nn.Module):
52 def _merge_vision_embeddings(input_ids: torch.Tensor,
53 inputs_embeds: torch.Tensor,
54 vision_embeddings: torch.Tensor,
53 - image_token_id: int):
55 + image_token_id: int) -> torch.Tensor:
56 """In place merges in vision_embeddings with inputs_embeds."""
57 mask = (input_ids == image_token_id)
56 - inputs_embeds[mask] = vision_embeddings.view(-1,
58 +
59 + image_feature_size = vision_embeddings.shape[0] * vision_embeddings.shape[1]
60 + if mask.sum() != image_feature_size:
61 + raise ValueError(f"image_feature_size should be {image_feature_size}, "
62 + f"but found: {mask.sum()}")
63 +
64 + inputs_embeds[mask] = vision_embeddings.view(image_feature_size,
65 vision_embeddings.shape[-1])
66
67 + return inputs_embeds
68
60 -class LlavaForConditionalGeneration(nn.Module):
69 +
70 +class LlavaForConditionalGeneration(VisionLanguageModelBase):
71
72 def __init__(self,
63 - config: "LlavaConfig",
73 + config: LlavaConfig,
74 vision_language_config: VisionLanguageConfig,
75 cache_config: Optional[CacheConfig] = None,
66 - quant_config: Optional["QuantizationConfig"] = None) -> None:
67 - super().__init__()
68 - self.config = config
69 -
70 - self.vision_language_config = vision_language_config
76 + quant_config: Optional[QuantizationConfig] = None) -> None:
77 + super().__init__(vision_language_config)
78
72 - assert self.vision_language_config, (
73 - "Provide `image_input_type` and other vision "
74 - "related configurations through LLM entrypoint "
75 - "or engine arguments.")
79 + self.config = config
80
81 if self.vision_language_config.image_input_type == (
82 VisionLanguageConfig.ImageInputType.PIXEL_VALUES):
@@ -98,14 +102,12 @@ class LlavaForConditionalGeneration(nn.Module):
102 config.vocab_size, logit_scale)
103 self.sampler = Sampler()
104
101 - def forward(
102 - self,
103 - input_ids: torch.Tensor,
104 - positions: torch.Tensor,
105 - kv_caches: List[torch.Tensor],
106 - attn_metadata: AttentionMetadata,
107 - image_input: Optional[torch.Tensor] = None
108 - ) -> SamplerOutput: # noqa: E501
105 + def forward(self,
106 + input_ids: torch.Tensor,
107 + positions: torch.Tensor,
108 + kv_caches: List[torch.Tensor],
109 + attn_metadata: AttentionMetadata,
110 + image_input: Optional[torch.Tensor] = None) -> SamplerOutput:
111 """Run forward pass for Llava 1.5.
112
113 One key thing to understand is the `input_ids` already accounts for the
@@ -172,7 +174,7 @@ class LlavaForConditionalGeneration(nn.Module):
174 image_features = image_input
175 vision_embeddings = self.multi_modal_projector(image_features)
176 inputs_embeds = self.language_model.get_input_embeddings(input_ids)
175 - _merge_vision_embeddings(
177 + inputs_embeds = _merge_vision_embeddings(
178 input_ids, inputs_embeds, vision_embeddings,
179 self.vision_language_config.image_token_id)
180 input_ids = None
vllm/model_executor/models/vlm_base.py new
+12
@@ -0,0 +1,12 @@
1 +from torch import nn
2 +
3 +from vllm.config import VisionLanguageConfig
4 +
5 +
6 +class VisionLanguageModelBase(nn.Module):
7 + """Base class for all vision language models (VLMs)."""
8 +
9 + def __init__(self, vision_language_config: VisionLanguageConfig) -> None:
10 + super().__init__()
11 +
12 + self.vision_language_config = vision_language_config