@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6287537a0c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Model] LLaVA model refactor (#4910)

Cyrus Leung committed May 20, 2024 at 16:11 UTC 6287537a0c970bda1fc8b31f2bde1bcf2d26e151
1 file changed +107 -30
vllm/model_executor/models/llava.py
+107 -30
@@ -1,4 +1,4 @@
1 -from typing import Iterable, List, Optional, Tuple
1 +from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
2
3 import torch
4 from torch import nn
@@ -67,6 +67,21 @@ def _merge_vision_embeddings(input_ids: torch.Tensor,
67 return inputs_embeds
68
69
70 +class LlavaImagePixelInputs(TypedDict):
71 + type: Literal["pixel_values"]
72 + data: torch.Tensor
73 + """Shape: (batch_size, num_channels, height, width)"""
74 +
75 +
76 +class LlavaImageFeatureInputs(TypedDict):
77 + type: Literal["image_features"]
78 + data: torch.Tensor
79 + """Shape: (batch_size, image_feature_size, hidden_size)"""
80 +
81 +
82 +LlavaImageInputs = Union[LlavaImagePixelInputs, LlavaImageFeatureInputs]
83 +
84 +
85 class LlavaForConditionalGeneration(VisionLanguageModelBase):
86
87 def __init__(self,
@@ -102,6 +117,90 @@ class LlavaForConditionalGeneration(VisionLanguageModelBase):
117 config.vocab_size, logit_scale)
118 self.sampler = Sampler()
119
120 + def _validate_image_data(self, data: torch.Tensor) -> torch.Tensor:
121 + if list(data.shape[1:]) != list(
122 + self.vision_language_config.image_input_shape[1:]):
123 + raise ValueError(
124 + f"The expected image tensor shape is batch dimension plus "
125 + f"{self.vision_language_config.image_input_shape[1:]}. "
126 + f"You supplied {data.shape}. "
127 + f"If you are using vLLM's entrypoint, make sure your "
128 + f"supplied image input is consistent with "
129 + f"image_input_shape in engine args.")
130 +
131 + return data
132 +
133 + def _parse_and_validate_image_input(
134 + self, data: object) -> Optional[LlavaImageInputs]:
135 + expected_input_type = self.vision_language_config.image_input_type
136 + ImageInputType = VisionLanguageConfig.ImageInputType
137 +
138 + if data is None:
139 + return None
140 +
141 + if expected_input_type == ImageInputType.PIXEL_VALUES:
142 + if not isinstance(data, torch.Tensor):
143 + raise TypeError("Image pixel vector should be a tensor, "
144 + f"but received type: {type(data)}")
145 +
146 + return LlavaImagePixelInputs(
147 + type="pixel_values",
148 + data=self._validate_image_data(data),
149 + )
150 + elif expected_input_type == ImageInputType.IMAGE_FEATURES:
151 + if not isinstance(data, torch.Tensor):
152 + raise TypeError("Image feature vector should be a tensor, "
153 + f"but received type: {type(data)}")
154 +
155 + return LlavaImageFeatureInputs(
156 + type="image_features",
157 + data=self._validate_image_data(data),
158 + )
159 +
160 + return None
161 +
162 + def _select_image_features(self, image_features: torch.Tensor, *,
163 + strategy: str) -> torch.Tensor:
164 + # Copied from https://github.com/huggingface/transformers/blob/39c3c0a72af6fbda5614dde02ff236069bb79827/src/transformers/models/llava/modeling_llava.py#L421 # noqa
165 + if strategy == "default":
166 + return image_features[:, 1:]
167 + elif strategy == "full":
168 + return image_features
169 +
170 + raise ValueError(f"Unexpected select feature strategy: {strategy}")
171 +
172 + def _image_pixels_to_features(self, vision_tower: CLIPVisionModel,
173 + pixel_values: torch.Tensor) -> torch.Tensor:
174 + # TODO(xwjiang): Maybe port minimal CLIPVisionModel over.
175 + image_outputs = vision_tower(pixel_values.to(vision_tower.device),
176 + output_hidden_states=True)
177 +
178 + image_features = image_outputs.hidden_states[
179 + self.config.vision_feature_layer]
180 +
181 + return self._select_image_features(
182 + image_features,
183 + strategy=self.config.vision_feature_select_strategy,
184 + )
185 +
186 + def _process_image_pixels(self,
187 + inputs: LlavaImagePixelInputs) -> torch.Tensor:
188 + assert self.vision_tower is not None
189 +
190 + pixel_values = inputs["data"]
191 +
192 + return self._image_pixels_to_features(self.vision_tower, pixel_values)
193 +
194 + def _process_image_input(self,
195 + image_input: LlavaImageInputs) -> torch.Tensor:
196 + if image_input["type"] == "pixel_values":
197 + assert self.vision_tower is not None
198 + image_features = self._process_image_pixels(image_input)
199 + else:
200 + image_features = image_input["data"]
201 +
202 + return self.multi_modal_projector(image_features)
203 +
204 def forward(self,
205 input_ids: torch.Tensor,
206 positions: torch.Tensor,
@@ -144,42 +243,20 @@ class LlavaForConditionalGeneration(VisionLanguageModelBase):
243 For PIXEL_VALUES, expecting [1, 3, 336, 336].
244 For IMAGE_FEATURES, expecting [1, 576, 1024].
245 """
147 - if image_input is not None:
148 - if list(image_input.shape[1:]) != list(
149 - self.vision_language_config.image_input_shape[1:]):
150 - raise ValueError(
151 - f"The expected image tensor shape is batch dimension "
152 - f"plus "
153 - f"{self.vision_language_config.image_input_shape[1:]}."
154 - f" You supplied {image_input.shape}. "
155 - f"If you are using vLLM's entrypoint, make sure your "
156 - f"supplied image input is consistent with "
157 - f"image_input_shape in engine args.")
158 - if self.vision_tower is not None:
159 - # TODO(xwjiang): Maybe port minimal CLIPVisionModel over.
160 - image_outputs = self.vision_tower(image_input,
161 - output_hidden_states=True)
162 - image_features = image_outputs.hidden_states[
163 - self.config.vision_feature_layer]
164 - # Copied from https://github.com/huggingface/transformers/blob/39c3c0a72af6fbda5614dde02ff236069bb79827/src/transformers/models/llava/modeling_llava.py#L421 # noqa
165 - if self.config.vision_feature_select_strategy == "default":
166 - image_features = image_features[:, 1:]
167 - elif self.config.vision_feature_select_strategy == "full":
168 - image_features = image_features
169 - else:
170 - raise ValueError(
171 - f"Unexpected select feature strategy: "
172 - f"{self.config.vision_feature_select_strategy}")
173 - else:
174 - image_features = image_input
175 - vision_embeddings = self.multi_modal_projector(image_features)
246 + parsed_image_input = self._parse_and_validate_image_input(image_input)
247 +
248 + if parsed_image_input is not None:
249 + vision_embeddings = self._process_image_input(parsed_image_input)
250 inputs_embeds = self.language_model.get_input_embeddings(input_ids)
251 +
252 inputs_embeds = _merge_vision_embeddings(
253 input_ids, inputs_embeds, vision_embeddings,
254 self.vision_language_config.image_token_id)
255 +
256 input_ids = None
257 else:
258 inputs_embeds = None
259 +
260 hidden_states = self.language_model(input_ids,
261 positions,
262 kv_caches,