A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Bugfix] Fix LLaVA-NeXT (#5380)
Cyrus Leung committed
Jun 10, 2024 at 23:38 UTC
2c0d9335942a76bdc72e3c51dadfc7397bc79cea
2 files changed
+25
-1
vllm/model_executor/models/llava_next.py
+24
@@ -216,6 +216,30 @@ class LlavaNextForConditionalGeneration(VisionLanguageModelBase):
216
217
return None
218
219
+ def _select_image_features(self, image_features: torch.Tensor, *,
220
+ strategy: str) -> torch.Tensor:
221
+ # Copied from https://github.com/huggingface/transformers/blob/39c3c0a72af6fbda5614dde02ff236069bb79827/src/transformers/models/llava/modeling_llava.py#L421 # noqa
222
+ if strategy == "default":
223
+ return image_features[:, 1:]
224
+ elif strategy == "full":
225
+ return image_features
226
+
227
+ raise ValueError(f"Unexpected select feature strategy: {strategy}")
228
+
229
+ def _image_pixels_to_features(self, vision_tower: CLIPVisionModel,
230
+ pixel_values: torch.Tensor) -> torch.Tensor:
231
+ # TODO(xwjiang): Maybe port minimal CLIPVisionModel over.
232
+ image_outputs = vision_tower(pixel_values.to(vision_tower.device),
233
+ output_hidden_states=True)
234
+
235
+ image_features = image_outputs.hidden_states[
236
+ self.config.vision_feature_layer]
237
+
238
+ return self._select_image_features(
239
+ image_features,
240
+ strategy=self.config.vision_feature_select_strategy,
241
+ )
242
+
243
def _merge_image_patch_embeddings(self, image_size: torch.Tensor,
244
patch_embeddings: torch.Tensor, *,
245
strategy: str) -> torch.Tensor:
vllm/multimodal/utils.py
+1
-1
@@ -77,7 +77,7 @@ def get_full_image_text_prompt(image_prompt: str, text_prompt: str,
77
"""Combine image and text prompts for vision language model depending on
78
the model architecture."""
79
80
- if config.hf_config.model_type == "llava":
80
+ if config.hf_config.model_type in ("llava", "llava_next"):
81
full_prompt = f"{image_prompt}\n{text_prompt}"
82
else:
83
raise ValueError(