@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2c0d933594
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Bugfix] Fix LLaVA-NeXT (#5380)

Cyrus Leung committed Jun 10, 2024 at 23:38 UTC 2c0d9335942a76bdc72e3c51dadfc7397bc79cea
2 files changed +25 -1
vllm/model_executor/models/llava_next.py
+24
@@ -216,6 +216,30 @@ class LlavaNextForConditionalGeneration(VisionLanguageModelBase):
216
217 return None
218
219 + def _select_image_features(self, image_features: torch.Tensor, *,
220 + strategy: str) -> torch.Tensor:
221 + # Copied from https://github.com/huggingface/transformers/blob/39c3c0a72af6fbda5614dde02ff236069bb79827/src/transformers/models/llava/modeling_llava.py#L421 # noqa
222 + if strategy == "default":
223 + return image_features[:, 1:]
224 + elif strategy == "full":
225 + return image_features
226 +
227 + raise ValueError(f"Unexpected select feature strategy: {strategy}")
228 +
229 + def _image_pixels_to_features(self, vision_tower: CLIPVisionModel,
230 + pixel_values: torch.Tensor) -> torch.Tensor:
231 + # TODO(xwjiang): Maybe port minimal CLIPVisionModel over.
232 + image_outputs = vision_tower(pixel_values.to(vision_tower.device),
233 + output_hidden_states=True)
234 +
235 + image_features = image_outputs.hidden_states[
236 + self.config.vision_feature_layer]
237 +
238 + return self._select_image_features(
239 + image_features,
240 + strategy=self.config.vision_feature_select_strategy,
241 + )
242 +
243 def _merge_image_patch_embeddings(self, image_size: torch.Tensor,
244 patch_embeddings: torch.Tensor, *,
245 strategy: str) -> torch.Tensor:
vllm/multimodal/utils.py
+1 -1
@@ -77,7 +77,7 @@ def get_full_image_text_prompt(image_prompt: str, text_prompt: str,
77 """Combine image and text prompts for vision language model depending on
78 the model architecture."""
79
80 - if config.hf_config.model_type == "llava":
80 + if config.hf_config.model_type in ("llava", "llava_next"):
81 full_prompt = f"{image_prompt}\n{text_prompt}"
82 else:
83 raise ValueError(