@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2f707fcb35
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Model] Multi-input support for LLaVA (#8238)

Cyrus Leung committed Sep 7, 2024 at 10:57 UTC 2f707fcb35c5bc4b9164cf2bbce0254a72f7348b
10 files changed +176 -45
docs/source/models/supported_models.rst
+8 -8
@@ -219,7 +219,7 @@ Multimodal Language Models
219 -
220 * - :code:`LlavaForConditionalGeneration`
221 - LLaVA-1.5
222 - - Image\ :sup:`E`
222 + - Image\ :sup:`E+`
223 - :code:`llava-hf/llava-1.5-7b-hf`, :code:`llava-hf/llava-1.5-13b-hf`, etc.
224 -
225 * - :code:`LlavaNextForConditionalGeneration`
@@ -227,6 +227,11 @@ Multimodal Language Models
227 - Image\ :sup:`E+`
228 - :code:`llava-hf/llava-v1.6-mistral-7b-hf`, :code:`llava-hf/llava-v1.6-vicuna-7b-hf`, etc.
229 -
230 + * - :code:`MiniCPMV`
231 + - MiniCPM-V
232 + - Image\ :sup:`+`
233 + - :code:`openbmb/MiniCPM-V-2` (see note), :code:`openbmb/MiniCPM-Llama3-V-2_5`, :code:`openbmb/MiniCPM-V-2_6`, etc.
234 + -
235 * - :code:`PaliGemmaForConditionalGeneration`
236 - PaliGemma
237 - Image\ :sup:`E`
@@ -237,14 +242,9 @@ Multimodal Language Models
242 - Image\ :sup:`E+`
243 - :code:`microsoft/Phi-3-vision-128k-instruct`, :code:`microsoft/Phi-3.5-vision-instruct` etc.
244 -
240 - * - :code:`MiniCPMV`
241 - - MiniCPM-V
242 - - Image\ :sup:`+`
243 - - :code:`openbmb/MiniCPM-V-2` (see note), :code:`openbmb/MiniCPM-Llama3-V-2_5`, :code:`openbmb/MiniCPM-V-2_6`, etc.
244 - -
245 * - :code:`QWenLMHeadModel`
246 - - Qwen
247 - - Image
246 + - Qwen-VL
247 + - Image\ :sup:`E`
248 - :code:`Qwen/Qwen-VL`, :code:`Qwen/Qwen-VL-Chat`, etc.
249 -
250 * - :code:`UltravoxModel`
tests/conftest.py
+6 -6
@@ -278,7 +278,7 @@ class HfRunner:
278 def generate(
279 self,
280 prompts: List[str],
281 - images: Optional[List[Image.Image]] = None,
281 + images: Optional[PromptImageInput] = None,
282 **kwargs: Any,
283 ) -> List[Tuple[List[List[int]], List[str]]]:
284 if images:
@@ -314,7 +314,7 @@ class HfRunner:
314 self,
315 prompts: List[str],
316 max_tokens: int,
317 - images: Optional[List[Image.Image]] = None,
317 + images: Optional[PromptImageInput] = None,
318 **kwargs: Any,
319 ) -> List[Tuple[List[int], str]]:
320 outputs = self.generate(prompts,
@@ -351,7 +351,7 @@ class HfRunner:
351 self,
352 prompts: List[str],
353 max_tokens: int,
354 - images: Optional[List[Image.Image]] = None,
354 + images: Optional[PromptImageInput] = None,
355 **kwargs: Any,
356 ) -> List[List[torch.Tensor]]:
357 all_logprobs: List[List[torch.Tensor]] = []
@@ -433,8 +433,8 @@ class HfRunner:
433 prompts: List[str],
434 max_tokens: int,
435 num_logprobs: int,
436 - images: Optional[List[Image.Image]] = None,
437 - audios: Optional[List[Tuple[np.ndarray, int]]] = None,
436 + images: Optional[PromptImageInput] = None,
437 + audios: Optional[PromptAudioInput] = None,
438 **kwargs: Any,
439 ) -> List[Tuple[List[int], str, List[Dict[int, float]]]]:
440 all_logprobs: List[List[Dict[int, float]]] = []
@@ -671,7 +671,7 @@ class VllmRunner:
671 self,
672 prompts: List[str],
673 max_tokens: int,
674 - images: Optional[List[Image.Image]] = None,
674 + images: Optional[PromptImageInput] = None,
675 ) -> List[Tuple[List[int], str]]:
676 greedy_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)
677 outputs = self.generate(prompts, greedy_params, images=images)
tests/distributed/test_multimodal_broadcast.py
+4 -2
@@ -35,9 +35,11 @@ def test_models(hf_runner, vllm_runner, image_assets, model: str,
35 if model.startswith("llava-hf/llava-1.5"):
36 from ..models.test_llava import models, run_test
37 elif model.startswith("llava-hf/llava-v1.6"):
38 - from ..models.test_llava_next import models, run_test
38 + from ..models.test_llava_next import run_test # type: ignore[no-redef]
39 + from ..models.test_llava_next import models
40 elif model.startswith("facebook/chameleon"):
40 - from ..models.test_chameleon import models, run_test
41 + from ..models.test_chameleon import run_test # type: ignore[no-redef]
42 + from ..models.test_chameleon import models
43 else:
44 raise NotImplementedError(f"Unsupported model: {model}")
45
tests/models/test_llava.py
+129 -12
@@ -1,4 +1,4 @@
1 -from typing import List, Optional, Tuple, Type
1 +from typing import List, Optional, Tuple, Type, overload
2
3 import pytest
4 from transformers import (AutoConfig, AutoModelForVision2Seq, AutoTokenizer,
@@ -8,11 +8,14 @@ from vllm.multimodal.utils import rescale_image_size
8 from vllm.sequence import SampleLogprobs
9 from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
10
11 -from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
11 +from ..conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
12 + _ImageAssets)
13 from .utils import check_logprobs_close
14
15 pytestmark = pytest.mark.vlm
16
17 +_LIMIT_IMAGE_PER_PROMPT = 4
18 +
19 HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
20 "stop_sign":
21 "USER: <image>\nWhat's the content of the image?\nASSISTANT:",
@@ -52,6 +55,7 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
55 return hf_output_ids, hf_output_str, out_logprobs
56
57
58 +@overload
59 def run_test(
60 hf_runner: Type[HfRunner],
61 vllm_runner: Type[VllmRunner],
@@ -64,6 +68,78 @@ def run_test(
68 num_logprobs: int,
69 tensor_parallel_size: int,
70 distributed_executor_backend: Optional[str] = None,
71 +):
72 + ...
73 +
74 +
75 +@overload
76 +def run_test(
77 + hf_runner: Type[HfRunner],
78 + vllm_runner: Type[VllmRunner],
79 + image_assets: _ImageAssets,
80 + model: str,
81 + *,
82 + sizes: List[Tuple[int, int]],
83 + dtype: str,
84 + max_tokens: int,
85 + num_logprobs: int,
86 + tensor_parallel_size: int,
87 + distributed_executor_backend: Optional[str] = None,
88 +):
89 + ...
90 +
91 +
92 +def run_test(
93 + hf_runner: Type[HfRunner],
94 + vllm_runner: Type[VllmRunner],
95 + image_assets: _ImageAssets,
96 + model: str,
97 + *,
98 + size_factors: Optional[List[float]] = None,
99 + sizes: Optional[List[Tuple[int, int]]] = None,
100 + dtype: str,
101 + max_tokens: int,
102 + num_logprobs: int,
103 + tensor_parallel_size: int,
104 + distributed_executor_backend: Optional[str] = None,
105 +):
106 + images = [asset.pil_image for asset in image_assets]
107 +
108 + if size_factors is not None:
109 + inputs_per_image = [(
110 + [prompt for _ in size_factors],
111 + [rescale_image_size(image, factor) for factor in size_factors],
112 + ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
113 + elif sizes is not None:
114 + inputs_per_image = [(
115 + [prompt for _ in sizes],
116 + [image.resize(size) for size in sizes],
117 + ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
118 + else:
119 + raise ValueError("You must provide either `size_factors` or `sizes`")
120 +
121 + _run_test(hf_runner,
122 + vllm_runner,
123 + inputs_per_image,
124 + model,
125 + dtype=dtype,
126 + max_tokens=max_tokens,
127 + num_logprobs=num_logprobs,
128 + tensor_parallel_size=tensor_parallel_size,
129 + distributed_executor_backend=distributed_executor_backend)
130 +
131 +
132 +def _run_test(
133 + hf_runner: Type[HfRunner],
134 + vllm_runner: Type[VllmRunner],
135 + inputs: List[Tuple[List[str], PromptImageInput]],
136 + model: str,
137 + *,
138 + dtype: str,
139 + max_tokens: int,
140 + num_logprobs: int,
141 + tensor_parallel_size: int,
142 + distributed_executor_backend: Optional[str] = None,
143 ):
144 """Inference result should be the same between hf and vllm.
145
@@ -85,13 +161,6 @@ def run_test(
161 else:
162 mantis_processor = None
163
88 - images = [asset.pil_image for asset in image_assets]
89 -
90 - inputs_per_image = [(
91 - [prompt for _ in size_factors],
92 - [rescale_image_size(image, factor) for factor in size_factors],
93 - ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
94 -
164 # NOTE: take care of the order. run vLLM first, and then run HF.
165 # vLLM needs a fresh new process without cuda initialization.
166 # if we run HF first, the cuda initialization will be done and it
@@ -100,15 +169,18 @@ def run_test(
169 # max_model_len should be greater than image_feature_size
170 with vllm_runner(model,
171 dtype=dtype,
172 + max_model_len=4096,
173 tensor_parallel_size=tensor_parallel_size,
174 distributed_executor_backend=distributed_executor_backend,
105 - enforce_eager=True) as vllm_model:
175 + enforce_eager=True,
176 + limit_mm_per_prompt={"image": _LIMIT_IMAGE_PER_PROMPT
177 + }) as vllm_model:
178 vllm_outputs_per_image = [
179 vllm_model.generate_greedy_logprobs(prompts,
180 max_tokens,
181 num_logprobs=num_logprobs,
182 images=images)
111 - for prompts, images in inputs_per_image
183 + for prompts, images in inputs
184 ]
185
186 if mantis_processor is not None:
@@ -131,7 +203,7 @@ def run_test(
203 max_tokens,
204 num_logprobs=num_logprobs,
205 images=images)
134 - for prompts, images in inputs_per_image
206 + for prompts, images in inputs
207 ]
208
209 for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
@@ -181,6 +253,51 @@ def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
253 )
254
255
256 +@pytest.mark.parametrize("model", models)
257 +@pytest.mark.parametrize("dtype", ["half"])
258 +@pytest.mark.parametrize("max_tokens", [128])
259 +@pytest.mark.parametrize("num_logprobs", [5])
260 +def test_models_multiple_image_inputs(hf_runner, vllm_runner, image_assets,
261 + model, dtype, max_tokens,
262 + num_logprobs) -> None:
263 + stop_sign = image_assets[0].pil_image
264 + cherry_blossom = image_assets[1].pil_image
265 +
266 + inputs = [(
267 + [
268 + "USER: <image><image>\nDescribe 2 images.\nASSISTANT:",
269 + "USER: <image><image>\nDescribe 2 images.\nASSISTANT:",
270 + "USER: <image><image><image><image>\nDescribe 4 images.\nASSISTANT:", # noqa: E501
271 + "USER: <image>\nWhat is the season?\nASSISTANT:",
272 + ],
273 + [
274 + [stop_sign, cherry_blossom],
275 + # Images with different sizes and aspect-ratios
276 + [
277 + rescale_image_size(stop_sign, 0.1),
278 + stop_sign,
279 + ],
280 + [
281 + stop_sign,
282 + rescale_image_size(stop_sign, 0.25),
283 + cherry_blossom.resize((183, 488)),
284 + cherry_blossom.resize((488, 183))
285 + ],
286 + cherry_blossom,
287 + ])]
288 +
289 + _run_test(
290 + hf_runner,
291 + vllm_runner,
292 + inputs,
293 + model,
294 + dtype=dtype,
295 + max_tokens=max_tokens,
296 + num_logprobs=num_logprobs,
297 + tensor_parallel_size=1,
298 + )
299 +
300 +
301 @pytest.mark.parametrize("model", models)
302 def test_context_length_too_short(vllm_runner, image_assets, model):
303 images = [asset.pil_image for asset in image_assets]
vllm/model_executor/models/clip.py
+1 -1
@@ -105,7 +105,7 @@ def input_processor_for_clip(
105 if isinstance(image_data, Image.Image):
106 image_feature_size = get_clip_image_feature_size(hf_config)
107 elif isinstance(image_data, torch.Tensor):
108 - image_feature_size = image_data.shape[0]
108 + num_images, image_feature_size, hidden_size = image_data.shape
109 else:
110 raise TypeError(f"Invalid image type: {type(image_data)}")
111 else:
vllm/model_executor/models/internvl.py
+1 -1
@@ -209,7 +209,7 @@ def input_processor_for_internvl(ctx: InputContext, llm_inputs: LLMInputs):
209 image_feature_size = num_blocks * num_patches
210
211 elif isinstance(image_data, torch.Tensor):
212 - image_feature_size = image_data.shape[0]
212 + num_images, image_feature_size, hidden_size = image_data.shape
213 else:
214 raise TypeError(f"Invalid image type: {type(image_data)}")
215
vllm/model_executor/models/llava.py
+20 -12
@@ -4,6 +4,7 @@ from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
4
5 import torch
6 import torch.nn as nn
7 +from PIL import Image
8 from transformers import CLIPVisionConfig, LlavaConfig, SiglipVisionConfig
9
10 from vllm.attention import AttentionMetadata
@@ -16,6 +17,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
17 from vllm.model_executor.sampling_metadata import SamplingMetadata
18 from vllm.multimodal import MULTIMODAL_REGISTRY
19 from vllm.sequence import IntermediateTensors
20 +from vllm.utils import is_list_of
21
22 from .clip import (CLIPVisionModel, dummy_image_for_clip,
23 dummy_seq_data_for_clip, get_max_clip_image_tokens,
@@ -24,7 +26,7 @@ from .interfaces import SupportsMultiModal
26 from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
27 dummy_seq_data_for_siglip, get_max_siglip_image_tokens,
28 input_processor_for_siglip)
27 -from .utils import (filter_weights, init_vllm_registered_model,
29 +from .utils import (filter_weights, flatten_bn, init_vllm_registered_model,
30 merge_multimodal_embeddings)
31
32
@@ -133,7 +135,18 @@ def input_processor_for_llava(ctx: InputContext, llm_inputs: LLMInputs):
135 hf_config = ctx.get_hf_config(LlavaConfig)
136 vision_config = hf_config.vision_config
137
136 - image_feature_size = get_max_llava_image_tokens(ctx)
138 + image_data = multi_modal_data["image"]
139 + if isinstance(image_data, Image.Image):
140 + image_feature_size = get_max_llava_image_tokens(ctx)
141 + elif is_list_of(image_data, Image.Image):
142 + image_feature_size = [get_max_llava_image_tokens(ctx)
143 + ] * len(image_data)
144 + elif isinstance(image_data, torch.Tensor):
145 + num_images, image_feature_size, hidden_size = image_data.shape
146 + elif is_list_of(image_data, torch.Tensor):
147 + image_feature_size = [item.shape[1] for item in image_data]
148 + else:
149 + raise TypeError(f"Invalid image type: {type(image_data)}")
150
151 if isinstance(vision_config, CLIPVisionConfig):
152 return input_processor_for_clip(
@@ -230,29 +243,24 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal):
243 return None
244
245 if pixel_values is not None:
233 - if not isinstance(pixel_values, torch.Tensor):
246 + if not isinstance(pixel_values, (torch.Tensor, list)):
247 raise ValueError("Incorrect type of pixel values. "
248 f"Got type: {type(pixel_values)}")
249
237 - # Remove the N dimension until multiple images are supported.
238 - pixel_values = pixel_values.squeeze(1)
239 -
250 return LlavaImagePixelInputs(
251 type="pixel_values",
242 - data=self._validate_pixel_values(pixel_values),
252 + data=self._validate_pixel_values(
253 + flatten_bn(pixel_values, concat=True)),
254 )
255
256 if image_embeds is not None:
246 - if not isinstance(image_embeds, torch.Tensor):
257 + if not isinstance(image_embeds, (torch.Tensor, list)):
258 raise ValueError("Incorrect type of image embeddings. "
259 f"Got type: {type(image_embeds)}")
260
250 - # Remove the N dimension until multiple images are supported.
251 - image_embeds = image_embeds.squeeze(1)
252 -
261 return LlavaImageEmbeddingInputs(
262 type="image_embeds",
255 - data=image_embeds,
263 + data=flatten_bn(image_embeds, concat=True),
264 )
265
266 raise AssertionError("This line should be unreachable.")
vllm/model_executor/models/llava_next.py
+3 -1
@@ -234,7 +234,9 @@ def input_processor_for_llava_next(ctx: InputContext, llm_inputs: LLMInputs):
234 for img in image_data
235 ]
236 elif isinstance(image_data, torch.Tensor):
237 - image_feature_size = image_data.shape[0]
237 + num_images, image_feature_size, hidden_size = image_data.shape
238 + elif is_list_of(image_data, torch.Tensor):
239 + image_feature_size = [item.shape[1] for item in image_data]
240 else:
241 raise TypeError(f"Invalid image type: {type(image_data)}")
242
vllm/model_executor/models/phi3v.py
+3 -1
@@ -424,7 +424,9 @@ def input_processor_for_phi3v(ctx: InputContext, llm_inputs: LLMInputs):
424 input_width=w,
425 input_height=h))
426 elif isinstance(image_data, torch.Tensor):
427 - image_feature_size = image_data.shape[0]
427 + num_images, image_feature_size, hidden_size = image_data.shape
428 + elif is_list_of(image_data, torch.Tensor):
429 + image_feature_size = [item.shape[1] for item in image_data]
430 else:
431 raise TypeError(f"Invalid image type: {type(image_data)}")
432
vllm/model_executor/models/siglip.py
+1 -1
@@ -110,7 +110,7 @@ def input_processor_for_siglip(
110 if isinstance(image_data, Image.Image):
111 image_feature_size = get_siglip_image_feature_size(hf_config)
112 elif isinstance(image_data, torch.Tensor):
113 - image_feature_size = image_data.shape[0]
113 + num_images, image_feature_size, hidden_size = image_data.shape
114 else:
115 raise TypeError(f"Invalid image type: {type(image_data)}")
116 else: