A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Model] Multi-input support for LLaVA (#8238)
Cyrus Leung committed
Sep 7, 2024 at 10:57 UTC
2f707fcb35c5bc4b9164cf2bbce0254a72f7348b
10 files changed
+176
-45
docs/source/models/supported_models.rst
+8
-8
@@ -219,7 +219,7 @@ Multimodal Language Models
219
-
220
* - :code:`LlavaForConditionalGeneration`
221
- LLaVA-1.5
222
- - Image\ :sup:`E`
222
+ - Image\ :sup:`E+`
223
- :code:`llava-hf/llava-1.5-7b-hf`, :code:`llava-hf/llava-1.5-13b-hf`, etc.
224
-
225
* - :code:`LlavaNextForConditionalGeneration`
@@ -227,6 +227,11 @@ Multimodal Language Models
227
- Image\ :sup:`E+`
228
- :code:`llava-hf/llava-v1.6-mistral-7b-hf`, :code:`llava-hf/llava-v1.6-vicuna-7b-hf`, etc.
229
-
230
+ * - :code:`MiniCPMV`
231
+ - MiniCPM-V
232
+ - Image\ :sup:`+`
233
+ - :code:`openbmb/MiniCPM-V-2` (see note), :code:`openbmb/MiniCPM-Llama3-V-2_5`, :code:`openbmb/MiniCPM-V-2_6`, etc.
234
+ -
235
* - :code:`PaliGemmaForConditionalGeneration`
236
- PaliGemma
237
- Image\ :sup:`E`
@@ -237,14 +242,9 @@ Multimodal Language Models
242
- Image\ :sup:`E+`
243
- :code:`microsoft/Phi-3-vision-128k-instruct`, :code:`microsoft/Phi-3.5-vision-instruct` etc.
244
-
240
- * - :code:`MiniCPMV`
241
- - MiniCPM-V
242
- - Image\ :sup:`+`
243
- - :code:`openbmb/MiniCPM-V-2` (see note), :code:`openbmb/MiniCPM-Llama3-V-2_5`, :code:`openbmb/MiniCPM-V-2_6`, etc.
244
- -
245
* - :code:`QWenLMHeadModel`
246
- - Qwen
247
- - Image
246
+ - Qwen-VL
247
+ - Image\ :sup:`E`
248
- :code:`Qwen/Qwen-VL`, :code:`Qwen/Qwen-VL-Chat`, etc.
249
-
250
* - :code:`UltravoxModel`
tests/conftest.py
+6
-6
@@ -278,7 +278,7 @@ class HfRunner:
278
def generate(
279
self,
280
prompts: List[str],
281
- images: Optional[List[Image.Image]] = None,
281
+ images: Optional[PromptImageInput] = None,
282
**kwargs: Any,
283
) -> List[Tuple[List[List[int]], List[str]]]:
284
if images:
@@ -314,7 +314,7 @@ class HfRunner:
314
self,
315
prompts: List[str],
316
max_tokens: int,
317
- images: Optional[List[Image.Image]] = None,
317
+ images: Optional[PromptImageInput] = None,
318
**kwargs: Any,
319
) -> List[Tuple[List[int], str]]:
320
outputs = self.generate(prompts,
@@ -351,7 +351,7 @@ class HfRunner:
351
self,
352
prompts: List[str],
353
max_tokens: int,
354
- images: Optional[List[Image.Image]] = None,
354
+ images: Optional[PromptImageInput] = None,
355
**kwargs: Any,
356
) -> List[List[torch.Tensor]]:
357
all_logprobs: List[List[torch.Tensor]] = []
@@ -433,8 +433,8 @@ class HfRunner:
433
prompts: List[str],
434
max_tokens: int,
435
num_logprobs: int,
436
- images: Optional[List[Image.Image]] = None,
437
- audios: Optional[List[Tuple[np.ndarray, int]]] = None,
436
+ images: Optional[PromptImageInput] = None,
437
+ audios: Optional[PromptAudioInput] = None,
438
**kwargs: Any,
439
) -> List[Tuple[List[int], str, List[Dict[int, float]]]]:
440
all_logprobs: List[List[Dict[int, float]]] = []
@@ -671,7 +671,7 @@ class VllmRunner:
671
self,
672
prompts: List[str],
673
max_tokens: int,
674
- images: Optional[List[Image.Image]] = None,
674
+ images: Optional[PromptImageInput] = None,
675
) -> List[Tuple[List[int], str]]:
676
greedy_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)
677
outputs = self.generate(prompts, greedy_params, images=images)
tests/distributed/test_multimodal_broadcast.py
+4
-2
@@ -35,9 +35,11 @@ def test_models(hf_runner, vllm_runner, image_assets, model: str,
35
if model.startswith("llava-hf/llava-1.5"):
36
from ..models.test_llava import models, run_test
37
elif model.startswith("llava-hf/llava-v1.6"):
38
- from ..models.test_llava_next import models, run_test
38
+ from ..models.test_llava_next import run_test # type: ignore[no-redef]
39
+ from ..models.test_llava_next import models
40
elif model.startswith("facebook/chameleon"):
40
- from ..models.test_chameleon import models, run_test
41
+ from ..models.test_chameleon import run_test # type: ignore[no-redef]
42
+ from ..models.test_chameleon import models
43
else:
44
raise NotImplementedError(f"Unsupported model: {model}")
45
tests/models/test_llava.py
+129
-12
@@ -1,4 +1,4 @@
1
-from typing import List, Optional, Tuple, Type
1
+from typing import List, Optional, Tuple, Type, overload
2
3
import pytest
4
from transformers import (AutoConfig, AutoModelForVision2Seq, AutoTokenizer,
@@ -8,11 +8,14 @@ from vllm.multimodal.utils import rescale_image_size
8
from vllm.sequence import SampleLogprobs
9
from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
10
11
-from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
11
+from ..conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
12
+ _ImageAssets)
13
from .utils import check_logprobs_close
14
15
pytestmark = pytest.mark.vlm
16
17
+_LIMIT_IMAGE_PER_PROMPT = 4
18
+
19
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
20
"stop_sign":
21
"USER: <image>\nWhat's the content of the image?\nASSISTANT:",
@@ -52,6 +55,7 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
55
return hf_output_ids, hf_output_str, out_logprobs
56
57
58
+@overload
59
def run_test(
60
hf_runner: Type[HfRunner],
61
vllm_runner: Type[VllmRunner],
@@ -64,6 +68,78 @@ def run_test(
68
num_logprobs: int,
69
tensor_parallel_size: int,
70
distributed_executor_backend: Optional[str] = None,
71
+):
72
+ ...
73
+
74
+
75
+@overload
76
+def run_test(
77
+ hf_runner: Type[HfRunner],
78
+ vllm_runner: Type[VllmRunner],
79
+ image_assets: _ImageAssets,
80
+ model: str,
81
+ *,
82
+ sizes: List[Tuple[int, int]],
83
+ dtype: str,
84
+ max_tokens: int,
85
+ num_logprobs: int,
86
+ tensor_parallel_size: int,
87
+ distributed_executor_backend: Optional[str] = None,
88
+):
89
+ ...
90
+
91
+
92
+def run_test(
93
+ hf_runner: Type[HfRunner],
94
+ vllm_runner: Type[VllmRunner],
95
+ image_assets: _ImageAssets,
96
+ model: str,
97
+ *,
98
+ size_factors: Optional[List[float]] = None,
99
+ sizes: Optional[List[Tuple[int, int]]] = None,
100
+ dtype: str,
101
+ max_tokens: int,
102
+ num_logprobs: int,
103
+ tensor_parallel_size: int,
104
+ distributed_executor_backend: Optional[str] = None,
105
+):
106
+ images = [asset.pil_image for asset in image_assets]
107
+
108
+ if size_factors is not None:
109
+ inputs_per_image = [(
110
+ [prompt for _ in size_factors],
111
+ [rescale_image_size(image, factor) for factor in size_factors],
112
+ ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
113
+ elif sizes is not None:
114
+ inputs_per_image = [(
115
+ [prompt for _ in sizes],
116
+ [image.resize(size) for size in sizes],
117
+ ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
118
+ else:
119
+ raise ValueError("You must provide either `size_factors` or `sizes`")
120
+
121
+ _run_test(hf_runner,
122
+ vllm_runner,
123
+ inputs_per_image,
124
+ model,
125
+ dtype=dtype,
126
+ max_tokens=max_tokens,
127
+ num_logprobs=num_logprobs,
128
+ tensor_parallel_size=tensor_parallel_size,
129
+ distributed_executor_backend=distributed_executor_backend)
130
+
131
+
132
+def _run_test(
133
+ hf_runner: Type[HfRunner],
134
+ vllm_runner: Type[VllmRunner],
135
+ inputs: List[Tuple[List[str], PromptImageInput]],
136
+ model: str,
137
+ *,
138
+ dtype: str,
139
+ max_tokens: int,
140
+ num_logprobs: int,
141
+ tensor_parallel_size: int,
142
+ distributed_executor_backend: Optional[str] = None,
143
):
144
"""Inference result should be the same between hf and vllm.
145
@@ -85,13 +161,6 @@ def run_test(
161
else:
162
mantis_processor = None
163
88
- images = [asset.pil_image for asset in image_assets]
89
-
90
- inputs_per_image = [(
91
- [prompt for _ in size_factors],
92
- [rescale_image_size(image, factor) for factor in size_factors],
93
- ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
94
-
164
# NOTE: take care of the order. run vLLM first, and then run HF.
165
# vLLM needs a fresh new process without cuda initialization.
166
# if we run HF first, the cuda initialization will be done and it
@@ -100,15 +169,18 @@ def run_test(
169
# max_model_len should be greater than image_feature_size
170
with vllm_runner(model,
171
dtype=dtype,
172
+ max_model_len=4096,
173
tensor_parallel_size=tensor_parallel_size,
174
distributed_executor_backend=distributed_executor_backend,
105
- enforce_eager=True) as vllm_model:
175
+ enforce_eager=True,
176
+ limit_mm_per_prompt={"image": _LIMIT_IMAGE_PER_PROMPT
177
+ }) as vllm_model:
178
vllm_outputs_per_image = [
179
vllm_model.generate_greedy_logprobs(prompts,
180
max_tokens,
181
num_logprobs=num_logprobs,
182
images=images)
111
- for prompts, images in inputs_per_image
183
+ for prompts, images in inputs
184
]
185
186
if mantis_processor is not None:
@@ -131,7 +203,7 @@ def run_test(
203
max_tokens,
204
num_logprobs=num_logprobs,
205
images=images)
134
- for prompts, images in inputs_per_image
206
+ for prompts, images in inputs
207
]
208
209
for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
@@ -181,6 +253,51 @@ def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
253
)
254
255
256
+@pytest.mark.parametrize("model", models)
257
+@pytest.mark.parametrize("dtype", ["half"])
258
+@pytest.mark.parametrize("max_tokens", [128])
259
+@pytest.mark.parametrize("num_logprobs", [5])
260
+def test_models_multiple_image_inputs(hf_runner, vllm_runner, image_assets,
261
+ model, dtype, max_tokens,
262
+ num_logprobs) -> None:
263
+ stop_sign = image_assets[0].pil_image
264
+ cherry_blossom = image_assets[1].pil_image
265
+
266
+ inputs = [(
267
+ [
268
+ "USER: <image><image>\nDescribe 2 images.\nASSISTANT:",
269
+ "USER: <image><image>\nDescribe 2 images.\nASSISTANT:",
270
+ "USER: <image><image><image><image>\nDescribe 4 images.\nASSISTANT:", # noqa: E501
271
+ "USER: <image>\nWhat is the season?\nASSISTANT:",
272
+ ],
273
+ [
274
+ [stop_sign, cherry_blossom],
275
+ # Images with different sizes and aspect-ratios
276
+ [
277
+ rescale_image_size(stop_sign, 0.1),
278
+ stop_sign,
279
+ ],
280
+ [
281
+ stop_sign,
282
+ rescale_image_size(stop_sign, 0.25),
283
+ cherry_blossom.resize((183, 488)),
284
+ cherry_blossom.resize((488, 183))
285
+ ],
286
+ cherry_blossom,
287
+ ])]
288
+
289
+ _run_test(
290
+ hf_runner,
291
+ vllm_runner,
292
+ inputs,
293
+ model,
294
+ dtype=dtype,
295
+ max_tokens=max_tokens,
296
+ num_logprobs=num_logprobs,
297
+ tensor_parallel_size=1,
298
+ )
299
+
300
+
301
@pytest.mark.parametrize("model", models)
302
def test_context_length_too_short(vllm_runner, image_assets, model):
303
images = [asset.pil_image for asset in image_assets]
vllm/model_executor/models/clip.py
+1
-1
@@ -105,7 +105,7 @@ def input_processor_for_clip(
105
if isinstance(image_data, Image.Image):
106
image_feature_size = get_clip_image_feature_size(hf_config)
107
elif isinstance(image_data, torch.Tensor):
108
- image_feature_size = image_data.shape[0]
108
+ num_images, image_feature_size, hidden_size = image_data.shape
109
else:
110
raise TypeError(f"Invalid image type: {type(image_data)}")
111
else:
vllm/model_executor/models/internvl.py
+1
-1
@@ -209,7 +209,7 @@ def input_processor_for_internvl(ctx: InputContext, llm_inputs: LLMInputs):
209
image_feature_size = num_blocks * num_patches
210
211
elif isinstance(image_data, torch.Tensor):
212
- image_feature_size = image_data.shape[0]
212
+ num_images, image_feature_size, hidden_size = image_data.shape
213
else:
214
raise TypeError(f"Invalid image type: {type(image_data)}")
215
vllm/model_executor/models/llava.py
+20
-12
@@ -4,6 +4,7 @@ from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
4
5
import torch
6
import torch.nn as nn
7
+from PIL import Image
8
from transformers import CLIPVisionConfig, LlavaConfig, SiglipVisionConfig
9
10
from vllm.attention import AttentionMetadata
@@ -16,6 +17,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
17
from vllm.model_executor.sampling_metadata import SamplingMetadata
18
from vllm.multimodal import MULTIMODAL_REGISTRY
19
from vllm.sequence import IntermediateTensors
20
+from vllm.utils import is_list_of
21
22
from .clip import (CLIPVisionModel, dummy_image_for_clip,
23
dummy_seq_data_for_clip, get_max_clip_image_tokens,
@@ -24,7 +26,7 @@ from .interfaces import SupportsMultiModal
26
from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
27
dummy_seq_data_for_siglip, get_max_siglip_image_tokens,
28
input_processor_for_siglip)
27
-from .utils import (filter_weights, init_vllm_registered_model,
29
+from .utils import (filter_weights, flatten_bn, init_vllm_registered_model,
30
merge_multimodal_embeddings)
31
32
@@ -133,7 +135,18 @@ def input_processor_for_llava(ctx: InputContext, llm_inputs: LLMInputs):
135
hf_config = ctx.get_hf_config(LlavaConfig)
136
vision_config = hf_config.vision_config
137
136
- image_feature_size = get_max_llava_image_tokens(ctx)
138
+ image_data = multi_modal_data["image"]
139
+ if isinstance(image_data, Image.Image):
140
+ image_feature_size = get_max_llava_image_tokens(ctx)
141
+ elif is_list_of(image_data, Image.Image):
142
+ image_feature_size = [get_max_llava_image_tokens(ctx)
143
+ ] * len(image_data)
144
+ elif isinstance(image_data, torch.Tensor):
145
+ num_images, image_feature_size, hidden_size = image_data.shape
146
+ elif is_list_of(image_data, torch.Tensor):
147
+ image_feature_size = [item.shape[1] for item in image_data]
148
+ else:
149
+ raise TypeError(f"Invalid image type: {type(image_data)}")
150
151
if isinstance(vision_config, CLIPVisionConfig):
152
return input_processor_for_clip(
@@ -230,29 +243,24 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal):
243
return None
244
245
if pixel_values is not None:
233
- if not isinstance(pixel_values, torch.Tensor):
246
+ if not isinstance(pixel_values, (torch.Tensor, list)):
247
raise ValueError("Incorrect type of pixel values. "
248
f"Got type: {type(pixel_values)}")
249
237
- # Remove the N dimension until multiple images are supported.
238
- pixel_values = pixel_values.squeeze(1)
239
-
250
return LlavaImagePixelInputs(
251
type="pixel_values",
242
- data=self._validate_pixel_values(pixel_values),
252
+ data=self._validate_pixel_values(
253
+ flatten_bn(pixel_values, concat=True)),
254
)
255
256
if image_embeds is not None:
246
- if not isinstance(image_embeds, torch.Tensor):
257
+ if not isinstance(image_embeds, (torch.Tensor, list)):
258
raise ValueError("Incorrect type of image embeddings. "
259
f"Got type: {type(image_embeds)}")
260
250
- # Remove the N dimension until multiple images are supported.
251
- image_embeds = image_embeds.squeeze(1)
252
-
261
return LlavaImageEmbeddingInputs(
262
type="image_embeds",
255
- data=image_embeds,
263
+ data=flatten_bn(image_embeds, concat=True),
264
)
265
266
raise AssertionError("This line should be unreachable.")
vllm/model_executor/models/llava_next.py
+3
-1
@@ -234,7 +234,9 @@ def input_processor_for_llava_next(ctx: InputContext, llm_inputs: LLMInputs):
234
for img in image_data
235
]
236
elif isinstance(image_data, torch.Tensor):
237
- image_feature_size = image_data.shape[0]
237
+ num_images, image_feature_size, hidden_size = image_data.shape
238
+ elif is_list_of(image_data, torch.Tensor):
239
+ image_feature_size = [item.shape[1] for item in image_data]
240
else:
241
raise TypeError(f"Invalid image type: {type(image_data)}")
242
vllm/model_executor/models/phi3v.py
+3
-1
@@ -424,7 +424,9 @@ def input_processor_for_phi3v(ctx: InputContext, llm_inputs: LLMInputs):
424
input_width=w,
425
input_height=h))
426
elif isinstance(image_data, torch.Tensor):
427
- image_feature_size = image_data.shape[0]
427
+ num_images, image_feature_size, hidden_size = image_data.shape
428
+ elif is_list_of(image_data, torch.Tensor):
429
+ image_feature_size = [item.shape[1] for item in image_data]
430
else:
431
raise TypeError(f"Invalid image type: {type(image_data)}")
432
vllm/model_executor/models/siglip.py
+1
-1
@@ -110,7 +110,7 @@ def input_processor_for_siglip(
110
if isinstance(image_data, Image.Image):
111
image_feature_size = get_siglip_image_feature_size(hf_config)
112
elif isinstance(image_data, torch.Tensor):
113
- image_feature_size = image_data.shape[0]
113
+ num_images, image_feature_size, hidden_size = image_data.shape
114
else:
115
raise TypeError(f"Invalid image type: {type(image_data)}")
116
else: