A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Model] Add multi-image input support for LLaVA-Next offline inference (#7230)
zifeitong committed
Aug 27, 2024 at 16:09 UTC
5340a2dccf06f502821b82db187a850ce566d07c
7 files changed
+174
-52
tests/conftest.py
+10
-11
@@ -41,6 +41,10 @@ _TEST_DIR = os.path.dirname(__file__)
41
_TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]
42
_LONG_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "summary.txt")]
43
44
+PromptImageInput = Union[List[Image.Image], List[List[Image.Image]]]
45
+PromptAudioInput = Union[List[Tuple[np.ndarray, int]],
46
+ List[List[Tuple[np.ndarray, int]]]]
47
+
48
49
def _read_prompts(filename: str) -> List[str]:
50
with open(filename, "r") as f:
@@ -161,7 +165,7 @@ def example_encoder_decoder_prompts(
165
decoder prompt) tuple.
166
167
Returns:
164
-
168
+
169
* Encoder prompt list
170
* Decoder prompt list (reverse of encoder prompt list)
171
'''
@@ -578,8 +582,7 @@ class VllmRunner:
582
self,
583
prompts: List[str],
584
sampling_params: SamplingParams,
581
- images: Optional[Union[List[Image.Image],
582
- List[List[Image.Image]]]] = None,
585
+ images: Optional[PromptImageInput] = None,
586
) -> List[Tuple[List[List[int]], List[str]]]:
587
if images is not None:
588
assert len(prompts) == len(images)
@@ -623,10 +626,8 @@ class VllmRunner:
626
self,
627
prompts: List[str],
628
sampling_params: SamplingParams,
626
- images: Optional[Union[List[Image.Image],
627
- List[List[Image.Image]]]] = None,
628
- audios: Optional[Union[List[Tuple[np.ndarray, int]],
629
- List[List[Tuple[np.ndarray, int]]]]] = None
629
+ images: Optional[PromptImageInput] = None,
630
+ audios: Optional[PromptAudioInput] = None,
631
) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
632
assert sampling_params.logprobs is not None
633
@@ -676,10 +677,8 @@ class VllmRunner:
677
prompts: List[str],
678
max_tokens: int,
679
num_logprobs: int,
679
- images: Optional[Union[List[Image.Image],
680
- List[List[Image.Image]]]] = None,
681
- audios: Optional[Union[List[Tuple[np.ndarray, int]],
682
- List[List[Tuple[np.ndarray, int]]]]] = None,
680
+ images: Optional[PromptImageInput] = None,
681
+ audios: Optional[PromptAudioInput] = None,
682
stop_token_ids: Optional[List[int]] = None,
683
) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
684
greedy_logprobs_params = SamplingParams(temperature=0.0,
tests/models/test_llava_next.py
+80
-13
@@ -6,24 +6,22 @@ from transformers import AutoConfig, AutoModelForVision2Seq, AutoTokenizer
6
from vllm.multimodal.utils import rescale_image_size
7
from vllm.sequence import SampleLogprobs
8
9
-from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
9
+from ..conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
10
+ _ImageAssets)
11
from .utils import check_logprobs_close
12
13
pytestmark = pytest.mark.vlm
14
14
-_PREFACE = (
15
- "A chat between a curious human and an artificial intelligence assistant. "
16
- "The assistant gives helpful, detailed, and polite answers to the human's "
17
- "questions.")
15
+_LIMIT_IMAGE_PER_PROMPT = 4
16
17
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
18
"stop_sign":
21
- f"{_PREFACE} USER: <image>\nWhat's the content of the image? ASSISTANT:",
19
+ "[INST] <image>\nWhat's the content of the image? [/INST]",
20
"cherry_blossom":
23
- f"{_PREFACE} USER: <image>\nWhat is the season? ASSISTANT:",
21
+ "[INST] <image>\nWhat is the season? [/INST]",
22
})
23
26
-models = ["llava-hf/llava-v1.6-vicuna-7b-hf"]
24
+models = ["llava-hf/llava-v1.6-mistral-7b-hf"]
25
26
27
def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
@@ -114,19 +112,43 @@ def run_test(
112
else:
113
raise ValueError("You must provide either `size_factors` or `sizes`")
114
115
+ _run_test(hf_runner,
116
+ vllm_runner,
117
+ inputs_per_image,
118
+ model,
119
+ dtype=dtype,
120
+ max_tokens=max_tokens,
121
+ num_logprobs=num_logprobs,
122
+ tensor_parallel_size=tensor_parallel_size,
123
+ distributed_executor_backend=distributed_executor_backend)
124
+
125
+
126
+def _run_test(
127
+ hf_runner: Type[HfRunner],
128
+ vllm_runner: Type[VllmRunner],
129
+ inputs: List[Tuple[List[str], PromptImageInput]],
130
+ model: str,
131
+ dtype: str,
132
+ max_tokens: int,
133
+ num_logprobs: int,
134
+ tensor_parallel_size: int,
135
+ distributed_executor_backend: Optional[str] = None,
136
+):
137
# max_model_len should be greater than image_feature_size
138
with vllm_runner(model,
139
dtype=dtype,
120
- max_model_len=4096,
140
+ max_model_len=10240,
141
tensor_parallel_size=tensor_parallel_size,
142
distributed_executor_backend=distributed_executor_backend,
123
- enforce_eager=True) as vllm_model:
143
+ enforce_eager=True,
144
+ limit_mm_per_prompt={"image": _LIMIT_IMAGE_PER_PROMPT
145
+ }) as vllm_model:
146
vllm_outputs_per_image = [
147
vllm_model.generate_greedy_logprobs(prompts,
148
max_tokens,
149
num_logprobs=num_logprobs,
150
images=images)
129
- for prompts, images in inputs_per_image
151
+ for prompts, images in inputs
152
]
153
154
with hf_runner(model, dtype=dtype,
@@ -136,7 +158,7 @@ def run_test(
158
max_tokens,
159
num_logprobs=num_logprobs,
160
images=images)
139
- for prompts, images in inputs_per_image
161
+ for prompts, images in inputs
162
]
163
164
for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
@@ -177,7 +199,7 @@ def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
199
200
All the image fixtures for the test is under tests/images.
201
For huggingface runner, we provide the PIL images as input.
180
- For vllm runner, we provide MultiModalDataDict objects
202
+ For vllm runner, we provide MultiModalDataDict objects
203
and corresponding MultiModalConfig as input.
204
Note, the text input is also adjusted to abide by vllm contract.
205
The text output is sanitized to be able to compare with hf.
@@ -216,3 +238,48 @@ def test_models_fixed_sizes(hf_runner, vllm_runner, image_assets, model, sizes,
238
num_logprobs=num_logprobs,
239
tensor_parallel_size=1,
240
)
241
+
242
+
243
+@pytest.mark.parametrize("model", models)
244
+@pytest.mark.parametrize("dtype", ["half"])
245
+@pytest.mark.parametrize("max_tokens", [128])
246
+@pytest.mark.parametrize("num_logprobs", [5])
247
+def test_models_multiple_image_inputs(hf_runner, vllm_runner, image_assets,
248
+ model, dtype, max_tokens,
249
+ num_logprobs) -> None:
250
+ stop_sign = image_assets[0].pil_image
251
+ cherry_blossom = image_assets[1].pil_image
252
+
253
+ inputs = [(
254
+ [
255
+ "[INST] <image><image>\nDescribe 2 images. [/INST]",
256
+ "[INST] <image><image>\nDescribe 2 images. [/INST]",
257
+ "[INST] <image><image><image><image>\nDescribe 4 images. [/INST]",
258
+ "[INST] <image>\nWhat is the season? [/INST]"
259
+ ],
260
+ [
261
+ [stop_sign, cherry_blossom],
262
+ # Images with different sizes and aspect-ratios
263
+ [
264
+ rescale_image_size(stop_sign, 0.1),
265
+ stop_sign,
266
+ ],
267
+ [
268
+ stop_sign,
269
+ rescale_image_size(stop_sign, 0.25),
270
+ cherry_blossom.resize((183, 488)),
271
+ cherry_blossom.resize((488, 183))
272
+ ],
273
+ cherry_blossom,
274
+ ])]
275
+
276
+ _run_test(
277
+ hf_runner,
278
+ vllm_runner,
279
+ inputs,
280
+ model,
281
+ dtype=dtype,
282
+ max_tokens=max_tokens,
283
+ num_logprobs=num_logprobs,
284
+ tensor_parallel_size=1,
285
+ )
tests/multimodal/test_utils.py
+34
-1
@@ -6,8 +6,10 @@ from typing import Dict, Tuple
6
import numpy as np
7
import pytest
8
from PIL import Image
9
+from transformers import AutoConfig, AutoTokenizer
10
10
-from vllm.multimodal.utils import async_fetch_image, fetch_image
11
+from vllm.multimodal.utils import (async_fetch_image, fetch_image,
12
+ repeat_and_pad_placeholder_tokens)
13
14
# Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
15
TEST_IMAGE_URLS = [
@@ -80,3 +82,34 @@ async def test_fetch_image_base64(url_images: Dict[str, Image.Image],
82
83
data_image_async = await async_fetch_image(data_url)
84
assert _image_equals(data_image_sync, data_image_async)
85
+
86
+
87
+@pytest.mark.parametrize("model", ["llava-hf/llava-v1.6-mistral-7b-hf"])
88
+def test_repeat_and_pad_placeholder_tokens(model):
89
+ config = AutoConfig.from_pretrained(model)
90
+ image_token_id = config.image_token_index
91
+
92
+ tokenizer = AutoTokenizer.from_pretrained(model)
93
+
94
+ test_cases = [
95
+ ("<image>", 2, "<image><image>", [32000, 32000]),
96
+ ("<image><image>", 2, "<image><image><image>", [32000, 32000, 32000]),
97
+ ("<image><image>", [3, 2], "<image><image><image><image><image>",
98
+ [32000, 32000, 32000, 32000, 32000]),
99
+ ("Image:<image>Image:<image>!", [3, 2],
100
+ "Image:<image><image><image>Image:<image><image>!",
101
+ [9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918]),
102
+ ("<image>", [3, 2], "<image><image><image>", [32000, 32000, 32000]),
103
+ ]
104
+
105
+ for prompt, repeat_count, expected_prompt, expected_token_ids in test_cases:
106
+ new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
107
+ tokenizer=tokenizer,
108
+ prompt=prompt,
109
+ prompt_token_ids=tokenizer.encode(prompt,
110
+ add_special_tokens=False),
111
+ placeholder_token_id=image_token_id,
112
+ repeat_count=repeat_count,
113
+ )
114
+ assert new_prompt == expected_prompt
115
+ assert new_token_ids == expected_token_ids
vllm/model_executor/models/clip.py
+4
-4
@@ -1,7 +1,7 @@
1
-"""Minimal implementation of CLIPVisionModel intended to be only used
1
+"""Minimal implementation of CLIPVisionModel intended to be only used
2
within a vision language model."""
3
from array import array
4
-from typing import Iterable, Optional, Tuple
4
+from typing import Iterable, List, Optional, Tuple, Union
5
6
import torch
7
import torch.nn as nn
@@ -84,7 +84,7 @@ def input_processor_for_clip(
84
llm_inputs: LLMInputs,
85
*,
86
image_token_id: int,
87
- image_feature_size_override: Optional[int] = None,
87
+ image_feature_size_override: Optional[Union[int, List[int]]] = None,
88
):
89
multi_modal_data = llm_inputs.get("multi_modal_data")
90
if multi_modal_data is None or "image" not in multi_modal_data:
@@ -217,7 +217,7 @@ class CLIPEncoderLayer(nn.Module):
217
218
class CLIPEncoder(nn.Module):
219
"""
220
- Transformer encoder consisting of `config.num_hidden_layers` self
220
+ Transformer encoder consisting of `config.num_hidden_layers` self
221
attention layers. Each layer is a [`CLIPEncoderLayer`].
222
223
Args:
vllm/model_executor/models/llava_next.py
+12
-2
@@ -19,6 +19,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
19
from vllm.model_executor.sampling_metadata import SamplingMetadata
20
from vllm.multimodal import MULTIMODAL_REGISTRY
21
from vllm.sequence import IntermediateTensors, SamplerOutput
22
+from vllm.utils import is_list_of
23
24
from .clip import (CLIPVisionModel, dummy_image_for_clip,
25
dummy_seq_data_for_clip, get_clip_image_feature_size,
@@ -223,6 +224,13 @@ def input_processor_for_llava_next(ctx: InputContext, llm_inputs: LLMInputs):
224
input_height=height,
225
input_width=width,
226
)
227
+ elif is_list_of(image_data, Image.Image):
228
+ image_feature_size = [
229
+ get_llava_next_image_feature_size(hf_config,
230
+ input_height=img.height,
231
+ input_width=img.width)
232
+ for img in image_data
233
+ ]
234
elif isinstance(image_data, torch.Tensor):
235
image_feature_size = image_data.shape[0]
236
else:
@@ -425,7 +433,10 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal):
433
self.config.image_grid_pinpoints,
434
self.config.vision_config.image_size,
435
)
428
- other_patch_embeds = other_patch_embeds \
436
+ num_patches = num_patch_height * num_patch_width
437
+
438
+ # Image patches might be padded for batch processing
439
+ other_patch_embeds = other_patch_embeds[:num_patches] \
440
.view(num_patch_height, num_patch_width, height, width, -1)
441
442
if "unpad" in strategy:
@@ -496,7 +507,6 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal):
507
self,
508
image_input: LlavaNextImageInputs,
509
) -> Union[torch.Tensor, List[torch.Tensor]]:
499
-
510
if image_input["type"] == "image_embeds":
511
return [image_input["data"]]
512
vllm/model_executor/models/siglip.py
+2
-2
@@ -3,7 +3,7 @@ within a vision language model."""
3
4
import math
5
from array import array
6
-from typing import Iterable, Optional, Tuple
6
+from typing import Iterable, List, Optional, Tuple, Union
7
8
import torch
9
from PIL import Image
@@ -93,7 +93,7 @@ def input_processor_for_siglip(
93
llm_inputs: LLMInputs,
94
*,
95
image_token_id: int,
96
- image_feature_size_override: Optional[int] = None,
96
+ image_feature_size_override: Optional[Union[int, List[int]]] = None,
97
):
98
multi_modal_data = llm_inputs.get("multi_modal_data")
99
if multi_modal_data is None or "image" not in multi_modal_data:
vllm/multimodal/utils.py
+32
-19
@@ -189,10 +189,13 @@ def repeat_and_pad_placeholder_tokens(
189
prompt_token_ids: List[int],
190
*,
191
placeholder_token_id: int,
192
- repeat_count: int = 1,
192
+ repeat_count: Union[int, List[int]],
193
pad_token_left: Optional[int] = None,
194
pad_token_right: Optional[int] = None,
195
) -> Tuple[Optional[str], List[int]]:
196
+ if isinstance(repeat_count, int):
197
+ repeat_count = [repeat_count]
198
+
199
if prompt is None:
200
new_prompt = None
201
else:
@@ -201,13 +204,6 @@ def repeat_and_pad_placeholder_tokens(
204
tokenizer.decode(pad_token_left))
205
pad_token_str_right = (None if pad_token_right is None else
206
tokenizer.decode(pad_token_right))
204
- replacement_str = "".join(
205
- repeat_and_pad_token(
206
- placeholder_token_str,
207
- repeat_count=repeat_count,
208
- pad_token_left=pad_token_str_left,
209
- pad_token_right=pad_token_str_right,
210
- ))
207
208
placeholder_token_count = prompt.count(placeholder_token_str)
209
# This is an arbitrary number to distinguish between the two cases
@@ -216,28 +212,45 @@ def repeat_and_pad_placeholder_tokens(
212
"Please follow the prompt format that is "
213
"documented on HuggingFace which does not involve "
214
"repeating %s tokens.", placeholder_token_str)
219
- elif placeholder_token_count > 1:
220
- logger.warning("Multiple multi-modal input is not supported yet, "
221
- "so any extra placeholder tokens will be treated "
222
- "as plain text.")
223
-
224
- # The image tokens are removed to be consistent with HuggingFace
225
- new_prompt = prompt.replace(placeholder_token_str, replacement_str, 1)
215
+ if placeholder_token_count < len(repeat_count):
216
+ logger.warning(
217
+ "The number of multi-modal placeholder tokens in the prompt "
218
+ "is less than the number of multi-modal inputs. Extra "
219
+ "placeholder tokens will be treated as plain text")
220
+ repeat_count = repeat_count[:placeholder_token_count]
221
+
222
+ prompt_parts = prompt.split(placeholder_token_str,
223
+ maxsplit=len(repeat_count))
224
+ new_prompt = ""
225
+ for i, repeat_count_item in enumerate(repeat_count):
226
+ replacement_str = "".join(
227
+ repeat_and_pad_token(
228
+ placeholder_token_str,
229
+ repeat_count=repeat_count_item,
230
+ pad_token_left=pad_token_str_left,
231
+ pad_token_right=pad_token_str_right,
232
+ ))
233
+ # The image tokens are removed to be consistent with HuggingFace
234
+ new_prompt += prompt_parts[i] + replacement_str
235
+ new_prompt += prompt_parts[-1]
236
237
new_token_ids: List[int] = []
238
+ placeholder_token_idx = 0
239
for i, token in enumerate(prompt_token_ids):
240
if token == placeholder_token_id:
241
replacement_ids = repeat_and_pad_token(
242
placeholder_token_id,
232
- repeat_count=repeat_count,
243
+ repeat_count=repeat_count[placeholder_token_idx],
244
pad_token_left=pad_token_left,
245
pad_token_right=pad_token_right,
246
)
247
new_token_ids.extend(replacement_ids)
248
+ placeholder_token_idx += 1
249
238
- # No need to further scan the list since we only replace once
239
- new_token_ids.extend(prompt_token_ids[i + 1:])
240
- break
250
+ # No need to further scan the list since we replaced all tokens
251
+ if placeholder_token_idx >= len(repeat_count):
252
+ new_token_ids.extend(prompt_token_ids[i + 1:])
253
+ break
254
else:
255
new_token_ids.append(token)
256