A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Model][VLM] Support multi-images inputs for InternVL2 models (#8201)
Isotr0py committed
Sep 7, 2024 at 16:38 UTC
e807125936a9db796746b67ba72c222b5c26582e
5 files changed
+199
-57
docs/source/models/supported_models.rst
+1
-1
@@ -214,7 +214,7 @@ Multimodal Language Models
214
-
215
* - :code:`InternVLChatModel`
216
- InternVL2
217
- - Image\ :sup:`E`
217
+ - Image\ :sup:`E+`
218
- :code:`OpenGVLab/InternVL2-4B`, :code:`OpenGVLab/InternVL2-8B`, etc.
219
-
220
* - :code:`LlavaForConditionalGeneration`
examples/offline_inference_vision_language_multi_image.py
+76
-18
@@ -6,7 +6,9 @@ by the model.
6
from argparse import Namespace
7
from typing import List
8
9
-from vllm import LLM
9
+from transformers import AutoTokenizer
10
+
11
+from vllm import LLM, SamplingParams
12
from vllm.multimodal.utils import fetch_image
13
from vllm.utils import FlexibleArgumentParser
14
@@ -17,36 +19,84 @@ IMAGE_URLS = [
19
]
20
21
20
-def _load_phi3v(image_urls: List[str]):
21
- return LLM(
22
+def load_phi3v(question, image_urls: List[str]):
23
+ llm = LLM(
24
model="microsoft/Phi-3.5-vision-instruct",
25
trust_remote_code=True,
26
max_model_len=4096,
27
limit_mm_per_prompt={"image": len(image_urls)},
28
)
27
-
28
-
29
-def run_phi3v_generate(question: str, image_urls: List[str]):
30
- llm = _load_phi3v(image_urls)
31
-
29
placeholders = "\n".join(f"<|image_{i}|>"
30
for i, _ in enumerate(image_urls, start=1))
31
prompt = f"<|user|>\n{placeholders}\n{question}<|end|>\n<|assistant|>\n"
32
+ stop_token_ids = None
33
+ return llm, prompt, stop_token_ids
34
36
- outputs = llm.generate({
37
- "prompt": prompt,
38
- "multi_modal_data": {
39
- "image": [fetch_image(url) for url in image_urls]
35
+
36
+def load_internvl(question, image_urls: List[str]):
37
+ model_name = "OpenGVLab/InternVL2-2B"
38
+
39
+ llm = LLM(
40
+ model=model_name,
41
+ trust_remote_code=True,
42
+ max_num_seqs=5,
43
+ max_model_len=4096,
44
+ limit_mm_per_prompt={"image": len(image_urls)},
45
+ )
46
+
47
+ placeholders = "\n".join(f"Image-{i}: <image>\n"
48
+ for i, _ in enumerate(image_urls, start=1))
49
+ messages = [{'role': 'user', 'content': f"{placeholders}\n{question}"}]
50
+
51
+ tokenizer = AutoTokenizer.from_pretrained(model_name,
52
+ trust_remote_code=True)
53
+ prompt = tokenizer.apply_chat_template(messages,
54
+ tokenize=False,
55
+ add_generation_prompt=True)
56
+
57
+ # Stop tokens for InternVL
58
+ # models variants may have different stop tokens
59
+ # please refer to the model card for the correct "stop words":
60
+ # https://huggingface.co/OpenGVLab/InternVL2-2B#service
61
+ stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"]
62
+ stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
63
+ return llm, prompt, stop_token_ids
64
+
65
+
66
+model_example_map = {
67
+ "phi3_v": load_phi3v,
68
+ "internvl_chat": load_internvl,
69
+}
70
+
71
+
72
+def run_generate(model, question: str, image_urls: List[str]):
73
+ llm, prompt, stop_token_ids = model_example_map[model](question,
74
+ image_urls)
75
+
76
+ sampling_params = SamplingParams(temperature=0.0,
77
+ max_tokens=128,
78
+ stop_token_ids=stop_token_ids)
79
+
80
+ outputs = llm.generate(
81
+ {
82
+ "prompt": prompt,
83
+ "multi_modal_data": {
84
+ "image": [fetch_image(url) for url in image_urls]
85
+ },
86
},
41
- })
87
+ sampling_params=sampling_params)
88
89
for o in outputs:
90
generated_text = o.outputs[0].text
91
print(generated_text)
92
93
48
-def run_phi3v_chat(question: str, image_urls: List[str]):
49
- llm = _load_phi3v(image_urls)
94
+def run_chat(model: str, question: str, image_urls: List[str]):
95
+ llm, _, stop_token_ids = model_example_map[model](question, image_urls)
96
+
97
+ sampling_params = SamplingParams(temperature=0.0,
98
+ max_tokens=128,
99
+ stop_token_ids=stop_token_ids)
100
101
outputs = llm.chat([{
102
"role":
@@ -63,7 +113,8 @@ def run_phi3v_chat(question: str, image_urls: List[str]):
113
},
114
} for image_url in image_urls),
115
],
66
- }])
116
+ }],
117
+ sampling_params=sampling_params)
118
119
for o in outputs:
120
generated_text = o.outputs[0].text
@@ -71,12 +122,13 @@ def run_phi3v_chat(question: str, image_urls: List[str]):
122
123
124
def main(args: Namespace):
125
+ model = args.model_type
126
method = args.method
127
128
if method == "generate":
77
- run_phi3v_generate(QUESTION, IMAGE_URLS)
129
+ run_generate(model, QUESTION, IMAGE_URLS)
130
elif method == "chat":
79
- run_phi3v_chat(QUESTION, IMAGE_URLS)
131
+ run_chat(model, QUESTION, IMAGE_URLS)
132
else:
133
raise ValueError(f"Invalid method: {method}")
134
@@ -85,6 +137,12 @@ if __name__ == "__main__":
137
parser = FlexibleArgumentParser(
138
description='Demo on using vLLM for offline inference with '
139
'vision language models that support multi-image input')
140
+ parser.add_argument('--model-type',
141
+ '-m',
142
+ type=str,
143
+ default="phi3_v",
144
+ choices=model_example_map.keys(),
145
+ help='Huggingface "model_type".')
146
parser.add_argument("--method",
147
type=str,
148
default="generate",
tests/models/test_internvl.py
+73
-19
@@ -1,5 +1,5 @@
1
import types
2
-from typing import List, Optional, Tuple, Type
2
+from typing import List, Optional, Tuple, Type, Union
3
4
import pytest
5
import torch
@@ -9,7 +9,8 @@ from transformers import AutoConfig
9
from vllm.multimodal.utils import rescale_image_size
10
from vllm.utils import is_cpu
11
12
-from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
12
+from ..conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
13
+ _ImageAssets)
14
from .utils import check_logprobs_close
15
16
pytestmark = pytest.mark.vlm
@@ -20,6 +21,7 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
21
"cherry_blossom":
22
"<|im_start|>User\n<image>\nWhat is the season?<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
23
})
24
+HF_MULTIIMAGE_IMAGE_PROMPT = "<|im_start|>User\nImage-1: <image>\nImage-2: <image>\nDescribe the two images in detail.<|im_end|>\n<|im_start|>Assistant\n" # noqa: E501
25
26
models = [
27
"OpenGVLab/InternVL2-1B",
@@ -64,13 +66,13 @@ def generate(
66
def run_test(
67
hf_runner: Type[HfRunner],
68
vllm_runner: Type[VllmRunner],
67
- image_assets: _ImageAssets,
69
+ inputs: List[Tuple[List[str], PromptImageInput]],
70
model: str,
71
*,
70
- size_factors: List[float],
72
dtype: str,
73
max_tokens: int,
74
num_logprobs: int,
75
+ mm_limit: int,
76
tensor_parallel_size: int,
77
distributed_executor_backend: Optional[str] = None,
78
):
@@ -83,12 +85,6 @@ def run_test(
85
Note, the text input is also adjusted to abide by vllm contract.
86
The text output is sanitized to be able to compare with hf.
87
"""
86
- images = [asset.pil_image for asset in image_assets]
87
-
88
- inputs_per_image = [(
89
- [prompt for _ in size_factors],
90
- [rescale_image_size(image, factor) for factor in size_factors],
91
- ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
88
89
# NOTE: take care of the order. run vLLM first, and then run HF.
90
# vLLM needs a fresh new process without cuda initialization.
@@ -110,13 +106,21 @@ def run_test(
106
self.max_num = self.config.max_dynamic_patch
107
self.image_size = self.vision_config.image_size
108
113
- def __call__(self, text: str, images: Image, **kwargs):
109
+ def __call__(self, text: str, images: Union[Image, List[Image]],
110
+ **kwargs):
111
from vllm.model_executor.models.internvl import (
112
IMG_CONTEXT, IMG_END, IMG_START, image_to_pixel_values)
116
- pixel_values = image_to_pixel_values(
117
- images, self.image_size, self.min_num, self.max_num,
118
- self.use_thumbnail).to(self.dtype)
119
- num_patches_list = [pixel_values.shape[0]]
113
+ images = [images] if isinstance(images, Image) else images
114
+ pixel_values = [
115
+ image_to_pixel_values(image, self.image_size, self.min_num,
116
+ self.max_num,
117
+ self.use_thumbnail).to(self.dtype)
118
+ for image in images
119
+ ]
120
+ num_patches_list = [
121
+ pixel_value.shape[0] for pixel_value in pixel_values
122
+ ]
123
+ pixel_values = torch.cat(pixel_values, dim=0)
124
for num_patches in num_patches_list:
125
context_tokens = IMG_CONTEXT * self.num_image_token \
126
* num_patches
@@ -130,6 +134,7 @@ def run_test(
134
with vllm_runner(model,
135
max_model_len=4096,
136
dtype=dtype,
137
+ limit_mm_per_prompt={"image": mm_limit},
138
tensor_parallel_size=tensor_parallel_size,
139
distributed_executor_backend=distributed_executor_backend,
140
enforce_eager=True) as vllm_model:
@@ -138,7 +143,7 @@ def run_test(
143
max_tokens,
144
num_logprobs=num_logprobs,
145
images=images)
141
- for prompts, images in inputs_per_image
146
+ for prompts, images in inputs
147
]
148
149
with hf_runner(model, dtype=dtype) as hf_model:
@@ -156,7 +161,7 @@ def run_test(
161
num_logprobs=num_logprobs,
162
images=hf_images,
163
eos_token_id=eos_token_id)
159
- for prompts, hf_images in inputs_per_image
164
+ for prompts, hf_images in inputs
165
]
166
167
for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
@@ -264,15 +269,64 @@ if is_cpu():
269
@torch.inference_mode()
270
def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
271
dtype: str, max_tokens: int, num_logprobs: int) -> None:
272
+ images = [asset.pil_image for asset in image_assets]
273
+
274
+ inputs_per_image = [(
275
+ [prompt for _ in size_factors],
276
+ [rescale_image_size(image, factor) for factor in size_factors],
277
+ ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
278
+
279
run_test(
280
hf_runner,
281
vllm_runner,
270
- image_assets,
282
+ inputs_per_image,
283
+ model,
284
+ dtype=dtype,
285
+ max_tokens=max_tokens,
286
+ num_logprobs=num_logprobs,
287
+ mm_limit=1,
288
+ tensor_parallel_size=1,
289
+ )
290
+
291
+
292
+@pytest.mark.parametrize("model", models)
293
+@pytest.mark.parametrize(
294
+ "size_factors",
295
+ [
296
+ # No image
297
+ [],
298
+ # Single-scale
299
+ [1.0],
300
+ # Single-scale, batched
301
+ [1.0, 1.0, 1.0],
302
+ # Multi-scale
303
+ [0.5, 0.75, 1.0],
304
+ ],
305
+)
306
+@pytest.mark.parametrize("dtype", [target_dtype])
307
+@pytest.mark.parametrize("max_tokens", [128])
308
+@pytest.mark.parametrize("num_logprobs", [5])
309
+@torch.inference_mode()
310
+def test_multi_images_models(hf_runner, vllm_runner, image_assets, model,
311
+ size_factors, dtype: str, max_tokens: int,
312
+ num_logprobs: int) -> None:
313
+ images = [asset.pil_image for asset in image_assets]
314
+
315
+ inputs_per_case = [
316
+ ([HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
317
+ [[rescale_image_size(image, factor) for image in images]
318
+ for factor in size_factors])
319
+ ]
320
+
321
+ run_test(
322
+ hf_runner,
323
+ vllm_runner,
324
+ inputs_per_case,
325
model,
272
- size_factors=size_factors,
326
dtype=dtype,
327
max_tokens=max_tokens,
328
num_logprobs=num_logprobs,
329
+ mm_limit=2,
330
tensor_parallel_size=1,
331
)
332
tests/models/test_phi3v.py
+3
-5
@@ -1,16 +1,15 @@
1
import os
2
import re
3
-from typing import List, Optional, Tuple, Type, Union
3
+from typing import List, Optional, Tuple, Type
4
5
import pytest
6
-from PIL import Image
6
from transformers import AutoTokenizer
7
8
from vllm.multimodal.utils import rescale_image_size
9
from vllm.sequence import SampleLogprobs
10
from vllm.utils import is_cpu, is_hip
11
13
-from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner
12
+from ..conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
13
from .utils import check_logprobs_close
14
15
pytestmark = pytest.mark.vlm
@@ -60,8 +59,7 @@ if is_hip():
59
def run_test(
60
hf_runner: Type[HfRunner],
61
vllm_runner: Type[VllmRunner],
63
- inputs: List[Tuple[List[str], Union[List[Image.Image],
64
- List[List[Image.Image]]]]],
62
+ inputs: List[Tuple[List[str], PromptImageInput]],
63
model: str,
64
*,
65
dtype: str,
vllm/model_executor/models/internvl.py
+46
-14
@@ -5,6 +5,7 @@
5
# Licensed under The MIT License [see LICENSE for details]
6
# --------------------------------------------------------
7
import itertools
8
+import re
9
from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
10
TypedDict, Union)
11
@@ -26,6 +27,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
27
from vllm.multimodal.base import MultiModalInputs
28
from vllm.multimodal.utils import cached_get_tokenizer
29
from vllm.sequence import IntermediateTensors
30
+from vllm.utils import is_list_of
31
32
from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
33
get_clip_num_patches)
@@ -95,8 +97,8 @@ def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height,
97
98
99
def calculate_num_blocks(orig_width: int, orig_height: int, min_num: int,
98
- max_num: int,
99
- image_size: int) -> Tuple[int, int, int]:
100
+ max_num: int, image_size: int,
101
+ use_thumbnail: bool) -> Tuple[int, int, int]:
102
aspect_ratio = orig_width / orig_height
103
104
# calculate the existing image aspect ratio
@@ -114,17 +116,26 @@ def calculate_num_blocks(orig_width: int, orig_height: int, min_num: int,
116
target_width = image_size * target_aspect_ratio[0]
117
target_height = image_size * target_aspect_ratio[1]
118
blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
119
+ # add thumbnail image if num_blocks > 1
120
+ if use_thumbnail and blocks > 1:
121
+ blocks += 1
122
return blocks, target_width, target_height
123
124
125
# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
126
def dynamic_preprocess(image: Image.Image, min_num: int, max_num: int,
127
image_size: int,
123
- use_thumbnail: int) -> List[Image.Image]:
128
+ use_thumbnail: bool) -> List[Image.Image]:
129
orig_width, orig_height = image.size
130
131
+ # calculate the number of blocks without thumbnail
132
blocks, target_width, target_height = calculate_num_blocks(
127
- orig_width, orig_height, min_num, max_num, image_size)
133
+ orig_width,
134
+ orig_height,
135
+ min_num,
136
+ max_num,
137
+ image_size,
138
+ use_thumbnail=False)
139
# resize the image
140
resized_img = image.resize((target_width, target_height))
141
processed_images = []
@@ -197,17 +208,23 @@ def input_processor_for_internvl(ctx: InputContext, llm_inputs: LLMInputs):
208
downsample_ratio)
209
210
image_data = multi_modal_data["image"]
211
+ min_num = hf_config.min_dynamic_patch
212
+ max_num = hf_config.max_dynamic_patch
213
+ use_thumbnail = hf_config.use_thumbnail
214
if isinstance(image_data, Image.Image):
215
width, height = image_data.size
202
- min_num = hf_config.min_dynamic_patch
203
- max_num = hf_config.max_dynamic_patch
216
num_blocks, _, _ = calculate_num_blocks(width, height, min_num,
205
- max_num, image_size)
206
- # add thumbnail image if num_blocks > 1
207
- if hf_config.use_thumbnail and num_blocks > 1:
208
- num_blocks += 1
209
- image_feature_size = num_blocks * num_patches
210
-
217
+ max_num, image_size,
218
+ use_thumbnail)
219
+ image_feature_size = [num_blocks * num_patches]
220
+ elif is_list_of(image_data, Image.Image):
221
+ image_feature_size = []
222
+ for image in image_data:
223
+ width, height = image.size
224
+ num_blocks, _, _ = calculate_num_blocks(width, height, min_num,
225
+ max_num, image_size,
226
+ use_thumbnail)
227
+ image_feature_size.append(num_blocks * num_patches)
228
elif isinstance(image_data, torch.Tensor):
229
num_images, image_feature_size, hidden_size = image_data.shape
230
else:
@@ -220,8 +237,14 @@ def input_processor_for_internvl(ctx: InputContext, llm_inputs: LLMInputs):
237
prompt_token_ids = llm_inputs["prompt_token_ids"]
238
if prompt is None:
239
prompt = tokenizer.decode(prompt_token_ids)
223
- image_prompt = IMG_START + IMG_CONTEXT * image_feature_size + IMG_END
224
- new_prompt = prompt.replace('<image>', image_prompt, 1)
240
+
241
+ new_prompt = prompt
242
+ image_idx = sorted(map(int, re.findall(r"Image-(\d+): <image>\n", prompt)))
243
+ for idx, feature_size in enumerate(image_feature_size, start=1):
244
+ image_prompt = IMG_START + IMG_CONTEXT * feature_size + IMG_END
245
+ if not image_idx:
246
+ image_prompt = f"Image-{idx}: {image_prompt}"
247
+ new_prompt = new_prompt.replace('<image>', image_prompt, 1)
248
new_prompt_token_ids = tokenizer.encode(new_prompt)
249
250
return LLMInputs(prompt=prompt,
@@ -245,6 +268,15 @@ def input_mapper_for_internvl(ctx: InputContext, data: object):
268
use_thumbnail=use_thumbnail)
269
# Add an N dimension for number of images per prompt (currently 1).
270
data = data.unsqueeze(0)
271
+ elif is_list_of(data, Image.Image):
272
+ data = [
273
+ image_to_pixel_values(img,
274
+ image_size,
275
+ min_num,
276
+ max_num,
277
+ use_thumbnail=use_thumbnail) for img in data
278
+ ]
279
+ data = torch.stack(data)
280
model_config = ctx.model_config
281
tokenizer = cached_get_tokenizer(model_config.tokenizer,
282
trust_remote_code=True)