@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e807125936
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Model][VLM] Support multi-images inputs for InternVL2 models (#8201)

Isotr0py committed Sep 7, 2024 at 16:38 UTC e807125936a9db796746b67ba72c222b5c26582e
5 files changed +199 -57
docs/source/models/supported_models.rst
+1 -1
@@ -214,7 +214,7 @@ Multimodal Language Models
214 -
215 * - :code:`InternVLChatModel`
216 - InternVL2
217 - - Image\ :sup:`E`
217 + - Image\ :sup:`E+`
218 - :code:`OpenGVLab/InternVL2-4B`, :code:`OpenGVLab/InternVL2-8B`, etc.
219 -
220 * - :code:`LlavaForConditionalGeneration`
examples/offline_inference_vision_language_multi_image.py
+76 -18
@@ -6,7 +6,9 @@ by the model.
6 from argparse import Namespace
7 from typing import List
8
9 -from vllm import LLM
9 +from transformers import AutoTokenizer
10 +
11 +from vllm import LLM, SamplingParams
12 from vllm.multimodal.utils import fetch_image
13 from vllm.utils import FlexibleArgumentParser
14
@@ -17,36 +19,84 @@ IMAGE_URLS = [
19 ]
20
21
20 -def _load_phi3v(image_urls: List[str]):
21 - return LLM(
22 +def load_phi3v(question, image_urls: List[str]):
23 + llm = LLM(
24 model="microsoft/Phi-3.5-vision-instruct",
25 trust_remote_code=True,
26 max_model_len=4096,
27 limit_mm_per_prompt={"image": len(image_urls)},
28 )
27 -
28 -
29 -def run_phi3v_generate(question: str, image_urls: List[str]):
30 - llm = _load_phi3v(image_urls)
31 -
29 placeholders = "\n".join(f"<|image_{i}|>"
30 for i, _ in enumerate(image_urls, start=1))
31 prompt = f"<|user|>\n{placeholders}\n{question}<|end|>\n<|assistant|>\n"
32 + stop_token_ids = None
33 + return llm, prompt, stop_token_ids
34
36 - outputs = llm.generate({
37 - "prompt": prompt,
38 - "multi_modal_data": {
39 - "image": [fetch_image(url) for url in image_urls]
35 +
36 +def load_internvl(question, image_urls: List[str]):
37 + model_name = "OpenGVLab/InternVL2-2B"
38 +
39 + llm = LLM(
40 + model=model_name,
41 + trust_remote_code=True,
42 + max_num_seqs=5,
43 + max_model_len=4096,
44 + limit_mm_per_prompt={"image": len(image_urls)},
45 + )
46 +
47 + placeholders = "\n".join(f"Image-{i}: <image>\n"
48 + for i, _ in enumerate(image_urls, start=1))
49 + messages = [{'role': 'user', 'content': f"{placeholders}\n{question}"}]
50 +
51 + tokenizer = AutoTokenizer.from_pretrained(model_name,
52 + trust_remote_code=True)
53 + prompt = tokenizer.apply_chat_template(messages,
54 + tokenize=False,
55 + add_generation_prompt=True)
56 +
57 + # Stop tokens for InternVL
58 + # models variants may have different stop tokens
59 + # please refer to the model card for the correct "stop words":
60 + # https://huggingface.co/OpenGVLab/InternVL2-2B#service
61 + stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"]
62 + stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
63 + return llm, prompt, stop_token_ids
64 +
65 +
66 +model_example_map = {
67 + "phi3_v": load_phi3v,
68 + "internvl_chat": load_internvl,
69 +}
70 +
71 +
72 +def run_generate(model, question: str, image_urls: List[str]):
73 + llm, prompt, stop_token_ids = model_example_map[model](question,
74 + image_urls)
75 +
76 + sampling_params = SamplingParams(temperature=0.0,
77 + max_tokens=128,
78 + stop_token_ids=stop_token_ids)
79 +
80 + outputs = llm.generate(
81 + {
82 + "prompt": prompt,
83 + "multi_modal_data": {
84 + "image": [fetch_image(url) for url in image_urls]
85 + },
86 },
41 - })
87 + sampling_params=sampling_params)
88
89 for o in outputs:
90 generated_text = o.outputs[0].text
91 print(generated_text)
92
93
48 -def run_phi3v_chat(question: str, image_urls: List[str]):
49 - llm = _load_phi3v(image_urls)
94 +def run_chat(model: str, question: str, image_urls: List[str]):
95 + llm, _, stop_token_ids = model_example_map[model](question, image_urls)
96 +
97 + sampling_params = SamplingParams(temperature=0.0,
98 + max_tokens=128,
99 + stop_token_ids=stop_token_ids)
100
101 outputs = llm.chat([{
102 "role":
@@ -63,7 +113,8 @@ def run_phi3v_chat(question: str, image_urls: List[str]):
113 },
114 } for image_url in image_urls),
115 ],
66 - }])
116 + }],
117 + sampling_params=sampling_params)
118
119 for o in outputs:
120 generated_text = o.outputs[0].text
@@ -71,12 +122,13 @@ def run_phi3v_chat(question: str, image_urls: List[str]):
122
123
124 def main(args: Namespace):
125 + model = args.model_type
126 method = args.method
127
128 if method == "generate":
77 - run_phi3v_generate(QUESTION, IMAGE_URLS)
129 + run_generate(model, QUESTION, IMAGE_URLS)
130 elif method == "chat":
79 - run_phi3v_chat(QUESTION, IMAGE_URLS)
131 + run_chat(model, QUESTION, IMAGE_URLS)
132 else:
133 raise ValueError(f"Invalid method: {method}")
134
@@ -85,6 +137,12 @@ if __name__ == "__main__":
137 parser = FlexibleArgumentParser(
138 description='Demo on using vLLM for offline inference with '
139 'vision language models that support multi-image input')
140 + parser.add_argument('--model-type',
141 + '-m',
142 + type=str,
143 + default="phi3_v",
144 + choices=model_example_map.keys(),
145 + help='Huggingface "model_type".')
146 parser.add_argument("--method",
147 type=str,
148 default="generate",
tests/models/test_internvl.py
+73 -19
@@ -1,5 +1,5 @@
1 import types
2 -from typing import List, Optional, Tuple, Type
2 +from typing import List, Optional, Tuple, Type, Union
3
4 import pytest
5 import torch
@@ -9,7 +9,8 @@ from transformers import AutoConfig
9 from vllm.multimodal.utils import rescale_image_size
10 from vllm.utils import is_cpu
11
12 -from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
12 +from ..conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
13 + _ImageAssets)
14 from .utils import check_logprobs_close
15
16 pytestmark = pytest.mark.vlm
@@ -20,6 +21,7 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
21 "cherry_blossom":
22 "<|im_start|>User\n<image>\nWhat is the season?<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
23 })
24 +HF_MULTIIMAGE_IMAGE_PROMPT = "<|im_start|>User\nImage-1: <image>\nImage-2: <image>\nDescribe the two images in detail.<|im_end|>\n<|im_start|>Assistant\n" # noqa: E501
25
26 models = [
27 "OpenGVLab/InternVL2-1B",
@@ -64,13 +66,13 @@ def generate(
66 def run_test(
67 hf_runner: Type[HfRunner],
68 vllm_runner: Type[VllmRunner],
67 - image_assets: _ImageAssets,
69 + inputs: List[Tuple[List[str], PromptImageInput]],
70 model: str,
71 *,
70 - size_factors: List[float],
72 dtype: str,
73 max_tokens: int,
74 num_logprobs: int,
75 + mm_limit: int,
76 tensor_parallel_size: int,
77 distributed_executor_backend: Optional[str] = None,
78 ):
@@ -83,12 +85,6 @@ def run_test(
85 Note, the text input is also adjusted to abide by vllm contract.
86 The text output is sanitized to be able to compare with hf.
87 """
86 - images = [asset.pil_image for asset in image_assets]
87 -
88 - inputs_per_image = [(
89 - [prompt for _ in size_factors],
90 - [rescale_image_size(image, factor) for factor in size_factors],
91 - ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
88
89 # NOTE: take care of the order. run vLLM first, and then run HF.
90 # vLLM needs a fresh new process without cuda initialization.
@@ -110,13 +106,21 @@ def run_test(
106 self.max_num = self.config.max_dynamic_patch
107 self.image_size = self.vision_config.image_size
108
113 - def __call__(self, text: str, images: Image, **kwargs):
109 + def __call__(self, text: str, images: Union[Image, List[Image]],
110 + **kwargs):
111 from vllm.model_executor.models.internvl import (
112 IMG_CONTEXT, IMG_END, IMG_START, image_to_pixel_values)
116 - pixel_values = image_to_pixel_values(
117 - images, self.image_size, self.min_num, self.max_num,
118 - self.use_thumbnail).to(self.dtype)
119 - num_patches_list = [pixel_values.shape[0]]
113 + images = [images] if isinstance(images, Image) else images
114 + pixel_values = [
115 + image_to_pixel_values(image, self.image_size, self.min_num,
116 + self.max_num,
117 + self.use_thumbnail).to(self.dtype)
118 + for image in images
119 + ]
120 + num_patches_list = [
121 + pixel_value.shape[0] for pixel_value in pixel_values
122 + ]
123 + pixel_values = torch.cat(pixel_values, dim=0)
124 for num_patches in num_patches_list:
125 context_tokens = IMG_CONTEXT * self.num_image_token \
126 * num_patches
@@ -130,6 +134,7 @@ def run_test(
134 with vllm_runner(model,
135 max_model_len=4096,
136 dtype=dtype,
137 + limit_mm_per_prompt={"image": mm_limit},
138 tensor_parallel_size=tensor_parallel_size,
139 distributed_executor_backend=distributed_executor_backend,
140 enforce_eager=True) as vllm_model:
@@ -138,7 +143,7 @@ def run_test(
143 max_tokens,
144 num_logprobs=num_logprobs,
145 images=images)
141 - for prompts, images in inputs_per_image
146 + for prompts, images in inputs
147 ]
148
149 with hf_runner(model, dtype=dtype) as hf_model:
@@ -156,7 +161,7 @@ def run_test(
161 num_logprobs=num_logprobs,
162 images=hf_images,
163 eos_token_id=eos_token_id)
159 - for prompts, hf_images in inputs_per_image
164 + for prompts, hf_images in inputs
165 ]
166
167 for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
@@ -264,15 +269,64 @@ if is_cpu():
269 @torch.inference_mode()
270 def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
271 dtype: str, max_tokens: int, num_logprobs: int) -> None:
272 + images = [asset.pil_image for asset in image_assets]
273 +
274 + inputs_per_image = [(
275 + [prompt for _ in size_factors],
276 + [rescale_image_size(image, factor) for factor in size_factors],
277 + ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
278 +
279 run_test(
280 hf_runner,
281 vllm_runner,
270 - image_assets,
282 + inputs_per_image,
283 + model,
284 + dtype=dtype,
285 + max_tokens=max_tokens,
286 + num_logprobs=num_logprobs,
287 + mm_limit=1,
288 + tensor_parallel_size=1,
289 + )
290 +
291 +
292 +@pytest.mark.parametrize("model", models)
293 +@pytest.mark.parametrize(
294 + "size_factors",
295 + [
296 + # No image
297 + [],
298 + # Single-scale
299 + [1.0],
300 + # Single-scale, batched
301 + [1.0, 1.0, 1.0],
302 + # Multi-scale
303 + [0.5, 0.75, 1.0],
304 + ],
305 +)
306 +@pytest.mark.parametrize("dtype", [target_dtype])
307 +@pytest.mark.parametrize("max_tokens", [128])
308 +@pytest.mark.parametrize("num_logprobs", [5])
309 +@torch.inference_mode()
310 +def test_multi_images_models(hf_runner, vllm_runner, image_assets, model,
311 + size_factors, dtype: str, max_tokens: int,
312 + num_logprobs: int) -> None:
313 + images = [asset.pil_image for asset in image_assets]
314 +
315 + inputs_per_case = [
316 + ([HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
317 + [[rescale_image_size(image, factor) for image in images]
318 + for factor in size_factors])
319 + ]
320 +
321 + run_test(
322 + hf_runner,
323 + vllm_runner,
324 + inputs_per_case,
325 model,
272 - size_factors=size_factors,
326 dtype=dtype,
327 max_tokens=max_tokens,
328 num_logprobs=num_logprobs,
329 + mm_limit=2,
330 tensor_parallel_size=1,
331 )
332
tests/models/test_phi3v.py
+3 -5
@@ -1,16 +1,15 @@
1 import os
2 import re
3 -from typing import List, Optional, Tuple, Type, Union
3 +from typing import List, Optional, Tuple, Type
4
5 import pytest
6 -from PIL import Image
6 from transformers import AutoTokenizer
7
8 from vllm.multimodal.utils import rescale_image_size
9 from vllm.sequence import SampleLogprobs
10 from vllm.utils import is_cpu, is_hip
11
13 -from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner
12 +from ..conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
13 from .utils import check_logprobs_close
14
15 pytestmark = pytest.mark.vlm
@@ -60,8 +59,7 @@ if is_hip():
59 def run_test(
60 hf_runner: Type[HfRunner],
61 vllm_runner: Type[VllmRunner],
63 - inputs: List[Tuple[List[str], Union[List[Image.Image],
64 - List[List[Image.Image]]]]],
62 + inputs: List[Tuple[List[str], PromptImageInput]],
63 model: str,
64 *,
65 dtype: str,
vllm/model_executor/models/internvl.py
+46 -14
@@ -5,6 +5,7 @@
5 # Licensed under The MIT License [see LICENSE for details]
6 # --------------------------------------------------------
7 import itertools
8 +import re
9 from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
10 TypedDict, Union)
11
@@ -26,6 +27,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
27 from vllm.multimodal.base import MultiModalInputs
28 from vllm.multimodal.utils import cached_get_tokenizer
29 from vllm.sequence import IntermediateTensors
30 +from vllm.utils import is_list_of
31
32 from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
33 get_clip_num_patches)
@@ -95,8 +97,8 @@ def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height,
97
98
99 def calculate_num_blocks(orig_width: int, orig_height: int, min_num: int,
98 - max_num: int,
99 - image_size: int) -> Tuple[int, int, int]:
100 + max_num: int, image_size: int,
101 + use_thumbnail: bool) -> Tuple[int, int, int]:
102 aspect_ratio = orig_width / orig_height
103
104 # calculate the existing image aspect ratio
@@ -114,17 +116,26 @@ def calculate_num_blocks(orig_width: int, orig_height: int, min_num: int,
116 target_width = image_size * target_aspect_ratio[0]
117 target_height = image_size * target_aspect_ratio[1]
118 blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
119 + # add thumbnail image if num_blocks > 1
120 + if use_thumbnail and blocks > 1:
121 + blocks += 1
122 return blocks, target_width, target_height
123
124
125 # adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
126 def dynamic_preprocess(image: Image.Image, min_num: int, max_num: int,
127 image_size: int,
123 - use_thumbnail: int) -> List[Image.Image]:
128 + use_thumbnail: bool) -> List[Image.Image]:
129 orig_width, orig_height = image.size
130
131 + # calculate the number of blocks without thumbnail
132 blocks, target_width, target_height = calculate_num_blocks(
127 - orig_width, orig_height, min_num, max_num, image_size)
133 + orig_width,
134 + orig_height,
135 + min_num,
136 + max_num,
137 + image_size,
138 + use_thumbnail=False)
139 # resize the image
140 resized_img = image.resize((target_width, target_height))
141 processed_images = []
@@ -197,17 +208,23 @@ def input_processor_for_internvl(ctx: InputContext, llm_inputs: LLMInputs):
208 downsample_ratio)
209
210 image_data = multi_modal_data["image"]
211 + min_num = hf_config.min_dynamic_patch
212 + max_num = hf_config.max_dynamic_patch
213 + use_thumbnail = hf_config.use_thumbnail
214 if isinstance(image_data, Image.Image):
215 width, height = image_data.size
202 - min_num = hf_config.min_dynamic_patch
203 - max_num = hf_config.max_dynamic_patch
216 num_blocks, _, _ = calculate_num_blocks(width, height, min_num,
205 - max_num, image_size)
206 - # add thumbnail image if num_blocks > 1
207 - if hf_config.use_thumbnail and num_blocks > 1:
208 - num_blocks += 1
209 - image_feature_size = num_blocks * num_patches
210 -
217 + max_num, image_size,
218 + use_thumbnail)
219 + image_feature_size = [num_blocks * num_patches]
220 + elif is_list_of(image_data, Image.Image):
221 + image_feature_size = []
222 + for image in image_data:
223 + width, height = image.size
224 + num_blocks, _, _ = calculate_num_blocks(width, height, min_num,
225 + max_num, image_size,
226 + use_thumbnail)
227 + image_feature_size.append(num_blocks * num_patches)
228 elif isinstance(image_data, torch.Tensor):
229 num_images, image_feature_size, hidden_size = image_data.shape
230 else:
@@ -220,8 +237,14 @@ def input_processor_for_internvl(ctx: InputContext, llm_inputs: LLMInputs):
237 prompt_token_ids = llm_inputs["prompt_token_ids"]
238 if prompt is None:
239 prompt = tokenizer.decode(prompt_token_ids)
223 - image_prompt = IMG_START + IMG_CONTEXT * image_feature_size + IMG_END
224 - new_prompt = prompt.replace('<image>', image_prompt, 1)
240 +
241 + new_prompt = prompt
242 + image_idx = sorted(map(int, re.findall(r"Image-(\d+): <image>\n", prompt)))
243 + for idx, feature_size in enumerate(image_feature_size, start=1):
244 + image_prompt = IMG_START + IMG_CONTEXT * feature_size + IMG_END
245 + if not image_idx:
246 + image_prompt = f"Image-{idx}: {image_prompt}"
247 + new_prompt = new_prompt.replace('<image>', image_prompt, 1)
248 new_prompt_token_ids = tokenizer.encode(new_prompt)
249
250 return LLMInputs(prompt=prompt,
@@ -245,6 +268,15 @@ def input_mapper_for_internvl(ctx: InputContext, data: object):
268 use_thumbnail=use_thumbnail)
269 # Add an N dimension for number of images per prompt (currently 1).
270 data = data.unsqueeze(0)
271 + elif is_list_of(data, Image.Image):
272 + data = [
273 + image_to_pixel_values(img,
274 + image_size,
275 + min_num,
276 + max_num,
277 + use_thumbnail=use_thumbnail) for img in data
278 + ]
279 + data = torch.stack(data)
280 model_config = ctx.model_config
281 tokenizer = cached_get_tokenizer(model_config.tokenizer,
282 trust_remote_code=True)