A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Bugfix] Fix input processor for InternVL2 model (#7164)
Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Isotr0py committed
Aug 8, 2024 at 00:32 UTC
b764547616e6ae1517929985400b1dc62cdbc3a3
2 files changed
+73
-34
tests/models/test_internvl.py
+19
-4
@@ -5,6 +5,7 @@ import pytest
5
import torch
6
from huggingface_hub import snapshot_download
7
from PIL.Image import Image
8
+from transformers import AutoConfig
9
10
from vllm.model_executor.models.internvl import (IMG_CONTEXT, IMG_END,
11
IMG_START,
@@ -26,10 +27,15 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
27
28
# we use snapshot_download to prevent conflicts between
29
# dynamic_module and trust_remote_code for hf_runner
30
+DOWNLOAD_PATTERN = ["*.json", "*.py", "*.safetensors", "*.txt", "*.model"]
31
models = [
30
- snapshot_download("OpenGVLab/InternVL2-1B"),
31
- snapshot_download("OpenGVLab/InternVL2-2B"),
32
- # snapshot_download("OpenGVLab/InternVL2-4B"), # broken
32
+ snapshot_download("OpenGVLab/InternVL2-1B",
33
+ allow_patterns=DOWNLOAD_PATTERN),
34
+ snapshot_download("OpenGVLab/InternVL2-2B",
35
+ allow_patterns=DOWNLOAD_PATTERN),
36
+ # Broken due to outdated implementation of Phi-3
37
+ # See: https://huggingface.co/OpenGVLab/InternVL2-4B/discussions/3
38
+ # snapshot_download("OpenGVLab/InternVL2-4B"),
39
]
40
41
@@ -41,8 +47,17 @@ class InternVLProcessor:
47
self.tokenizer = hf_runner.tokenizer
48
self.dtype = hf_runner.model.dtype
49
50
+ self.config = AutoConfig.from_pretrained(hf_runner.model_name)
51
+ self.vision_config = self.config.vision_config
52
+ self.use_thumbnail = self.config.use_thumbnail
53
+ self.min_num = self.config.min_dynamic_patch
54
+ self.max_num = self.config.max_dynamic_patch
55
+ self.image_size = self.vision_config.image_size
56
+
57
def __call__(self, text: str, images: Image, **kwargs):
45
- pixel_values = image_to_pixel_values(images).to(self.dtype)
58
+ pixel_values = image_to_pixel_values(images, self.image_size,
59
+ self.min_num, self.max_num,
60
+ self.use_thumbnail).to(self.dtype)
61
num_patches_list = [pixel_values.shape[0]]
62
for num_patches in num_patches_list:
63
context_tokens = IMG_CONTEXT * self.num_image_token * num_patches
vllm/model_executor/models/internvl.py
+54
-30
@@ -38,9 +38,6 @@ IMG_CONTEXT = '<IMG_CONTEXT>'
38
IMAGENET_MEAN = (0.485, 0.456, 0.406)
39
IMAGENET_STD = (0.229, 0.224, 0.225)
40
41
-MAX_IMAGE_FEATURE_SIZE_WIDTH = 3000
42
-MAX_IMAGE_FEATURE_SIZE_HEIGHT = 500
43
-
41
42
class InternVLImagePixelInputs(TypedDict):
43
type: Literal["pixel_values"]
@@ -84,11 +81,9 @@ def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height,
81
return best_ratio
82
83
87
-def calculate_num_blocks(orig_width: int,
88
- orig_height: int,
89
- min_num=1,
90
- max_num=6,
91
- image_size=448):
84
+def calculate_num_blocks(orig_width: int, orig_height: int, min_num: int,
85
+ max_num: int,
86
+ image_size: int) -> Tuple[int, int, int]:
87
aspect_ratio = orig_width / orig_height
88
89
# calculate the existing image aspect ratio
@@ -110,11 +105,9 @@ def calculate_num_blocks(orig_width: int,
105
106
107
# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
113
-def dynamic_preprocess(image,
114
- min_num=1,
115
- max_num=6,
116
- image_size=448,
117
- use_thumbnail=False):
108
+def dynamic_preprocess(image: Image.Image, min_num: int, max_num: int,
109
+ image_size: int,
110
+ use_thumbnail: int) -> List[Image.Image]:
111
orig_width, orig_height = image.size
112
113
blocks, target_width, target_height = calculate_num_blocks(
@@ -138,12 +131,14 @@ def dynamic_preprocess(image,
131
132
133
# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
141
-def image_to_pixel_values(image: Image.Image, input_size=448, max_num=6):
134
+def image_to_pixel_values(image: Image.Image, input_size: int, min_num: int,
135
+ max_num: int, use_thumbnail: bool) -> torch.Tensor:
136
transform = build_transform(input_size=input_size)
137
images = dynamic_preprocess(image,
138
+ min_num=min_num,
139
+ max_num=max_num,
140
image_size=input_size,
145
- use_thumbnail=True,
146
- max_num=max_num)
141
+ use_thumbnail=use_thumbnail)
142
pixel_values = [transform(image) for image in images]
143
pixel_values = torch.stack(pixel_values)
144
return pixel_values
@@ -159,12 +154,18 @@ def get_internvl_num_patches(image_size: int, patch_size: int,
154
def get_max_internvl_image_tokens(ctx: InputContext):
155
hf_config = ctx.get_hf_config(PretrainedConfig)
156
vision_config = hf_config.vision_config
157
+
158
+ use_thumbnail = hf_config.use_thumbnail
159
+ max_dynamic_patch = hf_config.max_dynamic_patch
160
+ if use_thumbnail:
161
+ max_dynamic_patch += 1
162
+ downsample_ratio = hf_config.downsample_ratio
163
+
164
image_size = vision_config.image_size
165
patch_size = vision_config.patch_size
164
- downsample_ratio = hf_config.downsample_ratio
166
num_patches = get_internvl_num_patches(image_size, patch_size,
167
downsample_ratio)
167
- return num_patches * 7
168
+ return num_patches * max_dynamic_patch
169
170
171
def input_processor_for_internvl(ctx: InputContext, llm_inputs: LLMInputs):
@@ -176,21 +177,27 @@ def input_processor_for_internvl(ctx: InputContext, llm_inputs: LLMInputs):
177
hf_config = ctx.get_hf_config(PretrainedConfig)
178
vision_config = hf_config.vision_config
179
180
+ image_size = vision_config.image_size
181
+ patch_size = vision_config.patch_size
182
+ downsample_ratio = hf_config.downsample_ratio
183
+ num_patches = get_internvl_num_patches(image_size, patch_size,
184
+ downsample_ratio)
185
+
186
image_data = multi_modal_data["image"]
187
if isinstance(image_data, Image.Image):
188
width, height = image_data.size
182
- num_blocks, _, _ = calculate_num_blocks(width, height)
189
+ min_num = hf_config.min_dynamic_patch
190
+ max_num = hf_config.max_dynamic_patch
191
+ num_blocks, _, _ = calculate_num_blocks(width, height, min_num,
192
+ max_num, image_size)
193
+ # add thumbnail image if num_blocks > 1
194
+ if hf_config.use_thumbnail and num_blocks > 1:
195
+ num_blocks += 1
196
elif isinstance(image_data, torch.Tensor):
197
raise NotImplementedError("Embeddings input is not supported yet")
198
else:
199
raise TypeError(f"Invalid image type: {type(image_data)}")
200
188
- image_size = vision_config.image_size
189
- patch_size = vision_config.patch_size
190
- downsample_ratio = hf_config.downsample_ratio
191
- num_patches = get_internvl_num_patches(image_size, patch_size,
192
- downsample_ratio)
193
-
201
tokenizer = cached_get_tokenizer(model_config.tokenizer,
202
trust_remote_code=True)
203
@@ -198,8 +205,7 @@ def input_processor_for_internvl(ctx: InputContext, llm_inputs: LLMInputs):
205
prompt_token_ids = llm_inputs["prompt_token_ids"]
206
if prompt is None:
207
prompt = tokenizer.decode(prompt_token_ids)
201
- image_prompt = IMG_START + IMG_CONTEXT * (num_blocks +
202
- 1) * num_patches + IMG_END
208
+ image_prompt = IMG_START + IMG_CONTEXT * num_blocks * num_patches + IMG_END
209
new_prompt = prompt.replace('<image>', image_prompt, 1)
210
new_prompt_token_ids = tokenizer.encode(new_prompt)
211
@@ -209,8 +215,19 @@ def input_processor_for_internvl(ctx: InputContext, llm_inputs: LLMInputs):
215
216
217
def input_mapper_for_internvl(ctx: InputContext, data: object):
218
+ hf_config = ctx.get_hf_config(PretrainedConfig)
219
+
220
+ use_thumbnail = hf_config.use_thumbnail
221
+ min_num = hf_config.min_dynamic_patch
222
+ max_num = hf_config.max_dynamic_patch
223
+ image_size = hf_config.vision_config.image_size
224
+
225
if isinstance(data, Image.Image):
213
- data = image_to_pixel_values(data)
226
+ data = image_to_pixel_values(data,
227
+ image_size,
228
+ min_num,
229
+ max_num,
230
+ use_thumbnail=use_thumbnail)
231
model_config = ctx.model_config
232
tokenizer = cached_get_tokenizer(model_config.tokenizer,
233
trust_remote_code=True)
@@ -240,10 +257,17 @@ def dummy_data_for_internvl(ctx: InputContext, seq_len: int):
257
add_special_tokens=False)[0],
258
image_feature_size_override=image_feature_size,
259
)
260
+
261
+ image_size = vision_config.image_size
262
+ min_num = hf_config.min_dynamic_patch
263
+ max_num = hf_config.max_dynamic_patch
264
+ max_image_width = max_num * image_size
265
+ max_image_height = min_num * image_size
266
+
267
mm_data = dummy_image_for_clip(
268
vision_config,
245
- image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH,
246
- image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
269
+ image_width_override=max_image_width,
270
+ image_height_override=max_image_height,
271
)
272
273
return seq_data, mm_data