@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b764547616
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Bugfix] Fix input processor for InternVL2 model (#7164)

Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

Isotr0py committed Aug 8, 2024 at 00:32 UTC b764547616e6ae1517929985400b1dc62cdbc3a3
2 files changed +73 -34
tests/models/test_internvl.py
+19 -4
@@ -5,6 +5,7 @@ import pytest
5 import torch
6 from huggingface_hub import snapshot_download
7 from PIL.Image import Image
8 +from transformers import AutoConfig
9
10 from vllm.model_executor.models.internvl import (IMG_CONTEXT, IMG_END,
11 IMG_START,
@@ -26,10 +27,15 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
27
28 # we use snapshot_download to prevent conflicts between
29 # dynamic_module and trust_remote_code for hf_runner
30 +DOWNLOAD_PATTERN = ["*.json", "*.py", "*.safetensors", "*.txt", "*.model"]
31 models = [
30 - snapshot_download("OpenGVLab/InternVL2-1B"),
31 - snapshot_download("OpenGVLab/InternVL2-2B"),
32 - # snapshot_download("OpenGVLab/InternVL2-4B"), # broken
32 + snapshot_download("OpenGVLab/InternVL2-1B",
33 + allow_patterns=DOWNLOAD_PATTERN),
34 + snapshot_download("OpenGVLab/InternVL2-2B",
35 + allow_patterns=DOWNLOAD_PATTERN),
36 + # Broken due to outdated implementation of Phi-3
37 + # See: https://huggingface.co/OpenGVLab/InternVL2-4B/discussions/3
38 + # snapshot_download("OpenGVLab/InternVL2-4B"),
39 ]
40
41
@@ -41,8 +47,17 @@ class InternVLProcessor:
47 self.tokenizer = hf_runner.tokenizer
48 self.dtype = hf_runner.model.dtype
49
50 + self.config = AutoConfig.from_pretrained(hf_runner.model_name)
51 + self.vision_config = self.config.vision_config
52 + self.use_thumbnail = self.config.use_thumbnail
53 + self.min_num = self.config.min_dynamic_patch
54 + self.max_num = self.config.max_dynamic_patch
55 + self.image_size = self.vision_config.image_size
56 +
57 def __call__(self, text: str, images: Image, **kwargs):
45 - pixel_values = image_to_pixel_values(images).to(self.dtype)
58 + pixel_values = image_to_pixel_values(images, self.image_size,
59 + self.min_num, self.max_num,
60 + self.use_thumbnail).to(self.dtype)
61 num_patches_list = [pixel_values.shape[0]]
62 for num_patches in num_patches_list:
63 context_tokens = IMG_CONTEXT * self.num_image_token * num_patches
vllm/model_executor/models/internvl.py
+54 -30
@@ -38,9 +38,6 @@ IMG_CONTEXT = '<IMG_CONTEXT>'
38 IMAGENET_MEAN = (0.485, 0.456, 0.406)
39 IMAGENET_STD = (0.229, 0.224, 0.225)
40
41 -MAX_IMAGE_FEATURE_SIZE_WIDTH = 3000
42 -MAX_IMAGE_FEATURE_SIZE_HEIGHT = 500
43 -
41
42 class InternVLImagePixelInputs(TypedDict):
43 type: Literal["pixel_values"]
@@ -84,11 +81,9 @@ def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height,
81 return best_ratio
82
83
87 -def calculate_num_blocks(orig_width: int,
88 - orig_height: int,
89 - min_num=1,
90 - max_num=6,
91 - image_size=448):
84 +def calculate_num_blocks(orig_width: int, orig_height: int, min_num: int,
85 + max_num: int,
86 + image_size: int) -> Tuple[int, int, int]:
87 aspect_ratio = orig_width / orig_height
88
89 # calculate the existing image aspect ratio
@@ -110,11 +105,9 @@ def calculate_num_blocks(orig_width: int,
105
106
107 # adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
113 -def dynamic_preprocess(image,
114 - min_num=1,
115 - max_num=6,
116 - image_size=448,
117 - use_thumbnail=False):
108 +def dynamic_preprocess(image: Image.Image, min_num: int, max_num: int,
109 + image_size: int,
110 + use_thumbnail: int) -> List[Image.Image]:
111 orig_width, orig_height = image.size
112
113 blocks, target_width, target_height = calculate_num_blocks(
@@ -138,12 +131,14 @@ def dynamic_preprocess(image,
131
132
133 # adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
141 -def image_to_pixel_values(image: Image.Image, input_size=448, max_num=6):
134 +def image_to_pixel_values(image: Image.Image, input_size: int, min_num: int,
135 + max_num: int, use_thumbnail: bool) -> torch.Tensor:
136 transform = build_transform(input_size=input_size)
137 images = dynamic_preprocess(image,
138 + min_num=min_num,
139 + max_num=max_num,
140 image_size=input_size,
145 - use_thumbnail=True,
146 - max_num=max_num)
141 + use_thumbnail=use_thumbnail)
142 pixel_values = [transform(image) for image in images]
143 pixel_values = torch.stack(pixel_values)
144 return pixel_values
@@ -159,12 +154,18 @@ def get_internvl_num_patches(image_size: int, patch_size: int,
154 def get_max_internvl_image_tokens(ctx: InputContext):
155 hf_config = ctx.get_hf_config(PretrainedConfig)
156 vision_config = hf_config.vision_config
157 +
158 + use_thumbnail = hf_config.use_thumbnail
159 + max_dynamic_patch = hf_config.max_dynamic_patch
160 + if use_thumbnail:
161 + max_dynamic_patch += 1
162 + downsample_ratio = hf_config.downsample_ratio
163 +
164 image_size = vision_config.image_size
165 patch_size = vision_config.patch_size
164 - downsample_ratio = hf_config.downsample_ratio
166 num_patches = get_internvl_num_patches(image_size, patch_size,
167 downsample_ratio)
167 - return num_patches * 7
168 + return num_patches * max_dynamic_patch
169
170
171 def input_processor_for_internvl(ctx: InputContext, llm_inputs: LLMInputs):
@@ -176,21 +177,27 @@ def input_processor_for_internvl(ctx: InputContext, llm_inputs: LLMInputs):
177 hf_config = ctx.get_hf_config(PretrainedConfig)
178 vision_config = hf_config.vision_config
179
180 + image_size = vision_config.image_size
181 + patch_size = vision_config.patch_size
182 + downsample_ratio = hf_config.downsample_ratio
183 + num_patches = get_internvl_num_patches(image_size, patch_size,
184 + downsample_ratio)
185 +
186 image_data = multi_modal_data["image"]
187 if isinstance(image_data, Image.Image):
188 width, height = image_data.size
182 - num_blocks, _, _ = calculate_num_blocks(width, height)
189 + min_num = hf_config.min_dynamic_patch
190 + max_num = hf_config.max_dynamic_patch
191 + num_blocks, _, _ = calculate_num_blocks(width, height, min_num,
192 + max_num, image_size)
193 + # add thumbnail image if num_blocks > 1
194 + if hf_config.use_thumbnail and num_blocks > 1:
195 + num_blocks += 1
196 elif isinstance(image_data, torch.Tensor):
197 raise NotImplementedError("Embeddings input is not supported yet")
198 else:
199 raise TypeError(f"Invalid image type: {type(image_data)}")
200
188 - image_size = vision_config.image_size
189 - patch_size = vision_config.patch_size
190 - downsample_ratio = hf_config.downsample_ratio
191 - num_patches = get_internvl_num_patches(image_size, patch_size,
192 - downsample_ratio)
193 -
201 tokenizer = cached_get_tokenizer(model_config.tokenizer,
202 trust_remote_code=True)
203
@@ -198,8 +205,7 @@ def input_processor_for_internvl(ctx: InputContext, llm_inputs: LLMInputs):
205 prompt_token_ids = llm_inputs["prompt_token_ids"]
206 if prompt is None:
207 prompt = tokenizer.decode(prompt_token_ids)
201 - image_prompt = IMG_START + IMG_CONTEXT * (num_blocks +
202 - 1) * num_patches + IMG_END
208 + image_prompt = IMG_START + IMG_CONTEXT * num_blocks * num_patches + IMG_END
209 new_prompt = prompt.replace('<image>', image_prompt, 1)
210 new_prompt_token_ids = tokenizer.encode(new_prompt)
211
@@ -209,8 +215,19 @@ def input_processor_for_internvl(ctx: InputContext, llm_inputs: LLMInputs):
215
216
217 def input_mapper_for_internvl(ctx: InputContext, data: object):
218 + hf_config = ctx.get_hf_config(PretrainedConfig)
219 +
220 + use_thumbnail = hf_config.use_thumbnail
221 + min_num = hf_config.min_dynamic_patch
222 + max_num = hf_config.max_dynamic_patch
223 + image_size = hf_config.vision_config.image_size
224 +
225 if isinstance(data, Image.Image):
213 - data = image_to_pixel_values(data)
226 + data = image_to_pixel_values(data,
227 + image_size,
228 + min_num,
229 + max_num,
230 + use_thumbnail=use_thumbnail)
231 model_config = ctx.model_config
232 tokenizer = cached_get_tokenizer(model_config.tokenizer,
233 trust_remote_code=True)
@@ -240,10 +257,17 @@ def dummy_data_for_internvl(ctx: InputContext, seq_len: int):
257 add_special_tokens=False)[0],
258 image_feature_size_override=image_feature_size,
259 )
260 +
261 + image_size = vision_config.image_size
262 + min_num = hf_config.min_dynamic_patch
263 + max_num = hf_config.max_dynamic_patch
264 + max_image_width = max_num * image_size
265 + max_image_height = min_num * image_size
266 +
267 mm_data = dummy_image_for_clip(
268 vision_config,
245 - image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH,
246 - image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
269 + image_width_override=max_image_width,
270 + image_height_override=max_image_height,
271 )
272
273 return seq_data, mm_data