@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2ae25f79cf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Model] Expose InternVL2 max_dynamic_patch as a mm_processor_kwarg (#8946)

Isotr0py committed Sep 30, 2024 at 13:01 UTC 2ae25f79cf1e8d21f7bcba097e4c039463c22be4
2 files changed +90 -61
examples/offline_inference_vision_language_multi_image.py
+1
@@ -115,6 +115,7 @@ def load_internvl(question: str, image_urls: List[str]) -> ModelRequestData:
115 trust_remote_code=True,
116 max_model_len=4096,
117 limit_mm_per_prompt={"image": len(image_urls)},
118 + mm_processor_kwargs={"max_dynamic_patch": 4},
119 )
120
121 placeholders = "\n".join(f"Image-{i}: <image>\n"
vllm/model_executor/models/internvl.py
+89 -61
@@ -5,8 +5,9 @@
5 # Licensed under The MIT License [see LICENSE for details]
6 # --------------------------------------------------------
7 import re
8 -from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
9 - TypedDict, Union)
8 +from functools import partial
9 +from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional,
10 + Tuple, TypedDict, Union)
11
12 import torch
13 import torch.nn as nn
@@ -122,6 +123,20 @@ def calculate_num_blocks(orig_width: int, orig_height: int, min_num: int,
123 return blocks, target_width, target_height
124
125
126 +def calculate_num_blocks_wrapper(hf_config: Dict[str, Any],
127 + max_dynamic_patch: Optional[int] = None):
128 + if max_dynamic_patch is None:
129 + max_dynamic_patch = hf_config.max_dynamic_patch
130 + min_num = hf_config.min_dynamic_patch
131 + image_size = hf_config.vision_config.image_size
132 + use_thumbnail = hf_config.use_thumbnail
133 + return partial(calculate_num_blocks,
134 + min_num=min_num,
135 + max_num=max_dynamic_patch,
136 + image_size=image_size,
137 + use_thumbnail=use_thumbnail)
138 +
139 +
140 # adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
141 def dynamic_preprocess(image: Image.Image, min_num: int, max_num: int,
142 image_size: int,
@@ -168,62 +183,85 @@ def image_to_pixel_values(image: Image.Image, input_size: int, min_num: int,
183 return pixel_values
184
185
171 -def get_internvl_num_patches(image_size: int, patch_size: int,
172 - downsample_ratio: float):
186 +def image_to_pixel_values_wrapper(hf_config: Dict[str, Any],
187 + max_dynamic_patch: Optional[int] = None):
188 + image_size = hf_config.vision_config.image_size
189 + min_num = hf_config.min_dynamic_patch
190 + if max_dynamic_patch is None:
191 + max_dynamic_patch = hf_config.max_dynamic_patch
192 + use_thumbnail = hf_config.use_thumbnail
193 + return partial(image_to_pixel_values,
194 + input_size=image_size,
195 + min_num=min_num,
196 + max_num=max_dynamic_patch,
197 + use_thumbnail=use_thumbnail)
198 +
199 +
200 +def get_internvl_num_patches(hf_config: Dict[str, Any]):
201 + vision_config = hf_config.vision_config
202 + downsample_ratio = hf_config.downsample_ratio
203 + image_size = vision_config.image_size
204 + patch_size = vision_config.patch_size
205 return int(
206 get_clip_num_patches(image_size=image_size, patch_size=patch_size) *
207 (downsample_ratio**2))
208
209
178 -def get_max_internvl_image_tokens(ctx: InputContext):
210 +def get_max_internvl_image_tokens(ctx: InputContext,
211 + *,
212 + max_dynamic_patch: Optional[int] = None):
213 hf_config = ctx.get_hf_config()
180 - vision_config = hf_config.vision_config
214
215 + if max_dynamic_patch is None:
216 + max_dynamic_patch = hf_config.max_dynamic_patch
217 use_thumbnail = hf_config.use_thumbnail
183 - max_dynamic_patch = hf_config.max_dynamic_patch
184 - if use_thumbnail:
218 + if use_thumbnail and max_dynamic_patch > 1:
219 max_dynamic_patch += 1
186 - downsample_ratio = hf_config.downsample_ratio
220
188 - image_size = vision_config.image_size
189 - patch_size = vision_config.patch_size
190 - num_patches = get_internvl_num_patches(image_size, patch_size,
191 - downsample_ratio)
221 + num_patches = get_internvl_num_patches(hf_config)
222 return num_patches * max_dynamic_patch
223
224
195 -def input_processor_for_internvl(ctx: InputContext, llm_inputs: LLMInputs):
225 +def get_max_internvl_image_size(ctx: InputContext,
226 + *,
227 + max_dynamic_patch: Optional[int] = None):
228 + hf_config = ctx.get_hf_config()
229 + image_size = hf_config.vision_config.image_size
230 +
231 + if max_dynamic_patch is None:
232 + max_dynamic_patch = hf_config.max_dynamic_patch
233 + use_thumbnail = hf_config.use_thumbnail
234 + if use_thumbnail and max_dynamic_patch > 1:
235 + max_dynamic_patch += 1
236 + width = image_size * max_dynamic_patch
237 + height = image_size
238 + return width, height
239 +
240 +
241 +def input_processor_for_internvl(ctx: InputContext,
242 + llm_inputs: LLMInputs,
243 + *,
244 + max_dynamic_patch: Optional[int] = None):
245 multi_modal_data = llm_inputs.get("multi_modal_data")
246 if multi_modal_data is None or "image" not in multi_modal_data:
247 return llm_inputs
248
249 model_config = ctx.model_config
250 hf_config = ctx.get_hf_config()
202 - vision_config = hf_config.vision_config
203 -
204 - image_size = vision_config.image_size
205 - patch_size = vision_config.patch_size
206 - downsample_ratio = hf_config.downsample_ratio
207 - num_patches = get_internvl_num_patches(image_size, patch_size,
208 - downsample_ratio)
251
252 image_data = multi_modal_data["image"]
211 - min_num = hf_config.min_dynamic_patch
212 - max_num = hf_config.max_dynamic_patch
213 - use_thumbnail = hf_config.use_thumbnail
253 + num_patches = get_internvl_num_patches(hf_config)
254 + num_blocks_calculator = calculate_num_blocks_wrapper(
255 + hf_config, max_dynamic_patch)
256 if isinstance(image_data, Image.Image):
257 width, height = image_data.size
216 - num_blocks, _, _ = calculate_num_blocks(width, height, min_num,
217 - max_num, image_size,
218 - use_thumbnail)
258 + num_blocks, _, _ = num_blocks_calculator(width, height)
259 image_feature_size = [num_blocks * num_patches]
260 elif is_list_of(image_data, Image.Image):
261 image_feature_size = []
262 for image in image_data:
263 width, height = image.size
224 - num_blocks, _, _ = calculate_num_blocks(width, height, min_num,
225 - max_num, image_size,
226 - use_thumbnail)
264 + num_blocks, _, _ = num_blocks_calculator(width, height)
265 image_feature_size.append(num_blocks * num_patches)
266 elif isinstance(image_data, torch.Tensor):
267 num_images, image_feature_size, hidden_size = image_data.shape
@@ -253,31 +291,21 @@ def input_processor_for_internvl(ctx: InputContext, llm_inputs: LLMInputs):
291 multi_modal_data=multi_modal_data)
292
293
256 -def input_mapper_for_internvl(ctx: InputContext, data: object):
294 +def input_mapper_for_internvl(ctx: InputContext,
295 + data: object,
296 + *,
297 + max_dynamic_patch: Optional[int] = None):
298 hf_config = ctx.get_hf_config()
299
259 - use_thumbnail = hf_config.use_thumbnail
260 - min_num = hf_config.min_dynamic_patch
261 - max_num = hf_config.max_dynamic_patch
262 - image_size = hf_config.vision_config.image_size
263 -
300 + image_pixel_values_mapper = image_to_pixel_values_wrapper(
301 + hf_config, max_dynamic_patch)
302 if isinstance(data, Image.Image):
265 - data = image_to_pixel_values(data,
266 - image_size,
267 - min_num,
268 - max_num,
269 - use_thumbnail=use_thumbnail)
303 + data = image_pixel_values_mapper(data)
304 # Add an N dimension for number of images per prompt (currently 1).
305 data = data.unsqueeze(0)
306 elif is_list_of(data, Image.Image):
307 # we can't stack here because the images may have different num_patches
274 - data = [
275 - image_to_pixel_values(img,
276 - image_size,
277 - min_num,
278 - max_num,
279 - use_thumbnail=use_thumbnail) for img in data
280 - ]
308 + data = [image_pixel_values_mapper(img) for img in data]
309 model_config = ctx.model_config
310 tokenizer = cached_get_tokenizer(
311 model_config.tokenizer,
@@ -292,20 +320,24 @@ def input_mapper_for_internvl(ctx: InputContext, data: object):
320 })
321
322
295 -def dummy_data_for_internvl(ctx: InputContext, seq_len: int,
296 - mm_counts: Mapping[str, int]):
323 +def dummy_data_for_internvl(ctx: InputContext,
324 + seq_len: int,
325 + mm_counts: Mapping[str, int],
326 + *,
327 + max_dynamic_patch: Optional[int] = None):
328 num_images = mm_counts["image"]
329
299 - image_feature_size = get_max_internvl_image_tokens(ctx)
300 - model_config = ctx.model_config
330 hf_config = ctx.get_hf_config()
302 - vision_config = hf_config.vision_config
331 +
332 + image_feature_size = get_max_internvl_image_tokens(
333 + ctx, max_dynamic_patch=max_dynamic_patch)
334 + model_config = ctx.model_config
335 tokenizer = cached_get_tokenizer(
336 model_config.tokenizer,
337 trust_remote_code=model_config.trust_remote_code)
338
339 seq_data = dummy_seq_data_for_clip(
308 - vision_config,
340 + hf_config.vision_config,
341 seq_len,
342 num_images,
343 image_token_id=tokenizer.encode(IMG_CONTEXT,
@@ -313,14 +345,11 @@ def dummy_data_for_internvl(ctx: InputContext, seq_len: int,
345 image_feature_size_override=image_feature_size,
346 )
347
316 - image_size = vision_config.image_size
317 - min_num = hf_config.min_dynamic_patch
318 - max_num = hf_config.max_dynamic_patch
319 - max_image_width = max_num * image_size
320 - max_image_height = min_num * image_size
348 + max_image_width, max_image_height = get_max_internvl_image_size(
349 + ctx, max_dynamic_patch=max_dynamic_patch)
350
351 mm_data = dummy_image_for_clip(
323 - vision_config,
352 + hf_config.vision_config,
353 num_images,
354 image_width_override=max_image_width,
355 image_height_override=max_image_height,
@@ -470,7 +499,6 @@ class InternVLChatModel(nn.Module, SupportsMultiModal):
499 self,
500 image_input: InternVLImageInputs,
501 ) -> torch.Tensor:
473 -
502 if image_input["type"] == "image_embeds":
503 return image_input["data"]
504