@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d5ec121f95
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Model] Expose `dynamic_image_size` as mm_processor_kwargs for InternVL2 models (#10518)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Nov 21, 2024 at 22:20 UTC d5ec121f95f51184acce4e2c27ad8fc01904d3d9
2 files changed +255 -14
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_internvl.py new
+206
@@ -0,0 +1,206 @@
1 +"""Tests for InternVL's multimodal preprocessing kwargs."""
2 +from typing import Callable, Optional
3 +
4 +import pytest
5 +from transformers import AutoTokenizer
6 +
7 +from vllm.inputs import InputContext, token_inputs
8 +from vllm.multimodal import MultiModalRegistry
9 +
10 +from .....conftest import _ImageAssets
11 +from ....utils import build_model_context
12 +
13 +models = ["OpenGVLab/InternVL2-2B"]
14 +
15 +
16 +# Wrap lazy imports to avoid initializing CUDA during test collection
17 +@pytest.fixture()
18 +def input_processor_for_internvl():
19 + from vllm.model_executor.models.internvl import InternVLInputPipeline
20 +
21 + pipeline = InternVLInputPipeline('<img>', '</img>', '<IMG_CONTEXT>')
22 + return pipeline.input_processor
23 +
24 +
25 +@pytest.fixture()
26 +def dummy_data_for_internvl():
27 + from vllm.model_executor.models.internvl import InternVLInputPipeline
28 +
29 + pipeline = InternVLInputPipeline('<img>', '</img>', '<IMG_CONTEXT>')
30 + return pipeline.dummy_data
31 +
32 +
33 +@pytest.fixture()
34 +def get_max_internvl_image_tokens():
35 + from vllm.model_executor.models.internvl import (
36 + get_max_internvl_image_tokens)
37 + return get_max_internvl_image_tokens
38 +
39 +
40 +@pytest.mark.parametrize("model", models)
41 +@pytest.mark.parametrize("max_dynamic_patch", [1, 4])
42 +@pytest.mark.parametrize("dynamic_image_size", [True, False, None])
43 +def test_input_mapper_override(
44 + model: str,
45 + image_assets: _ImageAssets,
46 + max_dynamic_patch: int,
47 + dynamic_image_size: Optional[bool],
48 +):
49 + mm_processor_kwargs = {
50 + "max_dynamic_patch": max_dynamic_patch,
51 + }
52 + if dynamic_image_size is not None:
53 + mm_processor_kwargs["dynamic_image_size"] = dynamic_image_size
54 +
55 + expected_num_patches = max_dynamic_patch + 1 if max_dynamic_patch > 1 else 1
56 + if dynamic_image_size is False:
57 + expected_num_patches = 1
58 +
59 + ctx = build_model_context(
60 + model_name=model,
61 + tokenizer_name=model,
62 + trust_remote_code=True,
63 + mm_processor_kwargs=mm_processor_kwargs,
64 + )
65 +
66 + mm_registry = MultiModalRegistry()
67 + mm_registry.init_mm_limits_per_prompt(ctx.model_config)
68 +
69 + image = image_assets[0].pil_image.resize((448 * 2, 448 * 2))
70 + vllm_result = mm_registry.map_input(
71 + ctx.model_config,
72 + {"image": image},
73 + )
74 + assert vllm_result["pixel_values"].size(1) == expected_num_patches
75 +
76 +
77 +@pytest.mark.parametrize("model", models)
78 +@pytest.mark.parametrize("max_dynamic_patch", [1, 4, None])
79 +@pytest.mark.parametrize("dynamic_image_size", [True, False, None])
80 +def test_max_tokens_override(
81 + get_max_internvl_image_tokens: Callable,
82 + model: str,
83 + max_dynamic_patch: Optional[int],
84 + dynamic_image_size: Optional[bool],
85 +):
86 + """Ensure get_max_internvl_image_tokens handles mm_processor_kwargs."""
87 + ctx = build_model_context(
88 + model_name=model,
89 + tokenizer_name=model,
90 + trust_remote_code=True,
91 + mm_processor_kwargs=None,
92 + )
93 +
94 + if max_dynamic_patch is None:
95 + max_dynamic_patch = ctx.get_hf_config().max_dynamic_patch
96 + expected_num_patches = max_dynamic_patch + 1 if max_dynamic_patch > 1 else 1
97 + if dynamic_image_size is False:
98 + expected_num_patches = 1
99 + expected_max_tokens = 256 * expected_num_patches
100 +
101 + actual_max_tokens = get_max_internvl_image_tokens(
102 + ctx=InputContext(ctx.model_config),
103 + max_dynamic_patch=max_dynamic_patch,
104 + dynamic_image_size=dynamic_image_size,
105 + )
106 + assert expected_max_tokens == actual_max_tokens
107 +
108 +
109 +@pytest.mark.parametrize("model", models)
110 +@pytest.mark.parametrize("num_imgs", [1, 2])
111 +@pytest.mark.parametrize("max_dynamic_patch", [1, 4, None])
112 +@pytest.mark.parametrize("dynamic_image_size", [True, False, None])
113 +def test_dummy_data_override(
114 + dummy_data_for_internvl: Callable,
115 + model: str,
116 + num_imgs: int,
117 + max_dynamic_patch: Optional[int],
118 + dynamic_image_size: Optional[bool],
119 +):
120 + """Ensure dummy_data_for_internvl handles kwargs properly."""
121 + # Same as the previous test - don't initialize mm_processor_kwargs
122 + # in this test and assume that the kwargs will be correctly expanded by
123 + # the partial when calling the dummy data func.
124 + ctx = build_model_context(
125 + model_name=model,
126 + tokenizer_name=model,
127 + trust_remote_code=True,
128 + mm_processor_kwargs=None,
129 + )
130 +
131 + if max_dynamic_patch is None:
132 + max_dynamic_patch = ctx.get_hf_config().max_dynamic_patch
133 + expected_num_patches = max_dynamic_patch + 1 if max_dynamic_patch > 1 else 1
134 + if dynamic_image_size is False:
135 + expected_num_patches = 1
136 + expected_max_tokens = 256 * expected_num_patches
137 +
138 + dummy_data = dummy_data_for_internvl(
139 + ctx=ctx,
140 + seq_len=8192, # Should be bigger than num_imgs * toks_per_img
141 + mm_counts={"image": num_imgs},
142 + max_dynamic_patch=max_dynamic_patch,
143 + dynamic_image_size=dynamic_image_size,
144 + )
145 + sequence_data = dummy_data.seq_data
146 +
147 + tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
148 + image_token_id = tokenizer.encode('<IMG_CONTEXT>',
149 + add_special_tokens=False)[0]
150 +
151 + # Ensure we have the right number of placeholders per size
152 + img_tok_count = sequence_data.get_token_ids().count(image_token_id)
153 + assert img_tok_count == expected_max_tokens * num_imgs
154 +
155 +
156 +@pytest.mark.parametrize("model", models)
157 +@pytest.mark.parametrize("max_dynamic_patch", [1, 4])
158 +@pytest.mark.parametrize("dynamic_image_size", [True, False, None])
159 +@pytest.mark.parametrize("num_imgs", [1, 2])
160 +def test_input_processor_override(
161 + input_processor_for_internvl: Callable,
162 + image_assets: _ImageAssets,
163 + model: str,
164 + num_imgs: int,
165 + max_dynamic_patch: int,
166 + dynamic_image_size: Optional[bool],
167 +):
168 + """Ensure input_processor_for_internvl handles kwargs properly."""
169 + # Same as the previous test - don't initialize mm_processor_kwargs
170 + # in this test and assume that the kwargs will be correctly expanded by
171 + # the partial when calling the custom input processor.
172 + expected_num_patches = max_dynamic_patch + 1 if max_dynamic_patch > 1 else 1
173 + if dynamic_image_size is False:
174 + expected_num_patches = 1
175 +
176 + ctx = build_model_context(
177 + model_name=model,
178 + tokenizer_name=model,
179 + trust_remote_code=True,
180 + mm_processor_kwargs=None,
181 + )
182 + expected_toks_per_img = 256 * expected_num_patches
183 +
184 + # Build the image str / prompt based on the number of images we pass
185 + tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
186 + placeholders = "<image>" if num_imgs == 1 else "\n".join(
187 + f"Image-{i}: <image>\n" for i in range(1, num_imgs + 1))
188 + prompt = placeholders
189 + images = [image_assets[0].pil_image.resize((448 * 2, 448 * 2))] * num_imgs
190 +
191 + inputs = token_inputs(prompt_token_ids=tokenizer.encode(prompt),
192 + prompt=prompt,
193 + multi_modal_data={"image": images})
194 +
195 + processed_inputs = input_processor_for_internvl(
196 + ctx,
197 + inputs,
198 + max_dynamic_patch=max_dynamic_patch,
199 + dynamic_image_size=dynamic_image_size,
200 + )
201 +
202 + # Ensure we have the right number of placeholders per num_crops size
203 + image_token_id = tokenizer.encode('<IMG_CONTEXT>',
204 + add_special_tokens=False)[0]
205 + img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
206 + assert img_tok_count == expected_toks_per_img * num_imgs
vllm/model_executor/models/internvl.py
+49 -14
@@ -123,8 +123,15 @@ def calculate_num_blocks(orig_width: int, orig_height: int, min_num: int,
123 return blocks, target_width, target_height
124
125
126 -def calculate_num_blocks_wrapper(hf_config: PretrainedConfig,
127 - max_dynamic_patch: Optional[int] = None):
126 +def calculate_num_blocks_wrapper(
127 + hf_config: PretrainedConfig,
128 + max_dynamic_patch: Optional[int] = None,
129 + dynamic_image_size: Optional[bool] = None,
130 +):
131 + if dynamic_image_size is None:
132 + dynamic_image_size = hf_config.dynamic_image_size
133 +
134 + max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
135 if max_dynamic_patch is None:
136 max_dynamic_patch = hf_config.max_dynamic_patch
137 min_num = hf_config.min_dynamic_patch
@@ -183,10 +190,17 @@ def image_to_pixel_values(image: Image.Image, input_size: int, min_num: int,
190 return pixel_values
191
192
186 -def image_to_pixel_values_wrapper(hf_config: PretrainedConfig,
187 - max_dynamic_patch: Optional[int] = None):
193 +def image_to_pixel_values_wrapper(
194 + hf_config: PretrainedConfig,
195 + max_dynamic_patch: Optional[int] = None,
196 + dynamic_image_size: Optional[bool] = None,
197 +):
198 image_size = hf_config.vision_config.image_size
199 min_num = hf_config.min_dynamic_patch
200 + if dynamic_image_size is None:
201 + dynamic_image_size = hf_config.dynamic_image_size
202 +
203 + max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
204 if max_dynamic_patch is None:
205 max_dynamic_patch = hf_config.max_dynamic_patch
206 use_thumbnail = hf_config.use_thumbnail
@@ -207,11 +221,17 @@ def get_internvl_num_patches(hf_config: PretrainedConfig):
221 (downsample_ratio**2))
222
223
210 -def get_max_internvl_image_tokens(ctx: InputContext,
211 - *,
212 - max_dynamic_patch: Optional[int] = None):
224 +def get_max_internvl_image_tokens(
225 + ctx: InputContext,
226 + *,
227 + max_dynamic_patch: Optional[int] = None,
228 + dynamic_image_size: Optional[bool] = None,
229 +):
230 hf_config = ctx.get_hf_config()
231 + if dynamic_image_size is None:
232 + dynamic_image_size = hf_config.dynamic_image_size
233
234 + max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
235 if max_dynamic_patch is None:
236 max_dynamic_patch = hf_config.max_dynamic_patch
237 use_thumbnail = hf_config.use_thumbnail
@@ -222,12 +242,18 @@ def get_max_internvl_image_tokens(ctx: InputContext,
242 return num_patches * max_dynamic_patch
243
244
225 -def get_max_internvl_image_size(ctx: InputContext,
226 - *,
227 - max_dynamic_patch: Optional[int] = None):
245 +def get_max_internvl_image_size(
246 + ctx: InputContext,
247 + *,
248 + max_dynamic_patch: Optional[int] = None,
249 + dynamic_image_size: Optional[bool] = None,
250 +):
251 hf_config = ctx.get_hf_config()
252 image_size = hf_config.vision_config.image_size
253 + if dynamic_image_size is None:
254 + dynamic_image_size = hf_config.dynamic_image_size
255
256 + max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
257 if max_dynamic_patch is None:
258 max_dynamic_patch = hf_config.max_dynamic_patch
259 use_thumbnail = hf_config.use_thumbnail
@@ -281,6 +307,7 @@ class InternVLInputPipeline:
307 inputs: DecoderOnlyInputs,
308 *,
309 max_dynamic_patch: Optional[int] = None,
310 + dynamic_image_size: Optional[bool] = None,
311 ) -> DecoderOnlyInputs:
312 multi_modal_data = inputs.get("multi_modal_data")
313 if multi_modal_data is None or "image" not in multi_modal_data:
@@ -292,7 +319,7 @@ class InternVLInputPipeline:
319 image_data = multi_modal_data["image"]
320 num_patches = get_internvl_num_patches(hf_config)
321 num_blocks_calculator = calculate_num_blocks_wrapper(
295 - hf_config, max_dynamic_patch)
322 + hf_config, max_dynamic_patch, dynamic_image_size)
323 if isinstance(image_data, Image.Image):
324 width, height = image_data.size
325 num_blocks, _, _ = num_blocks_calculator(width, height)
@@ -332,11 +359,12 @@ class InternVLInputPipeline:
359 data: object,
360 *,
361 max_dynamic_patch: Optional[int] = None,
362 + dynamic_image_size: Optional[bool] = None,
363 ):
364 hf_config = ctx.get_hf_config()
365
366 image_pixel_values_mapper = image_to_pixel_values_wrapper(
339 - hf_config, max_dynamic_patch)
367 + hf_config, max_dynamic_patch, dynamic_image_size)
368 if isinstance(data, Image.Image):
369 data = image_pixel_values_mapper(data)
370 # Add an N dimension for number of images per prompt (currently 1).
@@ -366,13 +394,17 @@ class InternVLInputPipeline:
394 mm_counts: Mapping[str, int],
395 *,
396 max_dynamic_patch: Optional[int] = None,
397 + dynamic_image_size: Optional[bool] = None,
398 ):
399 num_images = mm_counts["image"]
400
401 hf_config = ctx.get_hf_config()
402
403 image_feature_size = get_max_internvl_image_tokens(
375 - ctx, max_dynamic_patch=max_dynamic_patch)
404 + ctx,
405 + max_dynamic_patch=max_dynamic_patch,
406 + dynamic_image_size=dynamic_image_size,
407 + )
408 model_config = ctx.model_config
409 tokenizer = cached_get_tokenizer(
410 model_config.tokenizer,
@@ -388,7 +420,10 @@ class InternVLInputPipeline:
420 )
421
422 max_image_width, max_image_height = get_max_internvl_image_size(
391 - ctx, max_dynamic_patch=max_dynamic_patch)
423 + ctx,
424 + max_dynamic_patch=max_dynamic_patch,
425 + dynamic_image_size=dynamic_image_size,
426 + )
427
428 mm_data = dummy_image_for_clip(
429 hf_config.vision_config,