A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Model] Expose `dynamic_image_size` as mm_processor_kwargs for InternVL2 models (#10518)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Nov 21, 2024 at 22:20 UTC
d5ec121f95f51184acce4e2c27ad8fc01904d3d9
2 files changed
+255
-14
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_internvl.py
new
+206
@@ -0,0 +1,206 @@
1
+"""Tests for InternVL's multimodal preprocessing kwargs."""
2
+from typing import Callable, Optional
3
+
4
+import pytest
5
+from transformers import AutoTokenizer
6
+
7
+from vllm.inputs import InputContext, token_inputs
8
+from vllm.multimodal import MultiModalRegistry
9
+
10
+from .....conftest import _ImageAssets
11
+from ....utils import build_model_context
12
+
13
+models = ["OpenGVLab/InternVL2-2B"]
14
+
15
+
16
+# Wrap lazy imports to avoid initializing CUDA during test collection
17
+@pytest.fixture()
18
+def input_processor_for_internvl():
19
+ from vllm.model_executor.models.internvl import InternVLInputPipeline
20
+
21
+ pipeline = InternVLInputPipeline('<img>', '</img>', '<IMG_CONTEXT>')
22
+ return pipeline.input_processor
23
+
24
+
25
+@pytest.fixture()
26
+def dummy_data_for_internvl():
27
+ from vllm.model_executor.models.internvl import InternVLInputPipeline
28
+
29
+ pipeline = InternVLInputPipeline('<img>', '</img>', '<IMG_CONTEXT>')
30
+ return pipeline.dummy_data
31
+
32
+
33
+@pytest.fixture()
34
+def get_max_internvl_image_tokens():
35
+ from vllm.model_executor.models.internvl import (
36
+ get_max_internvl_image_tokens)
37
+ return get_max_internvl_image_tokens
38
+
39
+
40
+@pytest.mark.parametrize("model", models)
41
+@pytest.mark.parametrize("max_dynamic_patch", [1, 4])
42
+@pytest.mark.parametrize("dynamic_image_size", [True, False, None])
43
+def test_input_mapper_override(
44
+ model: str,
45
+ image_assets: _ImageAssets,
46
+ max_dynamic_patch: int,
47
+ dynamic_image_size: Optional[bool],
48
+):
49
+ mm_processor_kwargs = {
50
+ "max_dynamic_patch": max_dynamic_patch,
51
+ }
52
+ if dynamic_image_size is not None:
53
+ mm_processor_kwargs["dynamic_image_size"] = dynamic_image_size
54
+
55
+ expected_num_patches = max_dynamic_patch + 1 if max_dynamic_patch > 1 else 1
56
+ if dynamic_image_size is False:
57
+ expected_num_patches = 1
58
+
59
+ ctx = build_model_context(
60
+ model_name=model,
61
+ tokenizer_name=model,
62
+ trust_remote_code=True,
63
+ mm_processor_kwargs=mm_processor_kwargs,
64
+ )
65
+
66
+ mm_registry = MultiModalRegistry()
67
+ mm_registry.init_mm_limits_per_prompt(ctx.model_config)
68
+
69
+ image = image_assets[0].pil_image.resize((448 * 2, 448 * 2))
70
+ vllm_result = mm_registry.map_input(
71
+ ctx.model_config,
72
+ {"image": image},
73
+ )
74
+ assert vllm_result["pixel_values"].size(1) == expected_num_patches
75
+
76
+
77
+@pytest.mark.parametrize("model", models)
78
+@pytest.mark.parametrize("max_dynamic_patch", [1, 4, None])
79
+@pytest.mark.parametrize("dynamic_image_size", [True, False, None])
80
+def test_max_tokens_override(
81
+ get_max_internvl_image_tokens: Callable,
82
+ model: str,
83
+ max_dynamic_patch: Optional[int],
84
+ dynamic_image_size: Optional[bool],
85
+):
86
+ """Ensure get_max_internvl_image_tokens handles mm_processor_kwargs."""
87
+ ctx = build_model_context(
88
+ model_name=model,
89
+ tokenizer_name=model,
90
+ trust_remote_code=True,
91
+ mm_processor_kwargs=None,
92
+ )
93
+
94
+ if max_dynamic_patch is None:
95
+ max_dynamic_patch = ctx.get_hf_config().max_dynamic_patch
96
+ expected_num_patches = max_dynamic_patch + 1 if max_dynamic_patch > 1 else 1
97
+ if dynamic_image_size is False:
98
+ expected_num_patches = 1
99
+ expected_max_tokens = 256 * expected_num_patches
100
+
101
+ actual_max_tokens = get_max_internvl_image_tokens(
102
+ ctx=InputContext(ctx.model_config),
103
+ max_dynamic_patch=max_dynamic_patch,
104
+ dynamic_image_size=dynamic_image_size,
105
+ )
106
+ assert expected_max_tokens == actual_max_tokens
107
+
108
+
109
+@pytest.mark.parametrize("model", models)
110
+@pytest.mark.parametrize("num_imgs", [1, 2])
111
+@pytest.mark.parametrize("max_dynamic_patch", [1, 4, None])
112
+@pytest.mark.parametrize("dynamic_image_size", [True, False, None])
113
+def test_dummy_data_override(
114
+ dummy_data_for_internvl: Callable,
115
+ model: str,
116
+ num_imgs: int,
117
+ max_dynamic_patch: Optional[int],
118
+ dynamic_image_size: Optional[bool],
119
+):
120
+ """Ensure dummy_data_for_internvl handles kwargs properly."""
121
+ # Same as the previous test - don't initialize mm_processor_kwargs
122
+ # in this test and assume that the kwargs will be correctly expanded by
123
+ # the partial when calling the dummy data func.
124
+ ctx = build_model_context(
125
+ model_name=model,
126
+ tokenizer_name=model,
127
+ trust_remote_code=True,
128
+ mm_processor_kwargs=None,
129
+ )
130
+
131
+ if max_dynamic_patch is None:
132
+ max_dynamic_patch = ctx.get_hf_config().max_dynamic_patch
133
+ expected_num_patches = max_dynamic_patch + 1 if max_dynamic_patch > 1 else 1
134
+ if dynamic_image_size is False:
135
+ expected_num_patches = 1
136
+ expected_max_tokens = 256 * expected_num_patches
137
+
138
+ dummy_data = dummy_data_for_internvl(
139
+ ctx=ctx,
140
+ seq_len=8192, # Should be bigger than num_imgs * toks_per_img
141
+ mm_counts={"image": num_imgs},
142
+ max_dynamic_patch=max_dynamic_patch,
143
+ dynamic_image_size=dynamic_image_size,
144
+ )
145
+ sequence_data = dummy_data.seq_data
146
+
147
+ tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
148
+ image_token_id = tokenizer.encode('<IMG_CONTEXT>',
149
+ add_special_tokens=False)[0]
150
+
151
+ # Ensure we have the right number of placeholders per size
152
+ img_tok_count = sequence_data.get_token_ids().count(image_token_id)
153
+ assert img_tok_count == expected_max_tokens * num_imgs
154
+
155
+
156
+@pytest.mark.parametrize("model", models)
157
+@pytest.mark.parametrize("max_dynamic_patch", [1, 4])
158
+@pytest.mark.parametrize("dynamic_image_size", [True, False, None])
159
+@pytest.mark.parametrize("num_imgs", [1, 2])
160
+def test_input_processor_override(
161
+ input_processor_for_internvl: Callable,
162
+ image_assets: _ImageAssets,
163
+ model: str,
164
+ num_imgs: int,
165
+ max_dynamic_patch: int,
166
+ dynamic_image_size: Optional[bool],
167
+):
168
+ """Ensure input_processor_for_internvl handles kwargs properly."""
169
+ # Same as the previous test - don't initialize mm_processor_kwargs
170
+ # in this test and assume that the kwargs will be correctly expanded by
171
+ # the partial when calling the custom input processor.
172
+ expected_num_patches = max_dynamic_patch + 1 if max_dynamic_patch > 1 else 1
173
+ if dynamic_image_size is False:
174
+ expected_num_patches = 1
175
+
176
+ ctx = build_model_context(
177
+ model_name=model,
178
+ tokenizer_name=model,
179
+ trust_remote_code=True,
180
+ mm_processor_kwargs=None,
181
+ )
182
+ expected_toks_per_img = 256 * expected_num_patches
183
+
184
+ # Build the image str / prompt based on the number of images we pass
185
+ tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
186
+ placeholders = "<image>" if num_imgs == 1 else "\n".join(
187
+ f"Image-{i}: <image>\n" for i in range(1, num_imgs + 1))
188
+ prompt = placeholders
189
+ images = [image_assets[0].pil_image.resize((448 * 2, 448 * 2))] * num_imgs
190
+
191
+ inputs = token_inputs(prompt_token_ids=tokenizer.encode(prompt),
192
+ prompt=prompt,
193
+ multi_modal_data={"image": images})
194
+
195
+ processed_inputs = input_processor_for_internvl(
196
+ ctx,
197
+ inputs,
198
+ max_dynamic_patch=max_dynamic_patch,
199
+ dynamic_image_size=dynamic_image_size,
200
+ )
201
+
202
+ # Ensure we have the right number of placeholders per num_crops size
203
+ image_token_id = tokenizer.encode('<IMG_CONTEXT>',
204
+ add_special_tokens=False)[0]
205
+ img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
206
+ assert img_tok_count == expected_toks_per_img * num_imgs
vllm/model_executor/models/internvl.py
+49
-14
@@ -123,8 +123,15 @@ def calculate_num_blocks(orig_width: int, orig_height: int, min_num: int,
123
return blocks, target_width, target_height
124
125
126
-def calculate_num_blocks_wrapper(hf_config: PretrainedConfig,
127
- max_dynamic_patch: Optional[int] = None):
126
+def calculate_num_blocks_wrapper(
127
+ hf_config: PretrainedConfig,
128
+ max_dynamic_patch: Optional[int] = None,
129
+ dynamic_image_size: Optional[bool] = None,
130
+):
131
+ if dynamic_image_size is None:
132
+ dynamic_image_size = hf_config.dynamic_image_size
133
+
134
+ max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
135
if max_dynamic_patch is None:
136
max_dynamic_patch = hf_config.max_dynamic_patch
137
min_num = hf_config.min_dynamic_patch
@@ -183,10 +190,17 @@ def image_to_pixel_values(image: Image.Image, input_size: int, min_num: int,
190
return pixel_values
191
192
186
-def image_to_pixel_values_wrapper(hf_config: PretrainedConfig,
187
- max_dynamic_patch: Optional[int] = None):
193
+def image_to_pixel_values_wrapper(
194
+ hf_config: PretrainedConfig,
195
+ max_dynamic_patch: Optional[int] = None,
196
+ dynamic_image_size: Optional[bool] = None,
197
+):
198
image_size = hf_config.vision_config.image_size
199
min_num = hf_config.min_dynamic_patch
200
+ if dynamic_image_size is None:
201
+ dynamic_image_size = hf_config.dynamic_image_size
202
+
203
+ max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
204
if max_dynamic_patch is None:
205
max_dynamic_patch = hf_config.max_dynamic_patch
206
use_thumbnail = hf_config.use_thumbnail
@@ -207,11 +221,17 @@ def get_internvl_num_patches(hf_config: PretrainedConfig):
221
(downsample_ratio**2))
222
223
210
-def get_max_internvl_image_tokens(ctx: InputContext,
211
- *,
212
- max_dynamic_patch: Optional[int] = None):
224
+def get_max_internvl_image_tokens(
225
+ ctx: InputContext,
226
+ *,
227
+ max_dynamic_patch: Optional[int] = None,
228
+ dynamic_image_size: Optional[bool] = None,
229
+):
230
hf_config = ctx.get_hf_config()
231
+ if dynamic_image_size is None:
232
+ dynamic_image_size = hf_config.dynamic_image_size
233
234
+ max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
235
if max_dynamic_patch is None:
236
max_dynamic_patch = hf_config.max_dynamic_patch
237
use_thumbnail = hf_config.use_thumbnail
@@ -222,12 +242,18 @@ def get_max_internvl_image_tokens(ctx: InputContext,
242
return num_patches * max_dynamic_patch
243
244
225
-def get_max_internvl_image_size(ctx: InputContext,
226
- *,
227
- max_dynamic_patch: Optional[int] = None):
245
+def get_max_internvl_image_size(
246
+ ctx: InputContext,
247
+ *,
248
+ max_dynamic_patch: Optional[int] = None,
249
+ dynamic_image_size: Optional[bool] = None,
250
+):
251
hf_config = ctx.get_hf_config()
252
image_size = hf_config.vision_config.image_size
253
+ if dynamic_image_size is None:
254
+ dynamic_image_size = hf_config.dynamic_image_size
255
256
+ max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
257
if max_dynamic_patch is None:
258
max_dynamic_patch = hf_config.max_dynamic_patch
259
use_thumbnail = hf_config.use_thumbnail
@@ -281,6 +307,7 @@ class InternVLInputPipeline:
307
inputs: DecoderOnlyInputs,
308
*,
309
max_dynamic_patch: Optional[int] = None,
310
+ dynamic_image_size: Optional[bool] = None,
311
) -> DecoderOnlyInputs:
312
multi_modal_data = inputs.get("multi_modal_data")
313
if multi_modal_data is None or "image" not in multi_modal_data:
@@ -292,7 +319,7 @@ class InternVLInputPipeline:
319
image_data = multi_modal_data["image"]
320
num_patches = get_internvl_num_patches(hf_config)
321
num_blocks_calculator = calculate_num_blocks_wrapper(
295
- hf_config, max_dynamic_patch)
322
+ hf_config, max_dynamic_patch, dynamic_image_size)
323
if isinstance(image_data, Image.Image):
324
width, height = image_data.size
325
num_blocks, _, _ = num_blocks_calculator(width, height)
@@ -332,11 +359,12 @@ class InternVLInputPipeline:
359
data: object,
360
*,
361
max_dynamic_patch: Optional[int] = None,
362
+ dynamic_image_size: Optional[bool] = None,
363
):
364
hf_config = ctx.get_hf_config()
365
366
image_pixel_values_mapper = image_to_pixel_values_wrapper(
339
- hf_config, max_dynamic_patch)
367
+ hf_config, max_dynamic_patch, dynamic_image_size)
368
if isinstance(data, Image.Image):
369
data = image_pixel_values_mapper(data)
370
# Add an N dimension for number of images per prompt (currently 1).
@@ -366,13 +394,17 @@ class InternVLInputPipeline:
394
mm_counts: Mapping[str, int],
395
*,
396
max_dynamic_patch: Optional[int] = None,
397
+ dynamic_image_size: Optional[bool] = None,
398
):
399
num_images = mm_counts["image"]
400
401
hf_config = ctx.get_hf_config()
402
403
image_feature_size = get_max_internvl_image_tokens(
375
- ctx, max_dynamic_patch=max_dynamic_patch)
404
+ ctx,
405
+ max_dynamic_patch=max_dynamic_patch,
406
+ dynamic_image_size=dynamic_image_size,
407
+ )
408
model_config = ctx.model_config
409
tokenizer = cached_get_tokenizer(
410
model_config.tokenizer,
@@ -388,7 +420,10 @@ class InternVLInputPipeline:
420
)
421
422
max_image_width, max_image_height = get_max_internvl_image_size(
391
- ctx, max_dynamic_patch=max_dynamic_patch)
423
+ ctx,
424
+ max_dynamic_patch=max_dynamic_patch,
425
+ dynamic_image_size=dynamic_image_size,
426
+ )
427
428
mm_data = dummy_image_for_clip(
429
hf_config.vision_config,