A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[VLM] Merged multi-modal processor for InternVL-based models (#12553)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <2037008807@qq.com>
Cyrus Leung committed
Feb 4, 2025 at 16:44 UTC
d1ca7df84d9f8853001bdf1c2900321d9cb5d64e
34 files changed
+1434
-986
docs/source/contributing/model/multimodal.md
+5
-1
@@ -250,7 +250,11 @@ def get_max_image_tokens(self) -> int:
250
And thus, we can override the method as:
251
252
```python
253
-def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
253
+def get_mm_max_tokens_per_item(
254
+ self,
255
+ seq_len: int,
256
+ mm_counts: Mapping[str, int],
257
+) -> Mapping[str, int]:
258
return {"image": self.get_max_image_tokens()}
259
```
260
docs/source/models/supported_models.md
+7
-3
@@ -726,7 +726,7 @@ See [this page](#generative-models) for more information on how to use generativ
726
* `h2oai/h2ovl-mississippi-800m`, `h2oai/h2ovl-mississippi-2b`, etc.
727
*
728
* ✅︎
729
- *
729
+ * \*
730
- * `Idefics3ForConditionalGeneration`
731
* Idefics3
732
* T + I
@@ -799,7 +799,7 @@ See [this page](#generative-models) for more information on how to use generativ
799
* ✅︎
800
- * `NVLM_D_Model`
801
* NVLM-D 1.0
802
- * T + I<sup>E+</sup>
802
+ * T + I<sup>+</sup>
803
* `nvidia/NVLM-D-72B`, etc.
804
*
805
* ✅︎
@@ -859,7 +859,11 @@ See [this page](#generative-models) for more information on how to use generativ
859
<sup>+</sup> Multiple items can be inputted per text prompt for this modality.
860
861
:::{note}
862
-To use `DeepSeek-VL2` series models, you have to pass `--hf_overrides '{"architectures": ["DeepseekVLV2ForCausalLM"]}'` when running vLLM.
862
+To use DeepSeek-VL2 series models, you have to pass `--hf_overrides '{"architectures": ["DeepseekVLV2ForCausalLM"]}'` when running vLLM.
863
+:::
864
+
865
+:::{note}
866
+H2O-VL series models will be available in V1 once we support backends other than FlashAttention.
867
:::
868
869
:::{note}
tests/models/decoder_only/vision_language/test_h2ovl.py
deleted
-131
@@ -1,131 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-
3
-from typing import Optional, Tuple
4
-
5
-import pytest
6
-import torch
7
-from PIL.Image import Image
8
-from transformers import AutoConfig
9
-
10
-# Import the functions to test
11
-from vllm.model_executor.models.h2ovl import (calculate_num_blocks,
12
- image_to_pixel_values_wrapper)
13
-from vllm.multimodal.image import rescale_image_size
14
-
15
-models = [
16
- "h2oai/h2ovl-mississippi-800m", # Replace with your actual model names
17
- "h2oai/h2ovl-mississippi-2b",
18
-]
19
-
20
-
21
-def run_preprocessing_test(
22
- image: Image,
23
- config,
24
- max_dynamic_patch: Optional[int] = None,
25
-) -> Tuple[torch.Tensor, int]:
26
- """Test the image preprocessing and calculate expected blocks."""
27
-
28
- if max_dynamic_patch is None:
29
- max_dynamic_patch = config.max_dynamic_patch
30
-
31
- width, height = image.size
32
- use_MSAC = config.use_msac
33
-
34
- # Create the mapper function with the provided configuration
35
- mapper = image_to_pixel_values_wrapper(config, max_dynamic_patch, use_MSAC)
36
- pixel_values = mapper(image)
37
-
38
- # Calculate the expected number of blocks
39
- if use_MSAC:
40
- # First pass
41
- blocks1, _, _, aspect_ratio = calculate_num_blocks(
42
- width,
43
- height,
44
- config.min_dynamic_patch,
45
- max_dynamic_patch,
46
- config.vision_config.image_size,
47
- use_thumbnail=False, # Thumbnail is handled separately
48
- prior_aspect_ratio=None,
49
- )
50
-
51
- # Second pass
52
- blocks2, _, _, _ = calculate_num_blocks(
53
- width,
54
- height,
55
- config.min_dynamic_patch,
56
- max_dynamic_patch,
57
- config.vision_config.image_size,
58
- use_thumbnail=False,
59
- prior_aspect_ratio=aspect_ratio,
60
- )
61
-
62
- # Add thumbnail if use_thumbnail is True and total_blocks > 1
63
- if config.use_thumbnail:
64
- blocks1 += 1 if blocks1 > 1 else 0
65
- blocks2 += 1 if blocks2 > 1 else 0
66
-
67
- # Total blocks is the sum of blocks from both passes minus overlapping
68
- total_blocks = blocks1 + blocks2 - 1
69
-
70
- expected_blocks = total_blocks
71
-
72
- else:
73
- blocks, _, _, _ = calculate_num_blocks(
74
- width,
75
- height,
76
- config.min_dynamic_patch,
77
- max_dynamic_patch,
78
- config.vision_config.image_size,
79
- use_thumbnail=False,
80
- prior_aspect_ratio=None,
81
- )
82
- expected_blocks = blocks
83
-
84
- if config.use_thumbnail and expected_blocks > 1:
85
- expected_blocks += 1
86
-
87
- return pixel_values, expected_blocks
88
-
89
-
90
-@pytest.mark.parametrize("model_name", models)
91
-@pytest.mark.parametrize(
92
- "size_factors",
93
- [
94
- # Single-scale
95
- [1.0],
96
- # Single-scale, batched
97
- [1.0, 1.0, 1.0],
98
- # Multi-scale
99
- [0.25, 0.5, 1.0],
100
- ],
101
-)
102
-@pytest.mark.parametrize("max_dynamic_patch", [None, 2, 4, 8])
103
-def test_image_preprocessing(image_assets, model_name, size_factors,
104
- max_dynamic_patch):
105
- """Test image preprocessing pipeline with different configurations."""
106
- # Load the configuration from the model
107
- config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
108
-
109
- for asset in image_assets:
110
- image = asset.pil_image
111
- for factor in size_factors:
112
- scaled_image = rescale_image_size(image, factor)
113
-
114
- # Test preprocessing and get expected number of blocks
115
- pixel_values, expected_blocks = run_preprocessing_test(
116
- scaled_image, config, max_dynamic_patch)
117
-
118
- # Verify output shapes and properties
119
- actual_blocks = pixel_values.shape[0]
120
- assert actual_blocks == expected_blocks, (
121
- f"Expected {expected_blocks} blocks, got {actual_blocks}")
122
-
123
- # Check image dimensions
124
- expected_size = (
125
- 3, # Number of channels (C, H, W)
126
- config.vision_config.image_size,
127
- config.vision_config.image_size,
128
- )
129
- for img in pixel_values:
130
- assert img.shape == expected_size, (
131
- f"Expected image size {expected_size}, got {img.shape}")
tests/models/decoder_only/vision_language/test_models.py
+1
-1
@@ -250,6 +250,7 @@ VLM_TEST_SETTINGS = {
250
max_model_len=8192,
251
dtype="bfloat16",
252
use_tokenizer_eos=True,
253
+ num_logprobs=10,
254
patch_hf_runner=model_utils.h2ovl_patch_hf_runner,
255
),
256
"idefics3": VLMTestInfo(
@@ -282,7 +283,6 @@ VLM_TEST_SETTINGS = {
283
dtype="bfloat16",
284
use_tokenizer_eos=True,
285
patch_hf_runner=model_utils.internvl_patch_hf_runner,
285
- marks=[large_gpu_mark(min_gb=32)],
286
),
287
"llava_next": VLMTestInfo(
288
models=["llava-hf/llava-v1.6-mistral-7b-hf"],
tests/models/decoder_only/vision_language/vlm_utils/model_utils.py
+21
-16
@@ -334,12 +334,12 @@ def h2ovl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
334
def __init__(self, hf_runner: HfRunner):
335
self.num_image_token = hf_runner.model.num_image_token
336
self.tokenizer = hf_runner.tokenizer
337
- self.dtype = hf_runner.model.dtype
337
338
self.config = AutoConfig.from_pretrained(hf_runner.model_name,
339
trust_remote_code=True)
340
self.vision_config = self.config.vision_config
341
self.use_thumbnail = self.config.use_thumbnail
342
+ self.use_msac = self.config.use_msac
343
self.min_num = self.config.min_dynamic_patch
344
self.max_num = self.config.max_dynamic_patch
345
self.image_size = self.vision_config.image_size
@@ -348,18 +348,19 @@ def h2ovl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
348
**kwargs):
349
# yapf: disable
350
from vllm.model_executor.models.h2ovl import (
351
- IMG_CONTEXT, IMG_END, IMG_START, image_to_pixel_values)
351
+ IMG_CONTEXT, IMG_END, IMG_START, image_to_pixel_values_h2ovl)
352
353
# yapf: enable
354
images = [images] if isinstance(images, Image) else images
355
pixel_values = [
356
- image_to_pixel_values(image,
357
- self.image_size,
358
- self.min_num,
359
- self.max_num,
360
- self.use_thumbnail,
361
- use_MSAC=self.config.use_msac).to(
362
- self.dtype) for image in images
356
+ image_to_pixel_values_h2ovl(
357
+ image,
358
+ input_size=self.image_size,
359
+ min_num=self.min_num,
360
+ max_num=self.max_num,
361
+ use_thumbnail=self.use_thumbnail,
362
+ use_msac=self.use_msac,
363
+ ) for image in images
364
]
365
num_patches_list = [
366
pixel_value.shape[0] for pixel_value in pixel_values
@@ -394,7 +395,6 @@ def internvl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
395
def __init__(self, hf_runner: HfRunner):
396
self.num_image_token = hf_runner.model.num_image_token
397
self.tokenizer = hf_runner.tokenizer
397
- self.dtype = hf_runner.model.dtype
398
399
self.config = AutoConfig.from_pretrained(hf_runner.model_name,
400
trust_remote_code=True)
@@ -407,13 +407,17 @@ def internvl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
407
def __call__(self, text: str, images: Union[Image, List[Image]],
408
**kwargs):
409
from vllm.model_executor.models.internvl import (
410
- IMG_CONTEXT, IMG_END, IMG_START, image_to_pixel_values)
410
+ IMG_CONTEXT, IMG_END, IMG_START,
411
+ image_to_pixel_values_internvl)
412
images = [images] if isinstance(images, Image) else images
413
pixel_values = [
413
- image_to_pixel_values(image, self.image_size, self.min_num,
414
- self.max_num,
415
- self.use_thumbnail).to(self.dtype)
416
- for image in images
414
+ image_to_pixel_values_internvl(
415
+ image,
416
+ input_size=self.image_size,
417
+ min_num=self.min_num,
418
+ max_num=self.max_num,
419
+ use_thumbnail=self.use_thumbnail,
420
+ ) for image in images
421
]
422
num_patches_list = [
423
pixel_value.shape[0] for pixel_value in pixel_values
@@ -448,7 +452,8 @@ def _internvl_generate(
452
) -> torch.LongTensor:
453
"""Generate method for InternVL2 model without fixed use_cache."""
454
assert self.img_context_token_id is not None
451
- vit_embeds = self.extract_feature(pixel_values)
455
+ target_dtype = next(self.parameters()).dtype
456
+ vit_embeds = self.extract_feature(pixel_values.to(target_dtype))
457
input_embeds = self.language_model.get_input_embeddings()(input_ids)
458
B, N, C = input_embeds.shape
459
input_embeds = input_embeds.reshape(B * N, C)
tests/models/multimodal/processing/test_common.py
+3
-1
@@ -141,13 +141,14 @@ def _test_processing_correctness(
141
142
143
# yapf: disable
144
-# True if the model supports multiple data items of the modality per request
144
@pytest.mark.parametrize("model_id", [
145
"rhymes-ai/Aria",
146
"Salesforce/blip2-opt-2.7b",
147
"facebook/chameleon-7b",
148
"deepseek-ai/deepseek-vl2-tiny",
149
"adept/fuyu-8b",
150
+ "h2oai/h2ovl-mississippi-800m",
151
+ "OpenGVLab/InternVL2-1B",
152
"llava-hf/llava-1.5-7b-hf",
153
"llava-hf/llava-v1.6-mistral-7b-hf",
154
"llava-hf/LLaVA-NeXT-Video-7B-hf",
@@ -156,6 +157,7 @@ def _test_processing_correctness(
157
"mistral-community/pixtral-12b",
158
"openbmb/MiniCPM-o-2_6",
159
"openbmb/MiniCPM-V-2_6",
160
+ "nvidia/NVLM-D-72B",
161
"Qwen/Qwen-VL-Chat",
162
"Qwen/Qwen2-VL-2B-Instruct",
163
"Qwen/Qwen2-Audio-7B-Instruct",
tests/models/multimodal/processing/test_h2ovl.py
new
+142
@@ -0,0 +1,142 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+"""Tests for H2OVL's multimodal preprocessing kwargs."""
3
+from typing import Optional
4
+
5
+import pytest
6
+
7
+from vllm.multimodal import MULTIMODAL_REGISTRY
8
+from vllm.multimodal.image import rescale_image_size
9
+from vllm.multimodal.utils import cached_get_tokenizer
10
+
11
+from ....conftest import _ImageAssets
12
+from ...utils import build_model_context
13
+
14
+
15
+@pytest.mark.parametrize("model_id", [
16
+ "h2oai/h2ovl-mississippi-800m",
17
+ "h2oai/h2ovl-mississippi-2b",
18
+])
19
+@pytest.mark.parametrize(
20
+ "size_factors",
21
+ [
22
+ # Single-scale
23
+ [1.0],
24
+ # Single-scale, batched
25
+ [1.0, 1.0, 1.0],
26
+ # Multi-scale
27
+ [0.25, 0.5, 1.0],
28
+ ],
29
+)
30
+@pytest.mark.parametrize("max_dynamic_patch", [1, 2, 4, 8])
31
+@pytest.mark.parametrize("dynamic_image_size", [True, False])
32
+@pytest.mark.parametrize("num_imgs", [1, 2])
33
+def test_processor_override(
34
+ model_id: str,
35
+ image_assets: _ImageAssets,
36
+ size_factors: list[int],
37
+ max_dynamic_patch: int,
38
+ dynamic_image_size: Optional[bool],
39
+ num_imgs: int,
40
+):
41
+ from vllm.model_executor.models.h2ovl import (calculate_h2ovl_targets,
42
+ get_h2ovl_target_ratios)
43
+
44
+ ctx = build_model_context(
45
+ model_name=model_id,
46
+ tokenizer_name=model_id,
47
+ trust_remote_code=True,
48
+ mm_processor_kwargs=None,
49
+ limit_mm_per_prompt={"image": num_imgs},
50
+ )
51
+ tokenizer = cached_get_tokenizer(
52
+ ctx.model_config.tokenizer,
53
+ trust_remote_code=ctx.model_config.trust_remote_code,
54
+ )
55
+ processor = MULTIMODAL_REGISTRY.create_processor(
56
+ ctx.model_config,
57
+ tokenizer=tokenizer,
58
+ )
59
+
60
+ config = processor.info.get_hf_config()
61
+ use_msac = config.use_msac
62
+
63
+ mm_processor_kwargs = {
64
+ "max_dynamic_patch": max_dynamic_patch,
65
+ }
66
+ if dynamic_image_size is not None:
67
+ mm_processor_kwargs["dynamic_image_size"] = dynamic_image_size
68
+
69
+ min_num = config.min_dynamic_patch
70
+ max_num = max_dynamic_patch if dynamic_image_size else 1
71
+
72
+ # Build the image str / prompt based on the number of images we pass
73
+ prompt = "<image>" * num_imgs
74
+
75
+ for asset in image_assets:
76
+ for factor in size_factors:
77
+ image = rescale_image_size(asset.pil_image, factor)
78
+ mm_data = {"image": [image] * num_imgs}
79
+
80
+ width, height = image.size
81
+
82
+ # Calculate the expected number of blocks
83
+ if num_imgs == 1 and use_msac:
84
+ # First pass
85
+ blocks1, _, _, aspect_ratio = calculate_h2ovl_targets(
86
+ orig_width=width,
87
+ orig_height=height,
88
+ target_ratios=get_h2ovl_target_ratios(
89
+ min_num,
90
+ max_num,
91
+ prior_aspect_ratio=None,
92
+ ),
93
+ image_size=config.vision_config.image_size,
94
+ use_thumbnail=False, # Thumbnail is handled separately
95
+ )
96
+
97
+ # Second pass
98
+ blocks2, _, _, _ = calculate_h2ovl_targets(
99
+ orig_width=width,
100
+ orig_height=height,
101
+ target_ratios=get_h2ovl_target_ratios(
102
+ min_num,
103
+ max_num,
104
+ prior_aspect_ratio=aspect_ratio,
105
+ ),
106
+ image_size=config.vision_config.image_size,
107
+ use_thumbnail=False,
108
+ )
109
+
110
+ # Add thumbnail if use_thumbnail is True and total_blocks > 1
111
+ if config.use_thumbnail:
112
+ blocks1 += 1 if blocks1 > 1 else 0
113
+ blocks2 += 1 if blocks2 > 1 else 0
114
+
115
+ # Total blocks is the sum of blocks from both passes minus
116
+ # overlapping
117
+ total_blocks = blocks1 + blocks2 - 1
118
+
119
+ expected_num_patches = total_blocks
120
+ else:
121
+ blocks, _, _, _ = calculate_h2ovl_targets(
122
+ orig_width=width,
123
+ orig_height=height,
124
+ target_ratios=get_h2ovl_target_ratios(
125
+ min_num,
126
+ max_num,
127
+ prior_aspect_ratio=None,
128
+ ),
129
+ image_size=config.vision_config.image_size,
130
+ use_thumbnail=False,
131
+ )
132
+ expected_num_patches = blocks
133
+
134
+ if config.use_thumbnail and expected_num_patches != 1:
135
+ expected_num_patches += 1
136
+
137
+ processed_inputs = processor.apply(prompt, mm_data,
138
+ mm_processor_kwargs)
139
+ pixel_shape = (
140
+ processed_inputs["mm_kwargs"]["pixel_values_flat"].shape)
141
+
142
+ assert pixel_shape[0] == expected_num_patches * num_imgs
tests/models/multimodal/processing/test_internvl.py
+32
-175
@@ -1,207 +1,64 @@
1
# SPDX-License-Identifier: Apache-2.0
2
"""Tests for InternVL's multimodal preprocessing kwargs."""
3
-from typing import Callable, Optional
3
+from typing import Optional
4
5
import pytest
6
-from transformers import AutoTokenizer
6
8
-from vllm.inputs import InputContext, token_inputs
9
-from vllm.multimodal import MultiModalRegistry
7
+from vllm.multimodal import MULTIMODAL_REGISTRY
8
+from vllm.multimodal.utils import cached_get_tokenizer
9
10
from ....conftest import _ImageAssets
11
from ...utils import build_model_context
12
14
-models = ["OpenGVLab/InternVL2-2B"]
13
16
-
17
-# Wrap lazy imports to avoid initializing CUDA during test collection
18
-@pytest.fixture()
19
-def input_processor_for_internvl():
20
- from vllm.model_executor.models.internvl import InternVLInputPipeline
21
-
22
- pipeline = InternVLInputPipeline('<img>', '</img>', '<IMG_CONTEXT>')
23
- return pipeline.input_processor
24
-
25
-
26
-@pytest.fixture()
27
-def dummy_data_for_internvl():
28
- from vllm.model_executor.models.internvl import InternVLInputPipeline
29
-
30
- pipeline = InternVLInputPipeline('<img>', '</img>', '<IMG_CONTEXT>')
31
- return pipeline.dummy_data
32
-
33
-
34
-@pytest.fixture()
35
-def get_max_internvl_image_tokens():
36
- from vllm.model_executor.models.internvl import (
37
- get_max_internvl_image_tokens)
38
- return get_max_internvl_image_tokens
39
-
40
-
41
-@pytest.mark.parametrize("model", models)
14
+@pytest.mark.parametrize("model_id", ["OpenGVLab/InternVL2-2B"])
15
@pytest.mark.parametrize("max_dynamic_patch", [1, 4])
16
@pytest.mark.parametrize("dynamic_image_size", [True, False, None])
44
-def test_input_mapper_override(
45
- model: str,
17
+@pytest.mark.parametrize("num_imgs", [1, 2])
18
+def test_processor_override(
19
+ model_id: str,
20
image_assets: _ImageAssets,
21
max_dynamic_patch: int,
22
dynamic_image_size: Optional[bool],
49
-):
50
- mm_processor_kwargs = {
51
- "max_dynamic_patch": max_dynamic_patch,
52
- }
53
- if dynamic_image_size is not None:
54
- mm_processor_kwargs["dynamic_image_size"] = dynamic_image_size
55
-
56
- expected_num_patches = max_dynamic_patch + 1 if max_dynamic_patch > 1 else 1
57
- if dynamic_image_size is False:
58
- expected_num_patches = 1
59
-
60
- ctx = build_model_context(
61
- model_name=model,
62
- tokenizer_name=model,
63
- trust_remote_code=True,
64
- mm_processor_kwargs=mm_processor_kwargs,
65
- )
66
-
67
- mm_registry = MultiModalRegistry()
68
- mm_registry.init_mm_limits_per_prompt(ctx.model_config)
69
-
70
- image = image_assets[0].pil_image.resize((448 * 2, 448 * 2))
71
- vllm_result = mm_registry.map_input(
72
- ctx.model_config,
73
- {"image": image},
74
- )
75
- assert vllm_result["pixel_values"].size(1) == expected_num_patches
76
-
77
-
78
-@pytest.mark.parametrize("model", models)
79
-@pytest.mark.parametrize("max_dynamic_patch", [1, 4, None])
80
-@pytest.mark.parametrize("dynamic_image_size", [True, False, None])
81
-def test_max_tokens_override(
82
- get_max_internvl_image_tokens: Callable,
83
- model: str,
84
- max_dynamic_patch: Optional[int],
85
- dynamic_image_size: Optional[bool],
86
-):
87
- """Ensure get_max_internvl_image_tokens handles mm_processor_kwargs."""
88
- ctx = build_model_context(
89
- model_name=model,
90
- tokenizer_name=model,
91
- trust_remote_code=True,
92
- mm_processor_kwargs=None,
93
- )
94
-
95
- if max_dynamic_patch is None:
96
- max_dynamic_patch = ctx.get_hf_config().max_dynamic_patch
97
- expected_num_patches = max_dynamic_patch + 1 if max_dynamic_patch > 1 else 1
98
- if dynamic_image_size is False:
99
- expected_num_patches = 1
100
- expected_max_tokens = 256 * expected_num_patches
101
-
102
- actual_max_tokens = get_max_internvl_image_tokens(
103
- ctx=InputContext(ctx.model_config),
104
- max_dynamic_patch=max_dynamic_patch,
105
- dynamic_image_size=dynamic_image_size,
106
- )
107
- assert expected_max_tokens == actual_max_tokens
108
-
109
-
110
-@pytest.mark.parametrize("model", models)
111
-@pytest.mark.parametrize("num_imgs", [1, 2])
112
-@pytest.mark.parametrize("max_dynamic_patch", [1, 4, None])
113
-@pytest.mark.parametrize("dynamic_image_size", [True, False, None])
114
-def test_dummy_data_override(
115
- dummy_data_for_internvl: Callable,
116
- model: str,
23
num_imgs: int,
118
- max_dynamic_patch: Optional[int],
119
- dynamic_image_size: Optional[bool],
24
):
121
- """Ensure dummy_data_for_internvl handles kwargs properly."""
122
- # Same as the previous test - don't initialize mm_processor_kwargs
123
- # in this test and assume that the kwargs will be correctly expanded by
124
- # the partial when calling the dummy data func.
25
ctx = build_model_context(
126
- model_name=model,
127
- tokenizer_name=model,
26
+ model_name=model_id,
27
+ tokenizer_name=model_id,
28
trust_remote_code=True,
29
mm_processor_kwargs=None,
30
+ limit_mm_per_prompt={"image": num_imgs},
31
)
131
-
132
- if max_dynamic_patch is None:
133
- max_dynamic_patch = ctx.get_hf_config().max_dynamic_patch
134
- expected_num_patches = max_dynamic_patch + 1 if max_dynamic_patch > 1 else 1
135
- if dynamic_image_size is False:
136
- expected_num_patches = 1
137
- expected_max_tokens = 256 * expected_num_patches
138
-
139
- dummy_data = dummy_data_for_internvl(
140
- ctx=ctx,
141
- seq_len=8192, # Should be bigger than num_imgs * toks_per_img
142
- mm_counts={"image": num_imgs},
143
- max_dynamic_patch=max_dynamic_patch,
144
- dynamic_image_size=dynamic_image_size,
32
+ tokenizer = cached_get_tokenizer(
33
+ ctx.model_config.tokenizer,
34
+ trust_remote_code=ctx.model_config.trust_remote_code,
35
+ )
36
+ processor = MULTIMODAL_REGISTRY.create_processor(
37
+ ctx.model_config,
38
+ tokenizer=tokenizer,
39
)
146
- sequence_data = dummy_data.seq_data
147
-
148
- tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
149
- image_token_id = tokenizer.encode('<IMG_CONTEXT>',
150
- add_special_tokens=False)[0]
40
152
- # Ensure we have the right number of placeholders per size
153
- img_tok_count = sequence_data.get_token_ids().count(image_token_id)
154
- assert img_tok_count == expected_max_tokens * num_imgs
41
+ mm_processor_kwargs = {
42
+ "max_dynamic_patch": max_dynamic_patch,
43
+ }
44
+ if dynamic_image_size is not None:
45
+ mm_processor_kwargs["dynamic_image_size"] = dynamic_image_size
46
47
+ # Build the image str / prompt based on the number of images we pass
48
+ prompt = "<image>" * num_imgs
49
+ image = image_assets[0].pil_image.resize((448 * 2, 448 * 2))
50
+ mm_data = {"image": [image] * num_imgs}
51
157
-@pytest.mark.parametrize("model", models)
158
-@pytest.mark.parametrize("max_dynamic_patch", [1, 4])
159
-@pytest.mark.parametrize("dynamic_image_size", [True, False, None])
160
-@pytest.mark.parametrize("num_imgs", [1, 2])
161
-def test_input_processor_override(
162
- input_processor_for_internvl: Callable,
163
- image_assets: _ImageAssets,
164
- model: str,
165
- num_imgs: int,
166
- max_dynamic_patch: int,
167
- dynamic_image_size: Optional[bool],
168
-):
169
- """Ensure input_processor_for_internvl handles kwargs properly."""
170
- # Same as the previous test - don't initialize mm_processor_kwargs
171
- # in this test and assume that the kwargs will be correctly expanded by
172
- # the partial when calling the custom input processor.
52
expected_num_patches = max_dynamic_patch + 1 if max_dynamic_patch > 1 else 1
53
if dynamic_image_size is False:
54
expected_num_patches = 1
55
177
- ctx = build_model_context(
178
- model_name=model,
179
- tokenizer_name=model,
180
- trust_remote_code=True,
181
- mm_processor_kwargs=None,
182
- )
183
- expected_toks_per_img = 256 * expected_num_patches
184
-
185
- # Build the image str / prompt based on the number of images we pass
186
- tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
187
- placeholders = "<image>" if num_imgs == 1 else "\n".join(
188
- f"Image-{i}: <image>\n" for i in range(1, num_imgs + 1))
189
- prompt = placeholders
190
- images = [image_assets[0].pil_image.resize((448 * 2, 448 * 2))] * num_imgs
191
-
192
- inputs = token_inputs(prompt_token_ids=tokenizer.encode(prompt),
193
- prompt=prompt,
194
- multi_modal_data={"image": images})
195
-
196
- processed_inputs = input_processor_for_internvl(
197
- ctx,
198
- inputs,
199
- max_dynamic_patch=max_dynamic_patch,
200
- dynamic_image_size=dynamic_image_size,
201
- )
56
+ processed_inputs = processor.apply(prompt, mm_data, mm_processor_kwargs)
57
58
# Ensure we have the right number of placeholders per num_crops size
204
- image_token_id = tokenizer.encode('<IMG_CONTEXT>',
205
- add_special_tokens=False)[0]
59
+ image_token_id = tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
60
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
207
- assert img_tok_count == expected_toks_per_img * num_imgs
61
+ pixel_shape = processed_inputs["mm_kwargs"]["pixel_values_flat"].shape
62
+
63
+ assert img_tok_count == 256 * expected_num_patches * num_imgs
64
+ assert pixel_shape[0] == expected_num_patches * num_imgs
tests/models/multimodal/processing/test_llava_next.py
+12
-3
@@ -43,7 +43,10 @@ def test_processor_max_tokens(model_id):
43
)
44
processor = MULTIMODAL_REGISTRY.create_processor(
45
ctx.model_config,
46
- tokenizer=cached_get_tokenizer(ctx.model_config.tokenizer),
46
+ tokenizer=cached_get_tokenizer(
47
+ ctx.model_config.tokenizer,
48
+ trust_remote_code=ctx.model_config.trust_remote_code,
49
+ ),
50
)
51
info = processor.info
52
@@ -143,7 +146,10 @@ def test_processor_prompt_replacements_regression(model_id, num_imgs):
146
)
147
processor = MULTIMODAL_REGISTRY.create_processor(
148
ctx.model_config,
146
- tokenizer=cached_get_tokenizer(ctx.model_config.tokenizer),
149
+ tokenizer=cached_get_tokenizer(
150
+ ctx.model_config.tokenizer,
151
+ trust_remote_code=ctx.model_config.trust_remote_code,
152
+ ),
153
)
154
155
image_ratios = [(171, 152), (184, 161), (198, 176), (333, 296), (369, 328),
@@ -173,7 +179,10 @@ def test_processor_prompt_replacements_all(model_id, num_imgs):
179
)
180
processor = MULTIMODAL_REGISTRY.create_processor(
181
ctx.model_config,
176
- tokenizer=cached_get_tokenizer(ctx.model_config.tokenizer),
182
+ tokenizer=cached_get_tokenizer(
183
+ ctx.model_config.tokenizer,
184
+ trust_remote_code=ctx.model_config.trust_remote_code,
185
+ ),
186
)
187
188
seen_aspect_ratios = set[float]()
tests/models/multimodal/processing/test_llava_onevision.py
+12
-3
@@ -44,7 +44,10 @@ def test_processor_max_tokens(model_id):
44
)
45
processor = MULTIMODAL_REGISTRY.create_processor(
46
ctx.model_config,
47
- tokenizer=cached_get_tokenizer(ctx.model_config.tokenizer),
47
+ tokenizer=cached_get_tokenizer(
48
+ ctx.model_config.tokenizer,
49
+ trust_remote_code=ctx.model_config.trust_remote_code,
50
+ ),
51
)
52
info = processor.info
53
@@ -143,7 +146,10 @@ def test_processor_prompt_replacements_regression(model_id, num_imgs):
146
)
147
processor = MULTIMODAL_REGISTRY.create_processor(
148
ctx.model_config,
146
- tokenizer=cached_get_tokenizer(ctx.model_config.tokenizer),
149
+ tokenizer=cached_get_tokenizer(
150
+ ctx.model_config.tokenizer,
151
+ trust_remote_code=ctx.model_config.trust_remote_code,
152
+ ),
153
)
154
155
image_ratios = [(171, 152), (184, 161), (198, 176), (333, 296), (369, 328),
@@ -174,7 +180,10 @@ def test_processor_prompt_replacements_all(model_id, num_imgs):
180
)
181
processor = MULTIMODAL_REGISTRY.create_processor(
182
ctx.model_config,
177
- tokenizer=cached_get_tokenizer(ctx.model_config.tokenizer),
183
+ tokenizer=cached_get_tokenizer(
184
+ ctx.model_config.tokenizer,
185
+ trust_remote_code=ctx.model_config.trust_remote_code,
186
+ ),
187
)
188
189
seen_aspect_ratios = set[float]()
tests/models/multimodal/processing/test_phi3v.py
+4
-1
@@ -38,7 +38,10 @@ def test_processor_override(
38
trust_remote_code=True,
39
limit_mm_per_prompt={"image": num_imgs},
40
)
41
- tokenizer = cached_get_tokenizer(ctx.model_config.tokenizer)
41
+ tokenizer = cached_get_tokenizer(
42
+ ctx.model_config.tokenizer,
43
+ trust_remote_code=ctx.model_config.trust_remote_code,
44
+ )
45
processor = MULTIMODAL_REGISTRY.create_processor(
46
ctx.model_config,
47
tokenizer=tokenizer,
tests/models/multimodal/processing/test_qwen2_vl.py
+4
-1
@@ -33,7 +33,10 @@ def test_processor_override(
33
mm_processor_kwargs=None,
34
limit_mm_per_prompt={"image": num_imgs},
35
)
36
- tokenizer = cached_get_tokenizer(ctx.model_config.tokenizer)
36
+ tokenizer = cached_get_tokenizer(
37
+ ctx.model_config.tokenizer,
38
+ trust_remote_code=ctx.model_config.trust_remote_code,
39
+ )
40
processor = MULTIMODAL_REGISTRY.create_processor(
41
ctx.model_config,
42
tokenizer=tokenizer,
vllm/model_executor/models/aria.py
+5
-1
@@ -399,7 +399,11 @@ class AriaProcessingInfo(BaseProcessingInfo):
399
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
400
return {"image": None}
401
402
- def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
402
+ def get_mm_max_tokens_per_item(
403
+ self,
404
+ seq_len: int,
405
+ mm_counts: Mapping[str, int],
406
+ ) -> Mapping[str, int]:
407
return {"image": self.get_num_image_tokens()}
408
409
def get_num_image_tokens(self) -> int:
vllm/model_executor/models/blip2.py
+5
-1
@@ -407,7 +407,11 @@ class Blip2ProcessingInfo(BaseProcessingInfo):
407
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
408
return {"image": 1}
409
410
- def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
410
+ def get_mm_max_tokens_per_item(
411
+ self,
412
+ seq_len: int,
413
+ mm_counts: Mapping[str, int],
414
+ ) -> Mapping[str, int]:
415
return {"image": self.get_num_image_tokens()}
416
417
def get_num_image_tokens(self) -> int:
vllm/model_executor/models/chameleon.py
+5
-1
@@ -64,7 +64,11 @@ class ChameleonProcessingInfo(BaseProcessingInfo):
64
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
65
return {"image": 1}
66
67
- def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
67
+ def get_mm_max_tokens_per_item(
68
+ self,
69
+ seq_len: int,
70
+ mm_counts: Mapping[str, int],
71
+ ) -> Mapping[str, int]:
72
return {"image": self.get_num_image_tokens()}
73
74
def get_num_image_tokens(self) -> int:
vllm/model_executor/models/deepseek_vl2.py
+5
-1
@@ -165,7 +165,11 @@ class DeepseekVL2ProcessingInfo(BaseProcessingInfo):
165
image_width=x[1], image_height=x[0]))
166
return ImageSize(width=width, height=height)
167
168
- def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
168
+ def get_mm_max_tokens_per_item(
169
+ self,
170
+ seq_len: int,
171
+ mm_counts: Mapping[str, int],
172
+ ) -> Mapping[str, int]:
173
max_image_size = self.get_image_size_with_most_features()
174
max_image_tokens = self.get_num_image_tokens(
175
image_height=max_image_size.height,
vllm/model_executor/models/fuyu.py
+5
-1
@@ -80,7 +80,11 @@ class FuyuProcessingInfo(BaseProcessingInfo):
80
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
81
return {"image": 1}
82
83
- def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
83
+ def get_mm_max_tokens_per_item(
84
+ self,
85
+ seq_len: int,
86
+ mm_counts: Mapping[str, int],
87
+ ) -> Mapping[str, int]:
88
target_width, target_height = self.get_image_size_with_most_features()
89
90
max_ncols, max_nrows = self.get_image_feature_grid_size(
vllm/model_executor/models/h2ovl.py
+386
-235
@@ -7,43 +7,55 @@
7
# Copyright (c) 2024 H2O.AI
8
# Licensed under Apache 2.0 License [see LICENSE for details]
9
# --------------------------------------------------------
10
-from functools import partial
11
-from typing import List, Optional, Tuple
10
+from typing import Mapping, Optional
11
12
import torch
13
from PIL import Image
14
from transformers import PretrainedConfig
15
17
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
18
- token_inputs)
16
+from vllm.logger import init_logger
17
from vllm.model_executor.layers.quantization import QuantizationConfig
20
-from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
21
-from vllm.multimodal.utils import cached_get_tokenizer
22
-from vllm.utils import is_list_of
18
+from vllm.multimodal import MULTIMODAL_REGISTRY
19
+from vllm.multimodal.inputs import MultiModalKwargs
20
+from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
21
+ MultiModalDataItems)
22
+from vllm.multimodal.processing import (ProcessingCache, PromptReplacement,
23
+ PromptReplacementDetails)
24
+from vllm.multimodal.profiling import BaseDummyInputsBuilder
25
+from vllm.transformers_utils.tokenizer import AnyTokenizer
26
27
from .intern_vit import InternVisionModel
25
-from .internvl import (IMG_CONTEXT, IMG_END, IMG_START, InternVLChatModel,
26
- InternVLInputPipeline, build_transform,
27
- find_closest_aspect_ratio, get_internvl_num_patches)
28
+from .internvl import (IMG_CONTEXT, IMG_END, IMG_START,
29
+ BaseInternVLProcessingInfo, BaseInternVLProcessor,
30
+ InternVLChatModel, InternVLDummyInputsBuilder,
31
+ InternVLMultiModalProcessor, build_transform,
32
+ find_closest_aspect_ratio, get_internvl_target_ratios)
33
34
+logger = init_logger(__name__)
35
30
-# modified to include blocks generated in second pass
31
-def calculate_num_blocks(
32
- orig_width: int,
33
- orig_height: int,
34
- min_num: int,
35
- max_num: int,
36
- image_size: int,
36
+
37
+def resolve_h2ovl_min_max_num(
38
+ *,
39
+ min_dynamic_patch: int,
40
+ max_dynamic_patch: int,
41
+ dynamic_image_size: bool,
42
use_thumbnail: bool,
38
- prior_aspect_ratio=None,
39
-) -> Tuple[int, int, int, Tuple[int, int]]:
40
- aspect_ratio = orig_width / orig_height
43
+) -> tuple[int, int]:
44
+ max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
45
+
46
+ if use_thumbnail and max_dynamic_patch != 1:
47
+ max_dynamic_patch += 1
48
42
- # calculate the existing image aspect ratio
43
- target_ratios = set((i, j) for n in range(min_num, max_num + 1)
44
- for i in range(1, n + 1) for j in range(1, n + 1)
45
- if i * j <= max_num and i * j >= min_num)
46
- target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
49
+ return min_dynamic_patch, max_dynamic_patch
50
+
51
+
52
+def get_h2ovl_target_ratios(
53
+ min_num: int,
54
+ max_num: int,
55
+ *,
56
+ prior_aspect_ratio: Optional[tuple[int, int]],
57
+) -> list[tuple[int, int]]:
58
+ target_ratios = get_internvl_target_ratios(min_num, max_num)
59
60
# if prior_aspect_ratio is provided, filter the target ratios
61
if prior_aspect_ratio is not None:
@@ -52,44 +64,66 @@ def calculate_num_blocks(
64
ratio[0] != 0 and prior_aspect_ratio[1] % ratio[1] != 0
65
]
66
67
+ return target_ratios
68
+
69
+
70
+# modified to include blocks generated in second pass
71
+def calculate_h2ovl_targets(
72
+ *,
73
+ orig_width: int,
74
+ orig_height: int,
75
+ target_ratios: list[tuple[int, int]],
76
+ image_size: int,
77
+ use_thumbnail: bool,
78
+) -> tuple[int, int, int, tuple[int, int]]:
79
+ aspect_ratio = orig_width / orig_height
80
+
81
# find the closest aspect ratio to the target
56
- target_aspect_ratio = find_closest_aspect_ratio(aspect_ratio,
57
- target_ratios, orig_width,
58
- orig_height, image_size)
82
+ target_aspect_ratio = find_closest_aspect_ratio(
83
+ aspect_ratio,
84
+ target_ratios,
85
+ width=orig_width,
86
+ height=orig_height,
87
+ image_size=image_size,
88
+ )
89
90
# calculate the target width and height
91
target_width = image_size * target_aspect_ratio[0]
92
target_height = image_size * target_aspect_ratio[1]
93
blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
64
- # add thumbnail image if num_blocks > 1
65
- if use_thumbnail and blocks > 1:
94
+
95
+ # add thumbnail image if num_blocks != 1
96
+ if use_thumbnail and blocks != 1:
97
blocks += 1
98
+
99
return blocks, target_width, target_height, target_aspect_ratio
100
101
102
# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
71
-# refactored to handle prior_aspect_ratio as optional
72
-def dynamic_preprocess(
103
+# refactored to handle prior_aspect_ratio
104
+def dynamic_preprocess_h2ovl(
105
image: Image.Image,
74
- min_num: int,
75
- max_num: int,
106
+ *,
107
+ target_ratios: list[tuple[int, int]],
108
image_size: int,
109
use_thumbnail: bool,
78
- prior_aspect_ratio: Optional[Tuple[int, int]] = None,
79
-) -> Tuple[List[Image.Image], Tuple[int, int]]:
110
+) -> tuple[list[Image.Image], tuple[int, int]]:
111
orig_width, orig_height = image.size
112
82
- # calculate the number of blocks based on prior aspect ratio if available
83
- blocks, target_width, target_height, target_aspect_ratio = (
84
- calculate_num_blocks(
85
- orig_width,
86
- orig_height,
87
- min_num,
88
- max_num,
89
- image_size,
90
- use_thumbnail=False,
91
- prior_aspect_ratio=prior_aspect_ratio,
92
- ))
113
+ # calculate the number of blocks without thumbnail
114
+ (
115
+ blocks,
116
+ target_width,
117
+ target_height,
118
+ target_aspect_ratio,
119
+ ) = calculate_h2ovl_targets(
120
+ orig_width=orig_width,
121
+ orig_height=orig_height,
122
+ target_ratios=target_ratios,
123
+ image_size=image_size,
124
+ use_thumbnail=False,
125
+ )
126
+
127
# resize the image
128
resized_img = image.resize((target_width, target_height))
129
processed_images = []
@@ -103,276 +137,393 @@ def dynamic_preprocess(
137
# split the image
138
split_img = resized_img.crop(box)
139
processed_images.append(split_img)
140
+
141
assert len(processed_images) == blocks
142
+
143
if use_thumbnail and len(processed_images) != 1:
144
thumbnail_img = image.resize((image_size, image_size))
145
processed_images.append(thumbnail_img)
146
+
147
return processed_images, target_aspect_ratio
148
149
113
-def load_image(
150
+def _preprocess_image(
151
image: Image.Image,
115
- input_size=448,
116
- min_num=1,
117
- max_num=6,
118
- use_thumbnail=True,
119
- prior_aspect_ratio: Optional[Tuple[int, int]] = None,
120
-) -> Tuple[torch.Tensor, Tuple[int, int]]:
152
+ *,
153
+ input_size: int,
154
+ min_num: int,
155
+ max_num: int,
156
+ use_thumbnail: bool,
157
+ prior_aspect_ratio: Optional[tuple[int, int]],
158
+) -> tuple[torch.Tensor, tuple[int, int]]:
159
+ target_ratios = get_h2ovl_target_ratios(
160
+ min_num,
161
+ max_num,
162
+ prior_aspect_ratio=prior_aspect_ratio,
163
+ )
164
+
165
transform = build_transform(input_size=input_size)
122
- images, target_aspect_ratio = dynamic_preprocess(
166
+ images, target_aspect_ratio = dynamic_preprocess_h2ovl(
167
image,
168
image_size=input_size,
169
use_thumbnail=use_thumbnail,
126
- min_num=min_num,
127
- max_num=max_num,
128
- prior_aspect_ratio=prior_aspect_ratio,
170
+ target_ratios=target_ratios,
171
)
130
- pixel_values = [transform(image) for image in images]
131
- pixel_values = torch.stack(pixel_values)
172
+
173
+ pixel_values = torch.stack([transform(image) for image in images])
174
return pixel_values, target_aspect_ratio
175
176
135
-# refactored to use the combined load_image function
136
-def image_to_pixel_values(
177
+# refactored to use the _preprocess_image function
178
+def image_to_pixel_values_h2ovl(
179
image: Image.Image,
180
+ *,
181
input_size: int,
182
min_num: int,
183
max_num: int,
184
use_thumbnail: bool,
142
- use_MSAC: bool,
185
+ use_msac: bool,
186
) -> torch.Tensor:
187
# when MSAC is turned on, we need to process the image twice
145
- if use_MSAC:
188
+ if use_msac:
189
# first pass
147
- pixel_values, target_aspect_ratio = load_image(
190
+ pixel_values1, aspect_ratio1 = _preprocess_image(
191
image,
192
input_size=input_size,
193
min_num=min_num,
194
max_num=max_num,
195
use_thumbnail=True,
196
+ prior_aspect_ratio=None,
197
)
198
# second pass
155
- pixel_values2, _ = load_image(
199
+ pixel_values2, _ = _preprocess_image(
200
image,
201
input_size=input_size,
158
- min_num=min_num,
202
+ min_num=3, # Hardcoded value
203
max_num=max_num,
160
- prior_aspect_ratio=target_aspect_ratio,
204
+ use_thumbnail=True,
205
+ prior_aspect_ratio=aspect_ratio1,
206
)
207
# combine pixel values
208
pixel_values = torch.cat(
164
- [pixel_values2[:-1], pixel_values[:-1], pixel_values2[-1:]], 0)
209
+ [pixel_values2[:-1], pixel_values1[:-1], pixel_values2[-1:]], 0)
210
211
else:
167
- pixel_values, _ = load_image(
212
+ pixel_values, _ = _preprocess_image(
213
image,
214
input_size=input_size,
215
min_num=min_num,
216
max_num=max_num,
217
use_thumbnail=use_thumbnail,
218
+ prior_aspect_ratio=None,
219
)
220
221
return pixel_values
222
223
178
-def image_to_pixel_values_wrapper(hf_config: PretrainedConfig,
179
- max_dynamic_patch: Optional[int] = None,
180
- use_MSAC: Optional[bool] = None):
181
- image_size = hf_config.vision_config.image_size
182
- min_num = hf_config.min_dynamic_patch
183
- if max_dynamic_patch is None:
184
- max_dynamic_patch = hf_config.max_dynamic_patch
185
- if use_MSAC is None:
186
- use_MSAC = hf_config.use_msac
187
- use_thumbnail = hf_config.use_thumbnail
188
- return partial(
189
- image_to_pixel_values,
190
- input_size=image_size,
191
- min_num=min_num,
192
- max_num=max_dynamic_patch,
193
- use_thumbnail=use_thumbnail,
194
- use_MSAC=use_MSAC,
195
- )
196
-
197
-
198
-def get_max_internvl_image_tokens(ctx: InputContext,
199
- *,
200
- max_dynamic_patch: Optional[int] = None):
201
- """
202
- Calculate the maximum number of tokens with/without MSAC and thumbnail
203
- """
204
- hf_config = ctx.get_hf_config()
205
- use_thumbnail = hf_config.use_thumbnail
206
- use_MSAC = hf_config.use_msac
224
+class H2OVLProcessor(BaseInternVLProcessor):
225
208
- if max_dynamic_patch is None:
209
- max_dynamic_patch = hf_config.max_dynamic_patch
226
+ def __init__(
227
+ self,
228
+ config: PretrainedConfig,
229
+ tokenizer: AnyTokenizer,
230
+ *,
231
+ max_dynamic_patch: Optional[int] = None,
232
+ dynamic_image_size: Optional[bool] = None,
233
+ use_msac: Optional[bool] = None,
234
+ ) -> None:
235
+ super().__init__(
236
+ config,
237
+ tokenizer,
238
+ max_dynamic_patch=max_dynamic_patch,
239
+ dynamic_image_size=dynamic_image_size,
240
+ )
241
211
- num_patches = get_internvl_num_patches(hf_config)
242
+ if use_msac is None:
243
+ use_msac = config.use_msac
244
+ assert isinstance(use_msac, bool)
245
213
- coefficient = 2 if use_MSAC else 1
214
- num_blocks = coefficient * max_dynamic_patch + (1 if use_thumbnail else 0)
246
+ self.use_msac = use_msac
247
216
- return num_blocks * num_patches
248
+ @property
249
+ def image_token_id(self) -> int:
250
+ return self.tokenizer.get_vocab()[IMG_CONTEXT]
251
252
+ def get_image_repl_features(
253
+ self,
254
+ feature_size: int,
255
+ num_patches: Optional[int],
256
+ ) -> str:
257
+ return IMG_CONTEXT * feature_size
258
219
-class H2OVLInputPipeline(InternVLInputPipeline):
220
- """
221
- Input pipeline for processing image and text data for the H2OVL model.
222
- """
259
+ def get_image_repl_full(
260
+ self,
261
+ feature_size: int,
262
+ num_patches: Optional[int],
263
+ ) -> str:
264
+ features = self.get_image_repl_features(feature_size, num_patches)
265
+ return IMG_START + features + IMG_END
266
224
- def input_processor(
267
+ def resolve_min_max_num(
268
self,
226
- ctx: InputContext,
227
- inputs: DecoderOnlyInputs,
269
*,
270
max_dynamic_patch: Optional[int] = None,
230
- ) -> DecoderOnlyInputs:
231
- # get multi_modal_data
232
- multi_modal_data = inputs.get("multi_modal_data")
233
- if multi_modal_data is None or "image" not in multi_modal_data:
234
- return inputs
235
-
236
- model_config = ctx.model_config
237
- hf_config = ctx.get_hf_config()
238
- use_MSAC = hf_config.use_msac
239
-
240
- image_data = multi_modal_data["image"]
241
- num_patches = get_internvl_num_patches(hf_config)
242
-
243
- image_pixel_values_mapper = image_to_pixel_values_wrapper(
244
- hf_config, max_dynamic_patch=max_dynamic_patch)
245
-
246
- # single image
247
- if isinstance(image_data, Image.Image):
248
- pixel_values = image_pixel_values_mapper(image_data,
249
- use_MSAC=use_MSAC)
250
- num_blocks = pixel_values.shape[0]
251
- image_feature_sizes = [num_blocks * num_patches]
252
- pixel_values = pixel_values.unsqueeze(0)
253
-
254
- # multi images
255
- elif is_list_of(image_data, Image.Image):
256
- # Do not use MSAC for multi images
257
- image_feature_sizes = []
258
- pixel_values = [
259
- image_pixel_values_mapper(image, use_MSAC=False)
260
- for image in image_data
261
- ]
262
- for pixel_value in pixel_values:
263
- num_blocks = pixel_value.shape[0]
264
- image_feature_sizes.append(num_blocks * num_patches)
265
-
266
- # image embeddings as input
267
- elif isinstance(image_data, torch.Tensor):
268
- _, image_feature_size, _ = image_data.shape
269
- image_feature_sizes = [image_feature_size]
270
- pixel_values = None
271
-
272
- # multi-image image embeddings
273
- elif is_list_of(image_data, torch.Tensor):
274
-
275
- image_feature_sizes = []
276
- for image_embed in image_data:
277
- _, image_feature_size, _ = image_embed.shape
278
- image_feature_sizes.append(image_feature_size)
279
- pixel_values = None
271
+ dynamic_image_size: Optional[bool] = None,
272
+ use_thumbnail: Optional[bool] = None,
273
+ ) -> tuple[int, int]:
274
+ min_dynamic_patch = self.min_dynamic_patch
275
+ max_dynamic_patch = (self.max_dynamic_patch if max_dynamic_patch
276
+ is None else max_dynamic_patch)
277
+ dynamic_image_size = (self.dynamic_image_size if dynamic_image_size
278
+ is None else dynamic_image_size)
279
+ use_thumbnail = (self.use_thumbnail
280
+ if use_thumbnail is None else use_thumbnail)
281
+
282
+ return resolve_h2ovl_min_max_num(
283
+ min_dynamic_patch=min_dynamic_patch,
284
+ max_dynamic_patch=max_dynamic_patch,
285
+ dynamic_image_size=dynamic_image_size,
286
+ use_thumbnail=use_thumbnail,
287
+ )
288
281
- else:
282
- raise TypeError(f"Invalid image type: {type(image_data)}")
289
+ def resolve_target_ratios(
290
+ self,
291
+ *,
292
+ max_dynamic_patch: Optional[int] = None,
293
+ dynamic_image_size: Optional[bool] = None,
294
+ use_thumbnail: Optional[bool] = None,
295
+ prior_aspect_ratio: Optional[tuple[int, int]] = None,
296
+ ) -> list[tuple[int, int]]:
297
+ min_num, max_num = self.resolve_min_max_num(
298
+ max_dynamic_patch=max_dynamic_patch,
299
+ dynamic_image_size=dynamic_image_size,
300
+ use_thumbnail=use_thumbnail,
301
+ )
302
+ if prior_aspect_ratio: # hardcoded value for second pass of use_msac
303
+ min_num = 3
304
284
- tokenizer = cached_get_tokenizer(
285
- model_config.tokenizer,
286
- trust_remote_code=model_config.trust_remote_code,
305
+ return get_h2ovl_target_ratios(
306
+ min_num,
307
+ max_num,
308
+ prior_aspect_ratio=prior_aspect_ratio,
309
)
310
289
- prompt = inputs.get("prompt")
290
- prompt_token_ids = inputs["prompt_token_ids"]
291
- if prompt is None:
292
- prompt = tokenizer.decode(prompt_token_ids)
293
-
294
- new_prompt = self._expand_image_prompt(prompt, image_feature_sizes,
295
- num_patches)
296
- new_prompt_token_ids = tokenizer.encode(new_prompt)
297
-
298
- # Wrap image processing in input_processor to avoid duplication
299
- image_token_id = tokenizer.encode(
300
- self.img_context_token,
301
- add_special_tokens=False,
302
- return_tensors="pt",
303
- )[0]
304
-
305
- # Update multi_modal_data to return
306
- if pixel_values is not None:
307
- multi_modal_data = {
308
- "image": {
309
- "pixel_values": pixel_values,
310
- "image_token_id": image_token_id,
311
- }
312
- }
311
+ def get_num_image_tokens(
312
+ self,
313
+ *,
314
+ image_width: int,
315
+ image_height: int,
316
+ use_msac: Optional[bool] = None,
317
+ ) -> int:
318
+ use_msac = (self.use_msac if use_msac is None else use_msac)
319
+
320
+ use_thumbnail = self.use_thumbnail
321
+
322
+ if use_msac:
323
+ target_ratios_1 = self.resolve_target_ratios(
324
+ use_thumbnail=False, # Applied in calculate_targets
325
+ )
326
+ num_patches_1, _, _, aspect_ratio_1 = calculate_h2ovl_targets(
327
+ orig_width=image_width,
328
+ orig_height=image_height,
329
+ image_size=self.image_size,
330
+ target_ratios=target_ratios_1,
331
+ use_thumbnail=True,
332
+ )
333
+
334
+ target_ratios_2 = self.resolve_target_ratios(
335
+ use_thumbnail=False, # Applied in calculate_targets
336
+ prior_aspect_ratio=aspect_ratio_1,
337
+ )
338
+ num_patches_2, _, _, _ = calculate_h2ovl_targets(
339
+ orig_width=image_width,
340
+ orig_height=image_height,
341
+ image_size=self.image_size,
342
+ target_ratios=target_ratios_2,
343
+ use_thumbnail=True,
344
+ )
345
+
346
+ num_patches = num_patches_1 + num_patches_2 - 1
347
else:
314
- multi_modal_data = {"image": {"image_embeds": image_data}}
348
+ target_ratios = self.resolve_target_ratios(
349
+ use_thumbnail=False, # Applied in calculate_targets
350
+ )
351
+ num_patches, _, _, _ = calculate_h2ovl_targets(
352
+ orig_width=image_width,
353
+ orig_height=image_height,
354
+ image_size=self.image_size,
355
+ target_ratios=target_ratios,
356
+ use_thumbnail=use_thumbnail,
357
+ )
358
+
359
+ return num_patches * self.num_image_token
360
316
- return token_inputs(
317
- prompt=prompt,
318
- prompt_token_ids=new_prompt_token_ids,
319
- multi_modal_data=multi_modal_data,
361
+ def _images_to_pixel_values_lst(
362
+ self,
363
+ images: list[Image.Image],
364
+ max_dynamic_patch: Optional[int] = None,
365
+ dynamic_image_size: Optional[bool] = None,
366
+ ) -> list[torch.Tensor]:
367
+ use_msac = self.use_msac if len(images) == 1 else False
368
+
369
+ min_num, max_num = self.resolve_min_max_num(
370
+ max_dynamic_patch=max_dynamic_patch,
371
+ dynamic_image_size=dynamic_image_size,
372
+ use_thumbnail=False, # Applied in image_to_pixel_values
373
)
374
322
- def input_mapper(
375
+ return [
376
+ image_to_pixel_values_h2ovl(
377
+ image,
378
+ input_size=self.image_size,
379
+ min_num=min_num,
380
+ max_num=max_num,
381
+ use_thumbnail=self.use_thumbnail,
382
+ use_msac=use_msac,
383
+ ) for image in images
384
+ ]
385
+
386
+
387
+class H2OVLProcessingInfo(BaseInternVLProcessingInfo):
388
+
389
+ def get_hf_processor(
390
self,
324
- ctx: InputContext,
325
- data: object,
391
*,
392
max_dynamic_patch: Optional[int] = None,
328
- ) -> MultiModalKwargs:
393
+ dynamic_image_size: Optional[bool] = None,
394
+ ) -> H2OVLProcessor:
395
+ return H2OVLProcessor(
396
+ self.get_hf_config(),
397
+ self.get_tokenizer(),
398
+ max_dynamic_patch=max_dynamic_patch,
399
+ dynamic_image_size=dynamic_image_size,
400
+ )
401
+
402
+ def get_mm_max_tokens_per_item(
403
+ self,
404
+ seq_len: int,
405
+ mm_counts: Mapping[str, int],
406
+ ) -> Mapping[str, int]:
407
+ max_tokens_one_image = self.get_max_image_tokens(use_msac=None)
408
+ if mm_counts.get("image", 0) <= 1:
409
+ max_tokens_per_image = max_tokens_one_image
410
+ else:
411
+ max_tokens_per_image = self.get_max_image_tokens(use_msac=False)
412
+
413
+ return {"image": max_tokens_per_image}
414
330
- # NOTE: Preprocessing for the image data is done in the
331
- # 'input_processor' function during actual inference.
332
- if isinstance(data, dict):
333
- return MultiModalKwargs(data)
415
+ def get_num_image_tokens(
416
+ self,
417
+ *,
418
+ image_width: int,
419
+ image_height: int,
420
+ processor: Optional[H2OVLProcessor],
421
+ use_msac: Optional[bool] = None,
422
+ ) -> int:
423
+ if processor is None:
424
+ processor = self.get_hf_processor()
425
+
426
+ return processor.get_num_image_tokens(
427
+ image_width=image_width,
428
+ image_height=image_height,
429
+ use_msac=use_msac,
430
+ )
431
335
- # The section below is only used with dummy data during
336
- # memory profiling.
337
- hf_config = ctx.get_hf_config()
432
+ def get_max_image_tokens(self, use_msac: Optional[bool] = None) -> int:
433
+ target_width, target_height = self.get_image_size_with_most_features()
434
339
- image_pixel_values_mapper = image_to_pixel_values_wrapper(
340
- hf_config, max_dynamic_patch)
435
+ return self.get_num_image_tokens(
436
+ image_width=target_width,
437
+ image_height=target_height,
438
+ processor=None,
439
+ use_msac=use_msac,
440
+ )
441
342
- if isinstance(data, Image.Image):
343
- pixel_values = image_pixel_values_mapper(data)
344
- pixel_values = pixel_values.unsqueeze(0)
442
346
- elif is_list_of(data, Image.Image):
347
- hf_config.use_msac = False
348
- pixel_values = [image_pixel_values_mapper(img) for img in data]
443
+class H2OVLMultiModalProcessor(InternVLMultiModalProcessor[H2OVLProcessingInfo]
444
+ ):
445
+
446
+ def __init__(self,
447
+ info: H2OVLProcessingInfo,
448
+ dummy_inputs: "BaseDummyInputsBuilder[H2OVLProcessingInfo]",
449
+ *,
450
+ cache: Optional[ProcessingCache] = None,
451
+ enable_sanity_checks: bool = True) -> None:
452
+ super().__init__(
453
+ info,
454
+ dummy_inputs,
455
+ cache=cache,
456
+ enable_sanity_checks=enable_sanity_checks,
457
+ )
458
+
459
+ if self.cache is not None:
460
+ # The processor output depends on the number of images passed,
461
+ # making it incompatible with processing cache which is supposed
462
+ # to be invariant of how many images are passed per prompt
463
+ self.cache = None
464
+ logger.warning_once(
465
+ f"{type(self).__name__} does not support processing cache.")
466
467
+ def _get_prompt_replacements(
468
+ self,
469
+ mm_items: MultiModalDataItems,
470
+ hf_processor_mm_kwargs: Mapping[str, object],
471
+ out_mm_kwargs: MultiModalKwargs,
472
+ ) -> list[PromptReplacement]:
473
+ hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
474
+
475
+ if "image_num_patches" in out_mm_kwargs:
476
+ image_num_patches = out_mm_kwargs["image_num_patches"]
477
+ assert isinstance(image_num_patches, torch.Tensor)
478
+ image_num_patches = image_num_patches.tolist()
479
+ elif "image_embeds" in out_mm_kwargs:
480
+ # TODO: Use image size information in dictionary embedding inputs
481
+ # to compute num_patches (similar to Qwen2-VL)
482
+ image_num_patches = [None] * len(out_mm_kwargs["image_embeds"])
483
else:
351
- return MultiModalKwargs({"image_embeds": data})
352
- model_config = ctx.model_config
353
- tokenizer = cached_get_tokenizer(
354
- model_config.tokenizer,
355
- trust_remote_code=model_config.trust_remote_code,
356
- )
357
- image_token_id = tokenizer.encode(
358
- self.img_context_token,
359
- add_special_tokens=False,
360
- return_tensors="pt",
361
- )[0]
484
+ image_num_patches = []
485
+
486
+ num_images = len(image_num_patches)
487
363
- return MultiModalKwargs({
364
- "pixel_values": pixel_values,
365
- "image_token_id": image_token_id
366
- })
488
+ def get_replacement_internvl(item_idx: int):
489
+ images = mm_items.get_items(
490
+ "image", (ImageEmbeddingItems, ImageProcessorItems))
491
492
+ if isinstance(images, ImageEmbeddingItems):
493
+ feature_size = images.get_feature_size(item_idx)
494
+ else:
495
+ image_size = images.get_image_size(item_idx)
496
+ feature_size = self.info.get_num_image_tokens(
497
+ image_width=image_size.width,
498
+ image_height=image_size.height,
499
+ processor=hf_processor,
500
+ use_msac=None if num_images == 1 else False,
501
+ )
502
+
503
+ num_patches = image_num_patches[item_idx]
504
+ if num_patches is not None:
505
+ assert isinstance(num_patches, int)
506
+
507
+ return PromptReplacementDetails(
508
+ full=hf_processor.get_image_repl_full(feature_size,
509
+ num_patches),
510
+ features=hf_processor.get_image_repl_features(
511
+ feature_size, num_patches),
512
+ )
513
369
-input_pipeline = H2OVLInputPipeline(IMG_START, IMG_END, IMG_CONTEXT)
514
+ return [
515
+ PromptReplacement(
516
+ modality="image",
517
+ target="<image>",
518
+ replacement=get_replacement_internvl,
519
+ )
520
+ ]
521
522
372
-@MULTIMODAL_REGISTRY.register_image_input_mapper(input_pipeline.input_mapper)
373
-@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_internvl_image_tokens)
374
-@INPUT_REGISTRY.register_dummy_data(input_pipeline.dummy_data)
375
-@INPUT_REGISTRY.register_input_processor(input_pipeline.input_processor)
523
+@MULTIMODAL_REGISTRY.register_processor(
524
+ H2OVLMultiModalProcessor,
525
+ info=H2OVLProcessingInfo,
526
+ dummy_inputs=InternVLDummyInputsBuilder)
527
class H2OVLChatModel(InternVLChatModel):
528
529
def _init_vision_model(
vllm/model_executor/models/internvl.py
+503
-320
@@ -6,35 +6,37 @@
6
# Copyright (c) 2023 OpenGVLab
7
# Licensed under The MIT License [see LICENSE for details]
8
# --------------------------------------------------------
9
-import re
10
-from functools import cached_property, partial
9
+from abc import ABC, abstractmethod
10
+from functools import cached_property
11
from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
12
- TypedDict, Union)
12
+ TypedDict, TypeVar, Union)
13
14
import torch
15
import torch.nn as nn
16
import torchvision.transforms as T
17
from PIL import Image
18
-from transformers import PretrainedConfig
18
+from transformers import BatchFeature, PretrainedConfig, TensorType
19
20
from vllm.attention import AttentionMetadata
21
from vllm.config import VllmConfig
22
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
23
- InputContext, token_inputs)
22
from vllm.model_executor.layers.quantization import QuantizationConfig
23
from vllm.model_executor.layers.quantization.awq import AWQConfig
24
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
25
from vllm.model_executor.models.intern_vit import (InternVisionModel,
26
InternVisionPatchModel)
27
from vllm.model_executor.sampling_metadata import SamplingMetadata
30
-from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
31
-from vllm.multimodal.inputs import NestedTensors, PlaceholderRange
32
-from vllm.multimodal.utils import cached_get_tokenizer
28
+from vllm.multimodal import MULTIMODAL_REGISTRY
29
+from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
30
+ NestedTensors)
31
+from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
32
+ ImageSize, MultiModalDataItems)
33
+from vllm.multimodal.processing import (BaseMultiModalProcessor,
34
+ BaseProcessingInfo, PromptReplacement,
35
+ PromptReplacementDetails)
36
+from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
37
from vllm.sequence import IntermediateTensors
34
-from vllm.utils import is_list_of
38
+from vllm.transformers_utils.tokenizer import AnyTokenizer
39
36
-from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
37
- get_clip_num_patches)
40
from .interfaces import SupportsMultiModal, SupportsPP
41
from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
42
maybe_prefix, merge_multimodal_embeddings)
@@ -75,22 +77,27 @@ InternVLImageInputs = Union[InternVLImagePixelInputs,
77
InternVLImageEmbeddingInputs]
78
79
78
-# copied from https://huggingface.co/OpenGVLab/InternVL2-1B
79
-def build_transform(input_size):
80
+# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
81
+def build_transform(input_size: int):
82
MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
81
- transform = T.Compose([
83
+ return T.Compose([
84
T.Lambda(lambda img: img.convert('RGB') if img.mode != 'RGB' else img),
85
T.Resize((input_size, input_size),
86
interpolation=T.InterpolationMode.BICUBIC),
87
T.ToTensor(),
88
T.Normalize(mean=MEAN, std=STD)
89
])
88
- return transform
90
91
91
-# copied from https://huggingface.co/OpenGVLab/InternVL2-1B
92
-def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height,
93
- image_size):
92
+# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
93
+def find_closest_aspect_ratio(
94
+ aspect_ratio: float,
95
+ target_ratios: list[tuple[int, int]],
96
+ *,
97
+ width: int,
98
+ height: int,
99
+ image_size: int,
100
+) -> tuple[int, int]:
101
best_ratio_diff = float('inf')
102
best_ratio = (1, 1)
103
area = width * height
@@ -106,67 +113,82 @@ def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height,
113
return best_ratio
114
115
109
-def calculate_num_blocks(orig_width: int, orig_height: int, min_num: int,
110
- max_num: int, image_size: int,
111
- use_thumbnail: bool) -> Tuple[int, int, int]:
112
- aspect_ratio = orig_width / orig_height
116
+def resolve_internvl_min_max_num(
117
+ *,
118
+ min_dynamic_patch: int,
119
+ max_dynamic_patch: int,
120
+ dynamic_image_size: bool,
121
+ use_thumbnail: bool,
122
+) -> tuple[int, int]:
123
+ max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
124
+
125
+ if use_thumbnail and max_dynamic_patch != 1:
126
+ max_dynamic_patch += 1
127
+
128
+ return min_dynamic_patch, max_dynamic_patch
129
+
130
114
- # calculate the existing image aspect ratio
115
- target_ratios = set((i, j) for n in range(min_num, max_num + 1)
116
- for i in range(1, n + 1) for j in range(1, n + 1)
117
- if i * j <= max_num and i * j >= min_num)
118
- target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
131
+def get_internvl_target_ratios(
132
+ min_num: int,
133
+ max_num: int,
134
+) -> list[tuple[int, int]]:
135
+ target_ratios = {(i, j)
136
+ for n in range(min_num, max_num + 1)
137
+ for i in range(1, n + 1)
138
+ for j in range(1, n + 1) if min_num <= i * j <= max_num}
139
+ return sorted(target_ratios, key=lambda x: x[0] * x[1])
140
+
141
+
142
+def calculate_internvl_targets(
143
+ *,
144
+ orig_width: int,
145
+ orig_height: int,
146
+ target_ratios: list[tuple[int, int]],
147
+ image_size: int,
148
+ use_thumbnail: bool,
149
+) -> tuple[int, int, int]:
150
+ aspect_ratio = orig_width / orig_height
151
152
# find the closest aspect ratio to the target
121
- target_aspect_ratio = find_closest_aspect_ratio(aspect_ratio,
122
- target_ratios, orig_width,
123
- orig_height, image_size)
153
+ target_aspect_ratio = find_closest_aspect_ratio(
154
+ aspect_ratio,
155
+ target_ratios,
156
+ width=orig_width,
157
+ height=orig_height,
158
+ image_size=image_size,
159
+ )
160
161
# calculate the target width and height
162
target_width = image_size * target_aspect_ratio[0]
163
target_height = image_size * target_aspect_ratio[1]
164
blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
129
- # add thumbnail image if num_blocks > 1
130
- if use_thumbnail and blocks > 1:
131
- blocks += 1
132
- return blocks, target_width, target_height
133
-
165
135
-def calculate_num_blocks_wrapper(
136
- hf_config: PretrainedConfig,
137
- max_dynamic_patch: Optional[int] = None,
138
- dynamic_image_size: Optional[bool] = None,
139
-):
140
- if dynamic_image_size is None:
141
- dynamic_image_size = hf_config.dynamic_image_size
166
+ # add thumbnail image if num_blocks != 1
167
+ if use_thumbnail and blocks != 1:
168
+ blocks += 1
169
143
- max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
144
- if max_dynamic_patch is None:
145
- max_dynamic_patch = hf_config.max_dynamic_patch
146
- min_num = hf_config.min_dynamic_patch
147
- image_size = hf_config.vision_config.image_size
148
- use_thumbnail = hf_config.use_thumbnail
149
- return partial(calculate_num_blocks,
150
- min_num=min_num,
151
- max_num=max_dynamic_patch,
152
- image_size=image_size,
153
- use_thumbnail=use_thumbnail)
170
+ return blocks, target_width, target_height
171
172
173
# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
157
-def dynamic_preprocess(image: Image.Image, min_num: int, max_num: int,
158
- image_size: int,
159
- use_thumbnail: bool) -> List[Image.Image]:
174
+def dynamic_preprocess_internvl(
175
+ image: Image.Image,
176
+ *,
177
+ target_ratios: list[tuple[int, int]],
178
+ image_size: int,
179
+ use_thumbnail: bool,
180
+) -> list[Image.Image]:
181
orig_width, orig_height = image.size
182
183
# calculate the number of blocks without thumbnail
163
- blocks, target_width, target_height = calculate_num_blocks(
164
- orig_width,
165
- orig_height,
166
- min_num,
167
- max_num,
168
- image_size,
169
- use_thumbnail=False)
184
+ blocks, target_width, target_height = calculate_internvl_targets(
185
+ orig_width=orig_width,
186
+ orig_height=orig_height,
187
+ target_ratios=target_ratios,
188
+ image_size=image_size,
189
+ use_thumbnail=False,
190
+ )
191
+
192
# resize the image
193
resized_img = image.resize((target_width, target_height))
194
processed_images = []
@@ -178,301 +200,463 @@ def dynamic_preprocess(image: Image.Image, min_num: int, max_num: int,
200
# split the image
201
split_img = resized_img.crop(box)
202
processed_images.append(split_img)
203
+
204
assert len(processed_images) == blocks
205
+
206
if use_thumbnail and len(processed_images) != 1:
207
thumbnail_img = image.resize((image_size, image_size))
208
processed_images.append(thumbnail_img)
209
+
210
return processed_images
211
212
213
# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
189
-def image_to_pixel_values(image: Image.Image, input_size: int, min_num: int,
190
- max_num: int, use_thumbnail: bool) -> torch.Tensor:
214
+def image_to_pixel_values_internvl(
215
+ image: Image.Image,
216
+ *,
217
+ input_size: int,
218
+ min_num: int,
219
+ max_num: int,
220
+ use_thumbnail: bool,
221
+) -> torch.Tensor:
222
+ target_ratios = get_internvl_target_ratios(min_num, max_num)
223
+
224
transform = build_transform(input_size=input_size)
192
- images = dynamic_preprocess(image,
193
- min_num=min_num,
194
- max_num=max_num,
195
- image_size=input_size,
196
- use_thumbnail=use_thumbnail)
197
- pixel_values = [transform(image) for image in images]
198
- pixel_values = torch.stack(pixel_values)
225
+ images = dynamic_preprocess_internvl(
226
+ image,
227
+ target_ratios=target_ratios,
228
+ image_size=input_size,
229
+ use_thumbnail=use_thumbnail,
230
+ )
231
+
232
+ pixel_values = torch.stack([transform(image) for image in images])
233
return pixel_values
234
235
202
-def image_to_pixel_values_wrapper(
203
- hf_config: PretrainedConfig,
204
- max_dynamic_patch: Optional[int] = None,
205
- dynamic_image_size: Optional[bool] = None,
206
-):
207
- image_size = hf_config.vision_config.image_size
208
- min_num = hf_config.min_dynamic_patch
209
- if dynamic_image_size is None:
210
- dynamic_image_size = hf_config.dynamic_image_size
236
+class BaseInternVLProcessor(ABC):
237
+ """
238
+ This model doesn't define its own HF processor,
239
+ so we implement our own one here.
240
212
- max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
213
- if max_dynamic_patch is None:
214
- max_dynamic_patch = hf_config.max_dynamic_patch
215
- use_thumbnail = hf_config.use_thumbnail
216
- return partial(image_to_pixel_values,
217
- input_size=image_size,
218
- min_num=min_num,
219
- max_num=max_dynamic_patch,
220
- use_thumbnail=use_thumbnail)
221
-
222
-
223
-def get_internvl_num_patches(hf_config: PretrainedConfig):
224
- vision_config = hf_config.vision_config
225
- downsample_ratio = hf_config.downsample_ratio
226
- image_size = vision_config.image_size
227
- patch_size = vision_config.patch_size
228
- return int(
229
- get_clip_num_patches(image_size=image_size, patch_size=patch_size) *
230
- (downsample_ratio**2))
231
-
232
-
233
-def get_max_internvl_image_tokens(
234
- ctx: InputContext,
235
- *,
236
- max_dynamic_patch: Optional[int] = None,
237
- dynamic_image_size: Optional[bool] = None,
238
-):
239
- hf_config = ctx.get_hf_config()
240
- if dynamic_image_size is None:
241
- dynamic_image_size = hf_config.dynamic_image_size
241
+ The code to insert image tokens is based on:
242
+ https://huggingface.co/OpenGVLab/InternVL2-1B/blob/main/modeling_internvl_chat.py#L252
243
+ """
244
243
- max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
244
- if max_dynamic_patch is None:
245
- max_dynamic_patch = hf_config.max_dynamic_patch
246
- use_thumbnail = hf_config.use_thumbnail
247
- if use_thumbnail and max_dynamic_patch > 1:
248
- max_dynamic_patch += 1
245
+ def __init__(
246
+ self,
247
+ config: PretrainedConfig,
248
+ tokenizer: AnyTokenizer,
249
+ *,
250
+ max_dynamic_patch: Optional[int] = None,
251
+ dynamic_image_size: Optional[bool] = None,
252
+ ) -> None:
253
+ super().__init__()
254
250
- num_patches = get_internvl_num_patches(hf_config)
251
- return num_patches * max_dynamic_patch
255
+ self.config = config
256
+ self.tokenizer = tokenizer
257
258
+ image_size: int = config.vision_config.image_size
259
+ patch_size: int = config.vision_config.patch_size
260
254
-def get_max_internvl_image_size(
255
- ctx: InputContext,
256
- *,
257
- max_dynamic_patch: Optional[int] = None,
258
- dynamic_image_size: Optional[bool] = None,
259
-):
260
- hf_config = ctx.get_hf_config()
261
- image_size = hf_config.vision_config.image_size
262
- if dynamic_image_size is None:
263
- dynamic_image_size = hf_config.dynamic_image_size
261
+ if dynamic_image_size is None:
262
+ dynamic_image_size = config.dynamic_image_size
263
+ assert isinstance(dynamic_image_size, bool)
264
265
- max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
266
- if max_dynamic_patch is None:
267
- max_dynamic_patch = hf_config.max_dynamic_patch
268
- use_thumbnail = hf_config.use_thumbnail
269
- if use_thumbnail and max_dynamic_patch > 1:
270
- max_dynamic_patch += 1
271
- width = image_size * max_dynamic_patch
272
- height = image_size
273
- return width, height
265
+ if max_dynamic_patch is None:
266
+ max_dynamic_patch = config.max_dynamic_patch
267
+ assert isinstance(max_dynamic_patch, int)
268
269
+ self.num_image_token = int(
270
+ (image_size // patch_size)**2 * (config.downsample_ratio**2))
271
+ self.image_size = image_size
272
+ self.min_dynamic_patch: int = config.min_dynamic_patch
273
+ self.max_dynamic_patch = max_dynamic_patch
274
+ self.dynamic_image_size = dynamic_image_size
275
+ self.use_thumbnail: bool = config.use_thumbnail
276
+
277
+ @property
278
+ @abstractmethod
279
+ def image_token_id(self) -> int:
280
+ raise NotImplementedError
281
+
282
+ @abstractmethod
283
+ def get_image_repl_features(
284
+ self,
285
+ feature_size: int,
286
+ num_patches: Optional[int],
287
+ ) -> str:
288
+ raise NotImplementedError
289
276
-class InternVLInputPipeline:
290
+ @abstractmethod
291
+ def get_image_repl_full(
292
+ self,
293
+ feature_size: int,
294
+ num_patches: Optional[int],
295
+ ) -> str:
296
+ raise NotImplementedError
297
278
- def __init__(
298
+ def resolve_min_max_num(
299
self,
280
- img_start_token: str,
281
- img_end_token: str,
282
- img_context_token: str,
283
- ) -> None:
284
- super().__init__()
300
+ *,
301
+ max_dynamic_patch: Optional[int] = None,
302
+ dynamic_image_size: Optional[bool] = None,
303
+ use_thumbnail: Optional[bool] = None,
304
+ ) -> tuple[int, int]:
305
+ min_dynamic_patch = self.min_dynamic_patch
306
+ max_dynamic_patch = (self.max_dynamic_patch if max_dynamic_patch
307
+ is None else max_dynamic_patch)
308
+ dynamic_image_size = (self.dynamic_image_size if dynamic_image_size
309
+ is None else dynamic_image_size)
310
+ use_thumbnail = (self.use_thumbnail
311
+ if use_thumbnail is None else use_thumbnail)
312
+
313
+ return resolve_internvl_min_max_num(
314
+ min_dynamic_patch=min_dynamic_patch,
315
+ max_dynamic_patch=max_dynamic_patch,
316
+ dynamic_image_size=dynamic_image_size,
317
+ use_thumbnail=use_thumbnail,
318
+ )
319
286
- self.img_start_token = img_start_token
287
- self.img_end_token = img_end_token
288
- self.img_context_token = img_context_token
320
+ def resolve_target_ratios(
321
+ self,
322
+ *,
323
+ max_dynamic_patch: Optional[int] = None,
324
+ dynamic_image_size: Optional[bool] = None,
325
+ use_thumbnail: Optional[bool] = None,
326
+ ) -> list[tuple[int, int]]:
327
+ min_num, max_num = self.resolve_min_max_num(
328
+ max_dynamic_patch=max_dynamic_patch,
329
+ dynamic_image_size=dynamic_image_size,
330
+ use_thumbnail=use_thumbnail,
331
+ )
332
290
- def _create_image_prompt(self, feature_size: int, num_patches: int) -> str:
291
- return (self.img_start_token + self.img_context_token * feature_size +
292
- self.img_end_token)
333
+ return get_internvl_target_ratios(min_num, max_num)
334
294
- def _expand_image_prompt(
335
+ def get_num_image_tokens(
336
self,
296
- prompt: str,
297
- feature_sizes: List[int],
298
- num_patches: int,
299
- ) -> str:
300
- image_idx = sorted(
301
- map(int, re.findall(r"Image-(\d+): <image>\n", prompt)))
337
+ *,
338
+ image_width: int,
339
+ image_height: int,
340
+ ) -> int:
341
+ target_ratios = self.resolve_target_ratios(
342
+ use_thumbnail=False, # Applied in calculate_targets
343
+ )
344
+
345
+ num_patches, _, _ = calculate_internvl_targets(
346
+ orig_width=image_width,
347
+ orig_height=image_height,
348
+ image_size=self.image_size,
349
+ target_ratios=target_ratios,
350
+ use_thumbnail=self.use_thumbnail,
351
+ )
352
303
- new_prompt = prompt
304
- for idx, feature_size in enumerate(feature_sizes, start=1):
305
- image_prompt = self._create_image_prompt(feature_size, num_patches)
306
- if not image_idx:
307
- image_prompt = f"Image-{idx}: {image_prompt}"
353
+ return num_patches * self.num_image_token
354
309
- new_prompt = new_prompt.replace('<image>', image_prompt, 1)
355
+ def _images_to_pixel_values_lst(
356
+ self,
357
+ images: list[Image.Image],
358
+ max_dynamic_patch: Optional[int] = None,
359
+ dynamic_image_size: Optional[bool] = None,
360
+ ) -> list[torch.Tensor]:
361
+ min_num, max_num = self.resolve_min_max_num(
362
+ max_dynamic_patch=max_dynamic_patch,
363
+ dynamic_image_size=dynamic_image_size,
364
+ use_thumbnail=False, # Applied in image_to_pixel_values
365
+ )
366
311
- return new_prompt
367
+ return [
368
+ image_to_pixel_values_internvl(
369
+ image,
370
+ input_size=self.image_size,
371
+ min_num=min_num,
372
+ max_num=max_num,
373
+ use_thumbnail=self.use_thumbnail,
374
+ ) for image in images
375
+ ]
376
313
- def input_processor(
377
+ def __call__(
378
self,
315
- ctx: InputContext,
316
- inputs: DecoderOnlyInputs,
317
- *,
379
+ text: Optional[Union[str, list[str]]] = None,
380
+ images: Optional[Union[Image.Image, list[Image.Image]]] = None,
381
max_dynamic_patch: Optional[int] = None,
382
dynamic_image_size: Optional[bool] = None,
320
- ) -> DecoderOnlyInputs:
321
- multi_modal_data = inputs.get("multi_modal_data")
322
- if multi_modal_data is None or "image" not in multi_modal_data:
323
- return inputs
324
-
325
- model_config = ctx.model_config
326
- hf_config = ctx.get_hf_config()
327
-
328
- image_data = multi_modal_data["image"]
329
- num_patches = get_internvl_num_patches(hf_config)
330
- num_blocks_calculator = calculate_num_blocks_wrapper(
331
- hf_config, max_dynamic_patch, dynamic_image_size)
332
- if isinstance(image_data, Image.Image):
333
- width, height = image_data.size
334
- num_blocks, _, _ = num_blocks_calculator(width, height)
335
- image_feature_sizes = [num_blocks * num_patches]
336
- elif is_list_of(image_data, Image.Image):
337
- image_feature_sizes = []
338
- for image in image_data:
339
- width, height = image.size
340
- num_blocks, _, _ = num_blocks_calculator(width, height)
341
- image_feature_sizes.append(num_blocks * num_patches)
342
- elif isinstance(image_data, torch.Tensor):
343
- num_images, image_feature_size, hidden_size = image_data.shape
344
- image_feature_sizes = [image_feature_size]
383
+ return_tensors: Optional[Union[str, TensorType]] = None,
384
+ ) -> BatchFeature:
385
+ if text is None:
386
+ text = []
387
+ if not isinstance(text, list):
388
+ text = [text]
389
+ if images is None:
390
+ images = []
391
+ if not isinstance(images, list):
392
+ images = [images]
393
+
394
+ if len(images) == 0:
395
+ image_inputs = {}
396
else:
346
- raise TypeError(f"Invalid image type: {type(image_data)}")
347
-
348
- tokenizer = cached_get_tokenizer(
349
- model_config.tokenizer,
350
- trust_remote_code=model_config.trust_remote_code)
351
-
352
- prompt = inputs.get("prompt")
353
- prompt_token_ids = inputs["prompt_token_ids"]
354
- if prompt is None:
355
- prompt = tokenizer.decode(prompt_token_ids)
356
-
357
- new_prompt = self._expand_image_prompt(prompt, image_feature_sizes,
358
- num_patches)
359
- new_prompt_token_ids = tokenizer.encode(new_prompt)
360
- img_context_token_id = tokenizer.encode(self.img_context_token,
361
- add_special_tokens=False)
362
- assert len(img_context_token_id) == 1, \
363
- (f"Invalid image token '{self.img_context_token}': A valid image "
364
- f"token encodes to a single token ID, got {img_context_token_id}.")
365
- img_context_token_id = img_context_token_id[0]
366
-
367
- # Get precise tracking of placeholder positions
368
- token_idx = image_idx = 0
369
- placeholder_ranges = []
370
- while token_idx < len(new_prompt_token_ids):
371
- if new_prompt_token_ids[token_idx] == img_context_token_id:
372
- curr_image_featue_size = image_feature_sizes[image_idx]
373
- placeholder_ranges.append(
374
- PlaceholderRange(offset=token_idx,
375
- length=curr_image_featue_size))
376
- image_idx += 1
377
- token_idx += curr_image_featue_size
378
- else:
379
- token_idx += 1
397
+ pixel_values_lst = self._images_to_pixel_values_lst(
398
+ images,
399
+ max_dynamic_patch=max_dynamic_patch,
400
+ dynamic_image_size=dynamic_image_size,
401
+ )
402
+ image_inputs = {
403
+ "pixel_values_flat": torch.cat(pixel_values_lst),
404
+ "image_num_patches": list(map(len, pixel_values_lst)),
405
+ }
406
+
407
+ for pixel_values in pixel_values_lst:
408
+ num_patches = pixel_values.shape[0]
409
+ feature_size = num_patches * self.num_image_token
410
+
411
+ image_repl = self.get_image_repl_full(feature_size,
412
+ num_patches)
413
+ text = [t.replace('<image>', image_repl, 1) for t in text]
414
+
415
+ text_inputs = self.tokenizer(text)
416
+
417
+ return BatchFeature(
418
+ {
419
+ **text_inputs,
420
+ **image_inputs,
421
+ },
422
+ tensor_type=return_tensors,
423
+ )
424
381
- return token_inputs(
382
- prompt=prompt,
383
- prompt_token_ids=new_prompt_token_ids,
384
- multi_modal_data=multi_modal_data,
385
- multi_modal_placeholders={"image": placeholder_ranges})
425
387
- def input_mapper(
426
+class InternVLProcessor(BaseInternVLProcessor):
427
+
428
+ @property
429
+ def image_token_id(self) -> int:
430
+ return self.tokenizer.get_vocab()[IMG_CONTEXT]
431
+
432
+ def get_image_repl_features(
433
+ self,
434
+ feature_size: int,
435
+ num_patches: Optional[int],
436
+ ) -> str:
437
+ return IMG_CONTEXT * feature_size
438
+
439
+ def get_image_repl_full(
440
+ self,
441
+ feature_size: int,
442
+ num_patches: Optional[int],
443
+ ) -> str:
444
+ features = self.get_image_repl_features(feature_size, num_patches)
445
+ return IMG_START + features + IMG_END
446
+
447
+
448
+class BaseInternVLProcessingInfo(BaseProcessingInfo):
449
+
450
+ @abstractmethod
451
+ def get_hf_processor(
452
self,
389
- ctx: InputContext,
390
- data: object,
453
*,
454
max_dynamic_patch: Optional[int] = None,
455
dynamic_image_size: Optional[bool] = None,
394
- ):
395
- hf_config = ctx.get_hf_config()
396
-
397
- image_pixel_values_mapper = image_to_pixel_values_wrapper(
398
- hf_config, max_dynamic_patch, dynamic_image_size)
399
- if isinstance(data, Image.Image):
400
- data = image_pixel_values_mapper(data)
401
- # Add an N dimension for number of images per prompt (currently 1).
402
- data = data.unsqueeze(0)
403
- elif is_list_of(data, Image.Image):
404
- # we can't stack here because images may have different num_patches
405
- data = [image_pixel_values_mapper(img) for img in data]
406
- else:
407
- return MultiModalKwargs({"image_embeds": data})
408
- model_config = ctx.model_config
409
- tokenizer = cached_get_tokenizer(
410
- model_config.tokenizer,
411
- trust_remote_code=model_config.trust_remote_code)
412
- image_token_id = tokenizer.encode(self.img_context_token,
413
- add_special_tokens=False,
414
- return_tensors="pt")[0]
415
-
416
- return MultiModalKwargs({
417
- "pixel_values": data,
418
- "image_token_id": image_token_id
419
- })
420
-
421
- def dummy_data(
456
+ ) -> BaseInternVLProcessor:
457
+ raise NotImplementedError
458
+
459
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
460
+ return {"image": None}
461
+
462
+ def get_mm_max_tokens_per_item(
463
self,
423
- ctx: InputContext,
464
seq_len: int,
465
mm_counts: Mapping[str, int],
466
+ ) -> Mapping[str, int]:
467
+ return {"image": self.get_max_image_tokens()}
468
+
469
+ def get_num_image_tokens(
470
+ self,
471
*,
427
- max_dynamic_patch: Optional[int] = None,
428
- dynamic_image_size: Optional[bool] = None,
429
- ):
430
- num_images = mm_counts["image"]
472
+ image_width: int,
473
+ image_height: int,
474
+ processor: Optional[BaseInternVLProcessor],
475
+ ) -> int:
476
+ if processor is None:
477
+ processor = self.get_hf_processor()
478
+
479
+ return processor.get_num_image_tokens(
480
+ image_width=image_width,
481
+ image_height=image_height,
482
+ )
483
432
- hf_config = ctx.get_hf_config()
484
+ def get_max_image_tokens(self) -> int:
485
+ target_width, target_height = self.get_image_size_with_most_features()
486
434
- image_feature_size = get_max_internvl_image_tokens(
435
- ctx,
436
- max_dynamic_patch=max_dynamic_patch,
437
- dynamic_image_size=dynamic_image_size,
487
+ return self.get_num_image_tokens(
488
+ image_width=target_width,
489
+ image_height=target_height,
490
+ processor=None,
491
)
439
- model_config = ctx.model_config
440
- tokenizer = cached_get_tokenizer(
441
- model_config.tokenizer,
442
- trust_remote_code=model_config.trust_remote_code)
443
-
444
- seq_data, ranges = dummy_seq_data_for_clip(
445
- hf_config.vision_config,
446
- seq_len,
447
- num_images,
448
- image_token_id=tokenizer.encode(self.img_context_token,
449
- add_special_tokens=False)[0],
450
- image_feature_size_override=image_feature_size,
492
+
493
+ def get_image_size_with_most_features(self) -> ImageSize:
494
+ processor = self.get_hf_processor()
495
+
496
+ base_size = processor.image_size
497
+ target_ratios = processor.resolve_target_ratios()
498
+
499
+ largest_feature_size, largest_feature_pinpoint = 0, None
500
+ for wr, hr in target_ratios:
501
+ width, height = base_size * wr, base_size * hr
502
+
503
+ feat_size = self.get_num_image_tokens(
504
+ image_width=width,
505
+ image_height=height,
506
+ processor=processor,
507
+ )
508
+ if feat_size > largest_feature_size:
509
+ largest_feature_size = feat_size
510
+ largest_feature_pinpoint = ImageSize(width=width,
511
+ height=height)
512
+
513
+ if largest_feature_size == 0 or largest_feature_pinpoint is None:
514
+ raise ValueError("Cannot have a largest feature size of 0!")
515
+
516
+ return largest_feature_pinpoint
517
+
518
+
519
+_I = TypeVar("_I", bound=BaseInternVLProcessingInfo)
520
+
521
+
522
+class InternVLDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
523
+
524
+ def get_dummy_processor_inputs(
525
+ self,
526
+ seq_len: int,
527
+ mm_counts: Mapping[str, int],
528
+ ) -> ProcessorInputs:
529
+ target_width, target_height = \
530
+ self.info.get_image_size_with_most_features()
531
+ num_images = mm_counts.get("image", 0)
532
+
533
+ mm_data = {
534
+ "image":
535
+ self._get_dummy_images(width=target_width,
536
+ height=target_height,
537
+ num_images=num_images)
538
+ }
539
+
540
+ return ProcessorInputs(
541
+ prompt_text="<image>" * num_images,
542
+ mm_data=mm_data,
543
)
544
453
- max_image_width, max_image_height = get_max_internvl_image_size(
454
- ctx,
455
- max_dynamic_patch=max_dynamic_patch,
456
- dynamic_image_size=dynamic_image_size,
545
+
546
+class InternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
547
+
548
+ def _call_hf_processor(
549
+ self,
550
+ prompt: str,
551
+ mm_data: Mapping[str, object],
552
+ mm_kwargs: Mapping[str, object],
553
+ ) -> BatchFeature:
554
+ processed_outputs = super()._call_hf_processor(
555
+ prompt=prompt,
556
+ mm_data=mm_data,
557
+ mm_kwargs=mm_kwargs,
558
)
559
459
- mm_data = dummy_image_for_clip(
460
- hf_config.vision_config,
461
- num_images,
462
- image_width_override=max_image_width,
463
- image_height_override=max_image_height,
560
+ image_token_id = self.info.get_hf_processor(**mm_kwargs).image_token_id
561
+ image_data = mm_data.get("images", [])
562
+ assert isinstance(image_data, list)
563
+
564
+ # Since there may be extra tokens in the feature placeholders,
565
+ # we need to pass the image token ID to the model to select the
566
+ # tokens to merge from the vision encoder outputs
567
+ processed_outputs["image_token_id"] = [image_token_id
568
+ ] * len(image_data)
569
+
570
+ return processed_outputs
571
+
572
+ def _get_mm_fields_config(
573
+ self,
574
+ hf_inputs: BatchFeature,
575
+ hf_processor_mm_kwargs: Mapping[str, object],
576
+ ) -> Mapping[str, MultiModalFieldConfig]:
577
+ image_num_patches = hf_inputs.get("image_num_patches", torch.empty(0))
578
+
579
+ return dict(
580
+ pixel_values_flat=MultiModalFieldConfig.flat_from_sizes(
581
+ "image", image_num_patches),
582
+ image_num_patches=MultiModalFieldConfig.batched("image"),
583
+ image_embeds=MultiModalFieldConfig.batched("image"),
584
+ image_token_id=MultiModalFieldConfig.batched("image"),
585
)
586
466
- return DummyData(seq_data, mm_data, ranges)
587
+ def _get_prompt_replacements(
588
+ self,
589
+ mm_items: MultiModalDataItems,
590
+ hf_processor_mm_kwargs: Mapping[str, object],
591
+ out_mm_kwargs: MultiModalKwargs,
592
+ ) -> list[PromptReplacement]:
593
+ hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
594
+
595
+ if "image_num_patches" in out_mm_kwargs:
596
+ image_num_patches = out_mm_kwargs["image_num_patches"]
597
+ assert isinstance(image_num_patches, torch.Tensor)
598
+ image_num_patches = image_num_patches.tolist()
599
+ elif "image_embeds" in out_mm_kwargs:
600
+ # TODO: Use image size information in dictionary embedding inputs
601
+ # to compute num_patches (similar to Qwen2-VL)
602
+ image_num_patches = [None] * len(out_mm_kwargs["image_embeds"])
603
+ else:
604
+ image_num_patches = []
605
+
606
+ def get_replacement_internvl(item_idx: int):
607
+ images = mm_items.get_items(
608
+ "image", (ImageEmbeddingItems, ImageProcessorItems))
609
+
610
+ if isinstance(images, ImageEmbeddingItems):
611
+ feature_size = images.get_feature_size(item_idx)
612
+ else:
613
+ image_size = images.get_image_size(item_idx)
614
+ feature_size = self.info.get_num_image_tokens(
615
+ image_width=image_size.width,
616
+ image_height=image_size.height,
617
+ processor=hf_processor,
618
+ )
619
+
620
+ num_patches = image_num_patches[item_idx]
621
+ if num_patches is not None:
622
+ assert isinstance(num_patches, int)
623
+
624
+ return PromptReplacementDetails(
625
+ full=hf_processor.get_image_repl_full(feature_size,
626
+ num_patches),
627
+ features=hf_processor.get_image_repl_features(
628
+ feature_size, num_patches),
629
+ )
630
631
+ return [
632
+ PromptReplacement(
633
+ modality="image",
634
+ target="<image>",
635
+ replacement=get_replacement_internvl,
636
+ )
637
+ ]
638
469
-input_pipeline = InternVLInputPipeline(IMG_START, IMG_END, IMG_CONTEXT)
639
640
+class InternVLProcessingInfo(BaseInternVLProcessingInfo):
641
472
-@MULTIMODAL_REGISTRY.register_image_input_mapper(input_pipeline.input_mapper)
473
-@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_internvl_image_tokens)
474
-@INPUT_REGISTRY.register_dummy_data(input_pipeline.dummy_data)
475
-@INPUT_REGISTRY.register_input_processor(input_pipeline.input_processor)
642
+ def get_hf_processor(
643
+ self,
644
+ *,
645
+ max_dynamic_patch: Optional[int] = None,
646
+ dynamic_image_size: Optional[bool] = None,
647
+ ) -> InternVLProcessor:
648
+ return InternVLProcessor(
649
+ self.get_hf_config(),
650
+ self.get_tokenizer(),
651
+ max_dynamic_patch=max_dynamic_patch,
652
+ dynamic_image_size=dynamic_image_size,
653
+ )
654
+
655
+
656
+@MULTIMODAL_REGISTRY.register_processor(
657
+ InternVLMultiModalProcessor,
658
+ info=InternVLProcessingInfo,
659
+ dummy_inputs=InternVLDummyInputsBuilder)
660
class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
661
662
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
@@ -621,11 +805,11 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
805
806
def _parse_and_validate_image_input(
807
self, **kwargs: object) -> Optional[InternVLImageInputs]:
624
- pixel_values = kwargs.pop("pixel_values", None)
625
- image_token_id = kwargs.pop("image_token_id", None)
808
+ pixel_values_flat = kwargs.pop("pixel_values_flat", None)
809
+ image_num_patches = kwargs.pop("image_num_patches", None)
810
image_embeds = kwargs.pop("image_embeds", None)
811
628
- if pixel_values is None and image_embeds is None:
812
+ if pixel_values_flat is None and image_embeds is None:
813
return None
814
815
if image_embeds is not None:
@@ -638,31 +822,30 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
822
data=flatten_bn(image_embeds),
823
)
824
641
- self.img_context_token_id = image_token_id[0]
825
+ image_token_id = kwargs["image_token_id"]
826
+ assert isinstance(image_token_id, torch.Tensor)
827
+ self.img_context_token_id = image_token_id.flatten().unique().item()
828
643
- if pixel_values is not None:
644
- if not isinstance(pixel_values, (torch.Tensor, list)):
829
+ if pixel_values_flat is not None:
830
+ if not isinstance(pixel_values_flat, (torch.Tensor, list)):
831
raise ValueError("Incorrect type of pixel values. "
646
- f"Got type: {type(pixel_values)}")
647
-
648
- patches_per_image = []
649
- for request_pixel_values in pixel_values:
650
- for image_pixel_values in request_pixel_values:
651
- patches_per_image.append(image_pixel_values.shape[0])
652
- # We need to flatten (B, N, P) to (B*N*P),
653
- # so we call flatten_bn twice.
832
+ f"Got type: {type(pixel_values_flat)}")
833
+
834
+ assert isinstance(image_num_patches, (torch.Tensor, list))
835
+
836
return InternVLImagePixelInputs(
837
type="pixel_values",
838
data=self._validate_pixel_values(
657
- flatten_bn(flatten_bn(pixel_values), concat=True)),
658
- patches_per_image=patches_per_image)
839
+ flatten_bn(pixel_values_flat, concat=True)),
840
+ patches_per_image=flatten_bn(image_num_patches,
841
+ concat=True).tolist())
842
843
raise AssertionError("This line should be unreachable.")
844
845
def _process_image_input(
846
self,
847
image_input: InternVLImageInputs,
665
- ) -> Tuple[torch.Tensor]:
848
+ ) -> tuple[torch.Tensor, ...]:
849
if image_input["type"] == "image_embeds":
850
return image_input["data"]
851
@@ -689,7 +872,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
872
image_embeds = image_embeds.split(image_feature_sizes)
873
return image_embeds
874
692
- def _set_visual_token_mask(self, input_ids: torch.Tensor) -> torch.Tensor:
875
+ def _set_visual_token_mask(self, input_ids: torch.Tensor) -> None:
876
if self.is_mono:
877
self.visual_token_mask = (
878
input_ids == self.img_context_token_id).reshape(-1, 1)
vllm/model_executor/models/llava.py
+5
-1
@@ -125,7 +125,11 @@ class BaseLlavaProcessingInfo(BaseProcessingInfo):
125
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
126
return {"image": None}
127
128
- def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
128
+ def get_mm_max_tokens_per_item(
129
+ self,
130
+ seq_len: int,
131
+ mm_counts: Mapping[str, int],
132
+ ) -> Mapping[str, int]:
133
return {"image": self.get_max_image_tokens()}
134
135
def _apply_feature_select_strategy(
vllm/model_executor/models/llava_next_video.py
+5
-1
@@ -62,7 +62,11 @@ class LlavaNextVideoProcessingInfo(BaseProcessingInfo):
62
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
63
return {"video": 1}
64
65
- def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
65
+ def get_mm_max_tokens_per_item(
66
+ self,
67
+ seq_len: int,
68
+ mm_counts: Mapping[str, int],
69
+ ) -> Mapping[str, int]:
70
target_width, target_height = self.get_image_size_with_most_features()
71
72
max_video_tokens = self.get_num_video_tokens(
vllm/model_executor/models/llava_onevision.py
+5
-1
@@ -103,7 +103,11 @@ class LlavaOnevisionProcessingInfo(LlavaNextProcessingInfo):
103
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
104
return {"image": None, "video": None}
105
106
- def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
106
+ def get_mm_max_tokens_per_item(
107
+ self,
108
+ seq_len: int,
109
+ mm_counts: Mapping[str, int],
110
+ ) -> Mapping[str, int]:
111
return {
112
"image": self.get_max_image_tokens(),
113
"video": self.get_max_video_tokens(seq_len),
vllm/model_executor/models/minicpmo.py
+13
-15
@@ -23,7 +23,6 @@
23
# limitations under the License.
24
"""Inference-only MiniCPM-O model compatible with HuggingFace weights."""
25
from functools import partial
26
-from itertools import accumulate
26
from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional, Set,
27
Tuple, TypedDict, Union)
28
@@ -138,11 +137,15 @@ class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):
137
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
138
return {"image": None, "video": None, "audio": None}
139
141
- def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
140
+ def get_mm_max_tokens_per_item(
141
+ self,
142
+ seq_len: int,
143
+ mm_counts: Mapping[str, int],
144
+ ) -> Mapping[str, int]:
145
return {
146
"image": self.get_max_image_tokens(),
147
"audio": self.get_max_audio_tokens(),
145
- "video": self.get_max_video_tokens(seq_len)
148
+ "video": self.get_max_video_tokens(seq_len),
149
}
150
151
def get_default_audio_pool_step(self) -> int:
@@ -369,23 +372,18 @@ class MiniCPMOMultiModalProcessor(
372
hf_inputs,
373
hf_processor_mm_kwargs: Mapping[str, object],
374
) -> Mapping[str, MultiModalFieldConfig]:
375
+ audio_num_slices = hf_inputs.get("audio_num_slices", torch.empty(0))
376
373
- def get_slices(num_slices: List[int]) -> List[int]:
374
- slice_indices = [0] + list(accumulate(num_slices))
375
- slices = [(slice_indices[i], slice_indices[i + 1])
376
- for i in range(len(num_slices))]
377
- return [slice(*slice_item) for slice_item in slices]
378
-
379
- audio_slices = get_slices(
380
- hf_inputs.get("audio_num_slices", torch.empty(0)))
377
return dict(
378
**super()._get_mm_fields_config(hf_inputs, hf_processor_mm_kwargs),
383
- audio_features=MultiModalFieldConfig.flat("audio", audio_slices),
384
- audio_feature_lens=MultiModalFieldConfig.flat(
385
- "audio", audio_slices),
379
+ audio_features=MultiModalFieldConfig.flat_from_sizes(
380
+ "audio", audio_num_slices),
381
+ audio_feature_lens=MultiModalFieldConfig.flat_from_sizes(
382
+ "audio", audio_num_slices),
383
audio_num_slices=MultiModalFieldConfig.batched("audio"),
384
audio_orders_in_mm_data=MultiModalFieldConfig.batched("audio"),
388
- audio_embeds=MultiModalFieldConfig.flat("audio", audio_slices))
385
+ audio_embeds=MultiModalFieldConfig.flat_from_sizes(
386
+ "audio", audio_num_slices))
387
388
389
class MultiModalProjector(nn.Module):
vllm/model_executor/models/minicpmv.py
+24
-26
@@ -26,7 +26,6 @@ import math
26
import re
27
from collections import Counter
28
from functools import cached_property, partial
29
-from itertools import accumulate
29
from typing import (Any, Callable, Dict, Iterable, List, Literal, Mapping,
30
Optional, Set, Tuple, TypedDict, Union)
31
@@ -365,7 +364,11 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
364
else:
365
return {"image": None}
366
368
- def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
367
+ def get_mm_max_tokens_per_item(
368
+ self,
369
+ seq_len: int,
370
+ mm_counts: Mapping[str, int],
371
+ ) -> Mapping[str, int]:
372
mm_max_tokens = {"image": self.get_max_image_tokens()}
373
if self.get_model_version() == (2, 6):
374
mm_max_tokens["video"] = self.get_max_video_tokens(seq_len)
@@ -761,30 +764,25 @@ class MiniCPMVMultiModalProcessor(
764
hf_inputs,
765
hf_processor_mm_kwargs: Mapping[str, object],
766
) -> Mapping[str, MultiModalFieldConfig]:
764
-
765
- def get_slices(num_slices: List[int]) -> List[int]:
766
- slice_indices = [0] + list(accumulate(num_slices))
767
- slices = [(slice_indices[i], slice_indices[i + 1])
768
- for i in range(len(num_slices))]
769
- return [slice(*slice_item) for slice_item in slices]
770
-
771
- image_slices = get_slices(
772
- hf_inputs.get("image_num_slices", torch.empty(0)))
773
- video_slices = get_slices(
774
- hf_inputs.get("video_num_slices", torch.empty(0)))
775
-
776
- return dict(
777
- pixel_values=MultiModalFieldConfig.flat("image", image_slices),
778
- image_sizes=MultiModalFieldConfig.batched("image"),
779
- tgt_sizes=MultiModalFieldConfig.flat("image", image_slices),
780
- image_num_slices=MultiModalFieldConfig.batched("image"),
781
- image_embeds=MultiModalFieldConfig.flat("image", image_slices),
782
- video_pixel_values=MultiModalFieldConfig.flat(
783
- "video", video_slices),
784
- video_image_sizes=MultiModalFieldConfig.batched("video"),
785
- video_tgt_sizes=MultiModalFieldConfig.flat("video", video_slices),
786
- video_embeds=MultiModalFieldConfig.flat("video", video_slices),
787
- video_num_slices=MultiModalFieldConfig.batched("video"))
767
+ image_num_slices = hf_inputs.get("image_num_slices", torch.empty(0))
768
+ video_num_slices = hf_inputs.get("video_num_slices", torch.empty(0))
769
+
770
+ return dict(pixel_values=MultiModalFieldConfig.flat_from_sizes(
771
+ "image", image_num_slices),
772
+ image_sizes=MultiModalFieldConfig.batched("image"),
773
+ tgt_sizes=MultiModalFieldConfig.flat_from_sizes(
774
+ "image", image_num_slices),
775
+ image_num_slices=MultiModalFieldConfig.batched("image"),
776
+ image_embeds=MultiModalFieldConfig.flat_from_sizes(
777
+ "image", image_num_slices),
778
+ video_pixel_values=MultiModalFieldConfig.flat_from_sizes(
779
+ "video", video_num_slices),
780
+ video_image_sizes=MultiModalFieldConfig.batched("video"),
781
+ video_tgt_sizes=MultiModalFieldConfig.flat_from_sizes(
782
+ "video", video_num_slices),
783
+ video_embeds=MultiModalFieldConfig.flat_from_sizes(
784
+ "video", video_num_slices),
785
+ video_num_slices=MultiModalFieldConfig.batched("video"))
786
787
def apply(
788
self,
vllm/model_executor/models/nvlm_d.py
+166
-20
@@ -6,44 +6,190 @@
6
# Copyright (c) 2024 NVIDIA
7
# Licensed under Apache 2.0 License [see LICENSE for details]
8
# --------------------------------------------------------
9
-from typing import Optional
9
+from typing import Mapping, Optional
10
11
+import torch
12
import torch.nn as nn
13
from transformers import PretrainedConfig
14
14
-from vllm.inputs import INPUT_REGISTRY
15
from vllm.model_executor.layers.quantization import QuantizationConfig
16
from vllm.multimodal import MULTIMODAL_REGISTRY
17
+from vllm.multimodal.inputs import MultiModalKwargs
18
+from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
19
+ MultiModalDataItems)
20
+from vllm.multimodal.processing import (PromptReplacement,
21
+ PromptReplacementDetails)
22
+from vllm.multimodal.profiling import ProcessorInputs
23
24
from .intern_vit import InternVisionModel
19
-from .internvl import (InternVLChatModel, InternVLInputPipeline,
20
- get_max_internvl_image_tokens)
25
+from .internvl import (BaseInternVLProcessingInfo, BaseInternVLProcessor,
26
+ InternVLChatModel, InternVLDummyInputsBuilder,
27
+ InternVLMultiModalProcessor)
28
22
-IMG_START = '<|vision_start|>'
23
-IMG_END = '<|vision_end|>'
24
-IMG_CONTEXT = '<|vision_pad|>'
29
+IMG_PAD = "<|vision_pad|>"
30
31
27
-class NVLMInputPipeline(InternVLInputPipeline):
32
+class NVLMProcessor(BaseInternVLProcessor):
33
+
34
+ @property
35
+ def image_token_id(self) -> int:
36
+ return self.tokenizer.get_vocab()[IMG_PAD]
37
+
38
+ def get_image_repl_features(
39
+ self,
40
+ feature_size: int,
41
+ num_patches: Optional[int],
42
+ ) -> str:
43
+ if num_patches is None:
44
+ raise NotImplementedError("Embedding inputs are not supported")
45
+
46
+ tile_pos_identifiers = [f"<tile_{i}>" for i in range(1, num_patches)]
47
+ if self.use_thumbnail and num_patches != 1:
48
+ tile_pos_identifiers += ["<tile_global_thumbnail>"]
49
29
- def _create_image_prompt(self, feature_size: int, num_patches: int) -> str:
30
- tile_pos_identifiers = ([f"<tile_{i}>"
31
- for i in range(1, num_patches)] +
32
- ["<tile_global_thumbnail>"])
50
context_size = feature_size // num_patches
51
+ features = "".join(identifier + IMG_PAD * context_size
52
+ for identifier in tile_pos_identifiers)
53
+
54
+ # We include the start and end as well because "<Image><tile" is
55
+ # tokenized as ["<Image", "><", "tile"], resulting in assertion error
56
+ # when trying to find "<tile" as a subsequence of "<Image><tile"
57
+ return "<Image>" + features + "</Image>"
58
+
59
+ def get_image_repl_full(
60
+ self,
61
+ feature_size: int,
62
+ num_patches: Optional[int],
63
+ ) -> str:
64
+ return self.get_image_repl_features(feature_size, num_patches)
65
+
66
+
67
+class NVLMProcessingInfo(BaseInternVLProcessingInfo):
68
+
69
+ def get_hf_processor(
70
+ self,
71
+ *,
72
+ max_dynamic_patch: Optional[int] = None,
73
+ dynamic_image_size: Optional[bool] = None,
74
+ ) -> NVLMProcessor:
75
+ return NVLMProcessor(
76
+ self.get_hf_config(),
77
+ self.get_tokenizer(),
78
+ max_dynamic_patch=max_dynamic_patch,
79
+ dynamic_image_size=dynamic_image_size,
80
+ )
81
+
82
+ def get_max_image_tokens(self) -> int:
83
+ hf_processor = self.get_hf_processor()
84
+ tokenizer = hf_processor.tokenizer
85
+
86
+ max_num_patches = hf_processor.max_dynamic_patch
87
+ # we need +1 here because max_dynamic_patch in config doesn't
88
+ # include the thumbnail patch
89
+ tile_pos_identifiers = [
90
+ f"<tile_{i+1}>" for i in range(max_num_patches)
91
+ ]
92
+ if hf_processor.use_thumbnail and max_num_patches != 1:
93
+ tile_pos_identifiers += ["<tile_global_thumbnail>"]
94
+
95
+ # "<Image><tile" is tokenized as ["<Image", "><", "tile"]
96
+ # so we include <tile_1> in the start_str
97
+ start_str = "<Image>" + tile_pos_identifiers.pop(0)
98
+ end_str = "</Image>"
99
+ start_token_len = len(tokenizer.encode(start_str))
100
+ end_token_len = len(tokenizer.encode(end_str))
101
+ tile_token_len = sum(
102
+ len(tokenizer.encode(identifier))
103
+ for identifier in tile_pos_identifiers)
104
+ non_image_tokens_num = start_token_len + end_token_len + tile_token_len
105
+ return super().get_max_image_tokens() + non_image_tokens_num
106
+
107
+
108
+class NVLMDummyInputsBuilder(InternVLDummyInputsBuilder[NVLMProcessingInfo]):
109
+
110
+ def get_dummy_processor_inputs(
111
+ self,
112
+ seq_len: int,
113
+ mm_counts: Mapping[str, int],
114
+ ) -> ProcessorInputs:
115
+ target_width, target_height = \
116
+ self.info.get_image_size_with_most_features()
117
+ num_images = mm_counts.get("image", 0)
118
+
119
+ mm_data = {
120
+ "image":
121
+ self._get_dummy_images(width=target_width,
122
+ height=target_height,
123
+ num_images=num_images)
124
+ }
125
+
126
+ return ProcessorInputs(
127
+ # The newline is necessary to separate ">" of the current item
128
+ # and "<" of the next item
129
+ prompt_text="<image>\n" * num_images,
130
+ mm_data=mm_data,
131
+ )
132
+
133
+
134
+class NVLMMultiModalProcessor(InternVLMultiModalProcessor[NVLMProcessingInfo]):
135
35
- return '<Image>' + ''.join(
36
- tile_pos_identifier + self.img_context_token * context_size
37
- for tile_pos_identifier in tile_pos_identifiers) + '</Image>'
136
+ def _get_prompt_replacements(
137
+ self,
138
+ mm_items: MultiModalDataItems,
139
+ hf_processor_mm_kwargs: Mapping[str, object],
140
+ out_mm_kwargs: MultiModalKwargs,
141
+ ) -> list[PromptReplacement]:
142
+ hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
143
+
144
+ if "image_num_patches" in out_mm_kwargs:
145
+ image_num_patches = out_mm_kwargs["image_num_patches"]
146
+ assert isinstance(image_num_patches, torch.Tensor)
147
+ image_num_patches = image_num_patches.tolist()
148
+ elif "image_embeds" in out_mm_kwargs:
149
+ # TODO: Use image size information in dictionary embedding inputs
150
+ # to compute num_patches (similar to Qwen2-VL)
151
+ image_num_patches = [None] * len(out_mm_kwargs["image_embeds"])
152
+ else:
153
+ image_num_patches = []
154
+
155
+ def get_replacement_nvlm(item_idx: int):
156
+ images = mm_items.get_items(
157
+ "image", (ImageEmbeddingItems, ImageProcessorItems))
158
159
+ if isinstance(images, ImageEmbeddingItems):
160
+ feature_size = images.get_feature_size(item_idx)
161
+ else:
162
+ image_size = images.get_image_size(item_idx)
163
+ feature_size = self.info.get_num_image_tokens(
164
+ image_width=image_size.width,
165
+ image_height=image_size.height,
166
+ processor=hf_processor,
167
+ )
168
+
169
+ num_patches = image_num_patches[item_idx]
170
+ if num_patches is not None:
171
+ assert isinstance(num_patches, int)
172
+
173
+ return PromptReplacementDetails(
174
+ full=hf_processor.get_image_repl_full(feature_size,
175
+ num_patches) + "\n",
176
+ features=hf_processor.get_image_repl_features(
177
+ feature_size, num_patches) + "\n",
178
+ )
179
40
-input_pipeline = NVLMInputPipeline(IMG_START, IMG_END, IMG_CONTEXT)
180
+ # See note in dummy data regarding why we have the extra newline
181
+ return [
182
+ PromptReplacement(
183
+ modality="image",
184
+ target="<image>\n",
185
+ replacement=get_replacement_nvlm,
186
+ )
187
+ ]
188
189
43
-@MULTIMODAL_REGISTRY.register_image_input_mapper(input_pipeline.input_mapper)
44
-@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_internvl_image_tokens)
45
-@INPUT_REGISTRY.register_dummy_data(input_pipeline.dummy_data)
46
-@INPUT_REGISTRY.register_input_processor(input_pipeline.input_processor)
190
+@MULTIMODAL_REGISTRY.register_processor(NVLMMultiModalProcessor,
191
+ info=NVLMProcessingInfo,
192
+ dummy_inputs=NVLMDummyInputsBuilder)
193
class NVLM_D_Model(InternVLChatModel):
194
195
def _init_mlp1(self, config: PretrainedConfig) -> nn.Sequential:
vllm/model_executor/models/phi3v.py
+5
-1
@@ -322,7 +322,11 @@ class Phi3VProcessingInfo(BaseProcessingInfo):
322
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
323
return {"image": None}
324
325
- def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
325
+ def get_mm_max_tokens_per_item(
326
+ self,
327
+ seq_len: int,
328
+ mm_counts: Mapping[str, int],
329
+ ) -> Mapping[str, int]:
330
target_width, target_height = self.get_image_size_with_most_features()
331
332
max_image_tokens = self.get_num_image_tokens(
vllm/model_executor/models/qwen.py
+8
-4
@@ -779,7 +779,11 @@ class QWenVLProcessingInfo(BaseProcessingInfo):
779
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
780
return {"image": None}
781
782
- def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
782
+ def get_mm_max_tokens_per_item(
783
+ self,
784
+ seq_len: int,
785
+ mm_counts: Mapping[str, int],
786
+ ) -> Mapping[str, int]:
787
return {"image": self.get_num_image_tokens()}
788
789
def get_num_image_tokens(self) -> int:
@@ -799,13 +803,13 @@ class QWenVLDummyInputsBuilder(BaseDummyInputsBuilder[QWenVLProcessingInfo]):
803
804
vision_config = hf_config.visual
805
802
- max_image_size = vision_config["image_size"]
806
+ target_width = target_height = vision_config["image_size"]
807
num_images = mm_counts.get("image", 0)
808
809
mm_data = {
810
"image":
807
- self._get_dummy_images(width=max_image_size,
808
- height=max_image_size,
811
+ self._get_dummy_images(width=target_width,
812
+ height=target_height,
813
num_images=num_images)
814
}
815
vllm/model_executor/models/qwen2_audio.py
+5
-1
@@ -110,7 +110,11 @@ class Qwen2AudioProcessingInfo(BaseProcessingInfo):
110
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
111
return {"audio": None}
112
113
- def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
113
+ def get_mm_max_tokens_per_item(
114
+ self,
115
+ seq_len: int,
116
+ mm_counts: Mapping[str, int],
117
+ ) -> Mapping[str, int]:
118
hf_config = self.get_hf_config()
119
max_source_positions = hf_config.audio_config.max_source_positions
120
max_output_lengths = (max_source_positions - 2) // 2 + 1
vllm/model_executor/models/qwen2_vl.py
+15
-16
@@ -758,7 +758,11 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
758
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
759
return {"image": None, "video": None}
760
761
- def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
761
+ def get_mm_max_tokens_per_item(
762
+ self,
763
+ seq_len: int,
764
+ mm_counts: Mapping[str, int],
765
+ ) -> Mapping[str, int]:
766
return {
767
"image": self.get_max_image_tokens(),
768
"video": self.get_max_video_tokens(seq_len),
@@ -989,26 +993,21 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo]
993
hf_processor_mm_kwargs: Mapping[str, object],
994
) -> Mapping[str, MultiModalFieldConfig]:
995
image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3)))
992
- image_slice_idxs = [0] + image_grid_thw.prod(-1).cumsum_(0).tolist()
993
- image_slices = [
994
- slice(image_slice_idxs[i], image_slice_idxs[i + 1])
995
- for i in range(len(image_grid_thw))
996
- ]
996
+ image_grid_sizes = image_grid_thw.prod(-1)
997
998
video_grid_thw = hf_inputs.get("video_grid_thw", torch.empty((0, 3)))
999
- video_slice_idxs = [0] + video_grid_thw.prod(-1).cumsum_(0).tolist()
1000
- video_slices = [
1001
- slice(video_slice_idxs[i], video_slice_idxs[i + 1])
1002
- for i in range(len(video_grid_thw))
1003
- ]
999
+ video_grid_sizes = video_grid_thw.prod(-1)
1000
1001
return dict(
1006
- pixel_values=MultiModalFieldConfig.flat("image", image_slices),
1007
- image_embeds=MultiModalFieldConfig.flat("image", image_slices),
1002
+ pixel_values=MultiModalFieldConfig.flat_from_sizes(
1003
+ "image", image_grid_sizes),
1004
+ image_embeds=MultiModalFieldConfig.flat_from_sizes(
1005
+ "image", image_grid_sizes),
1006
image_grid_thw=MultiModalFieldConfig.batched("image"),
1009
- pixel_values_videos=MultiModalFieldConfig.flat(
1010
- "video", video_slices),
1011
- video_embeds=MultiModalFieldConfig.flat("video", video_slices),
1007
+ pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
1008
+ "video", video_grid_sizes),
1009
+ video_embeds=MultiModalFieldConfig.flat_from_sizes(
1010
+ "video", video_grid_sizes),
1011
video_grid_thw=MultiModalFieldConfig.batched("video"),
1012
)
1013
vllm/model_executor/models/ultravox.py
+5
-1
@@ -92,7 +92,11 @@ class UltravoxProcessingInfo(BaseProcessingInfo):
92
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
93
return {"audio": None}
94
95
- def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
95
+ def get_mm_max_tokens_per_item(
96
+ self,
97
+ seq_len: int,
98
+ mm_counts: Mapping[str, int],
99
+ ) -> Mapping[str, int]:
100
feature_extractor = self.get_feature_extractor()
101
max_audio_tokens = math.ceil(feature_extractor.chunk_length *
102
_AUDIO_TOKENS_PER_SECOND)
vllm/multimodal/inputs.py
+11
@@ -4,6 +4,7 @@ from abc import ABC, abstractmethod
4
from collections import UserDict, defaultdict
5
from collections.abc import Mapping, Sequence
6
from dataclasses import dataclass
7
+from itertools import accumulate
8
from typing import (TYPE_CHECKING, Any, Literal, Optional, TypedDict, TypeVar,
9
Union, cast, final)
10
@@ -258,6 +259,16 @@ class MultiModalFieldConfig:
259
slices=slices,
260
)
261
262
+ @staticmethod
263
+ def flat_from_sizes(modality: str, size_per_item: torch.Tensor):
264
+ slice_idxs = [0, *accumulate(size_per_item)]
265
+ slices = [
266
+ slice(slice_idxs[i], slice_idxs[i + 1])
267
+ for i in range(len(size_per_item))
268
+ ]
269
+
270
+ return MultiModalFieldConfig.flat(modality, slices)
271
+
272
def __init__(
273
self,
274
field_cls: type[BaseMultiModalField],
vllm/multimodal/processing.py
+5
-1
@@ -680,7 +680,11 @@ class BaseProcessingInfo:
680
raise NotImplementedError
681
682
@abstractmethod
683
- def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
683
+ def get_mm_max_tokens_per_item(
684
+ self,
685
+ seq_len: int,
686
+ mm_counts: Mapping[str, int],
687
+ ) -> Mapping[str, int]:
688
"""
689
Get the maximum possible number of tokens per data item
690
for each modality.
vllm/multimodal/profiling.py
+2
-1
@@ -151,7 +151,8 @@ class MultiModalProfiler(Generic[_I]):
151
mm_counts = self.get_mm_limits()
152
153
info = self.processing_info
154
- mm_max_tokens_per_item = info.get_mm_max_tokens_per_item(seq_len)
154
+ mm_max_tokens_per_item = info.get_mm_max_tokens_per_item(
155
+ seq_len, mm_counts)
156
157
if mm_counts.keys() != mm_max_tokens_per_item.keys():
158
raise AssertionError(
vllm/multimodal/registry.py
+3
-1
@@ -264,7 +264,9 @@ class MultiModalRegistry:
264
)
265
processor = self.create_processor(model_config, tokenizer)
266
seq_len = model_config.max_model_len
267
- return processor.info.get_mm_max_tokens_per_item(seq_len)
267
+ mm_limits = self.get_mm_limits_per_prompt(model_config)
268
+ return processor.info.get_mm_max_tokens_per_item(
269
+ seq_len, mm_limits)
270
271
return {
272
key: plugin.get_max_multimodal_tokens(model_config)