@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d1ca7df84d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[VLM] Merged multi-modal processor for InternVL-based models (#12553)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <2037008807@qq.com>

Cyrus Leung committed Feb 4, 2025 at 16:44 UTC d1ca7df84d9f8853001bdf1c2900321d9cb5d64e
34 files changed +1434 -986
docs/source/contributing/model/multimodal.md
+5 -1
@@ -250,7 +250,11 @@ def get_max_image_tokens(self) -> int:
250 And thus, we can override the method as:
251
252 ```python
253 -def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
253 +def get_mm_max_tokens_per_item(
254 + self,
255 + seq_len: int,
256 + mm_counts: Mapping[str, int],
257 +) -> Mapping[str, int]:
258 return {"image": self.get_max_image_tokens()}
259 ```
260
docs/source/models/supported_models.md
+7 -3
@@ -726,7 +726,7 @@ See [this page](#generative-models) for more information on how to use generativ
726 * `h2oai/h2ovl-mississippi-800m`, `h2oai/h2ovl-mississippi-2b`, etc.
727 *
728 * ✅︎
729 - *
729 + * \*
730 - * `Idefics3ForConditionalGeneration`
731 * Idefics3
732 * T + I
@@ -799,7 +799,7 @@ See [this page](#generative-models) for more information on how to use generativ
799 * ✅︎
800 - * `NVLM_D_Model`
801 * NVLM-D 1.0
802 - * T + I<sup>E+</sup>
802 + * T + I<sup>+</sup>
803 * `nvidia/NVLM-D-72B`, etc.
804 *
805 * ✅︎
@@ -859,7 +859,11 @@ See [this page](#generative-models) for more information on how to use generativ
859 <sup>+</sup> Multiple items can be inputted per text prompt for this modality.
860
861 :::{note}
862 -To use `DeepSeek-VL2` series models, you have to pass `--hf_overrides '{"architectures": ["DeepseekVLV2ForCausalLM"]}'` when running vLLM.
862 +To use DeepSeek-VL2 series models, you have to pass `--hf_overrides '{"architectures": ["DeepseekVLV2ForCausalLM"]}'` when running vLLM.
863 +:::
864 +
865 +:::{note}
866 +H2O-VL series models will be available in V1 once we support backends other than FlashAttention.
867 :::
868
869 :::{note}
tests/models/decoder_only/vision_language/test_h2ovl.py deleted
-131
@@ -1,131 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -
3 -from typing import Optional, Tuple
4 -
5 -import pytest
6 -import torch
7 -from PIL.Image import Image
8 -from transformers import AutoConfig
9 -
10 -# Import the functions to test
11 -from vllm.model_executor.models.h2ovl import (calculate_num_blocks,
12 - image_to_pixel_values_wrapper)
13 -from vllm.multimodal.image import rescale_image_size
14 -
15 -models = [
16 - "h2oai/h2ovl-mississippi-800m", # Replace with your actual model names
17 - "h2oai/h2ovl-mississippi-2b",
18 -]
19 -
20 -
21 -def run_preprocessing_test(
22 - image: Image,
23 - config,
24 - max_dynamic_patch: Optional[int] = None,
25 -) -> Tuple[torch.Tensor, int]:
26 - """Test the image preprocessing and calculate expected blocks."""
27 -
28 - if max_dynamic_patch is None:
29 - max_dynamic_patch = config.max_dynamic_patch
30 -
31 - width, height = image.size
32 - use_MSAC = config.use_msac
33 -
34 - # Create the mapper function with the provided configuration
35 - mapper = image_to_pixel_values_wrapper(config, max_dynamic_patch, use_MSAC)
36 - pixel_values = mapper(image)
37 -
38 - # Calculate the expected number of blocks
39 - if use_MSAC:
40 - # First pass
41 - blocks1, _, _, aspect_ratio = calculate_num_blocks(
42 - width,
43 - height,
44 - config.min_dynamic_patch,
45 - max_dynamic_patch,
46 - config.vision_config.image_size,
47 - use_thumbnail=False, # Thumbnail is handled separately
48 - prior_aspect_ratio=None,
49 - )
50 -
51 - # Second pass
52 - blocks2, _, _, _ = calculate_num_blocks(
53 - width,
54 - height,
55 - config.min_dynamic_patch,
56 - max_dynamic_patch,
57 - config.vision_config.image_size,
58 - use_thumbnail=False,
59 - prior_aspect_ratio=aspect_ratio,
60 - )
61 -
62 - # Add thumbnail if use_thumbnail is True and total_blocks > 1
63 - if config.use_thumbnail:
64 - blocks1 += 1 if blocks1 > 1 else 0
65 - blocks2 += 1 if blocks2 > 1 else 0
66 -
67 - # Total blocks is the sum of blocks from both passes minus overlapping
68 - total_blocks = blocks1 + blocks2 - 1
69 -
70 - expected_blocks = total_blocks
71 -
72 - else:
73 - blocks, _, _, _ = calculate_num_blocks(
74 - width,
75 - height,
76 - config.min_dynamic_patch,
77 - max_dynamic_patch,
78 - config.vision_config.image_size,
79 - use_thumbnail=False,
80 - prior_aspect_ratio=None,
81 - )
82 - expected_blocks = blocks
83 -
84 - if config.use_thumbnail and expected_blocks > 1:
85 - expected_blocks += 1
86 -
87 - return pixel_values, expected_blocks
88 -
89 -
90 -@pytest.mark.parametrize("model_name", models)
91 -@pytest.mark.parametrize(
92 - "size_factors",
93 - [
94 - # Single-scale
95 - [1.0],
96 - # Single-scale, batched
97 - [1.0, 1.0, 1.0],
98 - # Multi-scale
99 - [0.25, 0.5, 1.0],
100 - ],
101 -)
102 -@pytest.mark.parametrize("max_dynamic_patch", [None, 2, 4, 8])
103 -def test_image_preprocessing(image_assets, model_name, size_factors,
104 - max_dynamic_patch):
105 - """Test image preprocessing pipeline with different configurations."""
106 - # Load the configuration from the model
107 - config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
108 -
109 - for asset in image_assets:
110 - image = asset.pil_image
111 - for factor in size_factors:
112 - scaled_image = rescale_image_size(image, factor)
113 -
114 - # Test preprocessing and get expected number of blocks
115 - pixel_values, expected_blocks = run_preprocessing_test(
116 - scaled_image, config, max_dynamic_patch)
117 -
118 - # Verify output shapes and properties
119 - actual_blocks = pixel_values.shape[0]
120 - assert actual_blocks == expected_blocks, (
121 - f"Expected {expected_blocks} blocks, got {actual_blocks}")
122 -
123 - # Check image dimensions
124 - expected_size = (
125 - 3, # Number of channels (C, H, W)
126 - config.vision_config.image_size,
127 - config.vision_config.image_size,
128 - )
129 - for img in pixel_values:
130 - assert img.shape == expected_size, (
131 - f"Expected image size {expected_size}, got {img.shape}")
tests/models/decoder_only/vision_language/test_models.py
+1 -1
@@ -250,6 +250,7 @@ VLM_TEST_SETTINGS = {
250 max_model_len=8192,
251 dtype="bfloat16",
252 use_tokenizer_eos=True,
253 + num_logprobs=10,
254 patch_hf_runner=model_utils.h2ovl_patch_hf_runner,
255 ),
256 "idefics3": VLMTestInfo(
@@ -282,7 +283,6 @@ VLM_TEST_SETTINGS = {
283 dtype="bfloat16",
284 use_tokenizer_eos=True,
285 patch_hf_runner=model_utils.internvl_patch_hf_runner,
285 - marks=[large_gpu_mark(min_gb=32)],
286 ),
287 "llava_next": VLMTestInfo(
288 models=["llava-hf/llava-v1.6-mistral-7b-hf"],
tests/models/decoder_only/vision_language/vlm_utils/model_utils.py
+21 -16
@@ -334,12 +334,12 @@ def h2ovl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
334 def __init__(self, hf_runner: HfRunner):
335 self.num_image_token = hf_runner.model.num_image_token
336 self.tokenizer = hf_runner.tokenizer
337 - self.dtype = hf_runner.model.dtype
337
338 self.config = AutoConfig.from_pretrained(hf_runner.model_name,
339 trust_remote_code=True)
340 self.vision_config = self.config.vision_config
341 self.use_thumbnail = self.config.use_thumbnail
342 + self.use_msac = self.config.use_msac
343 self.min_num = self.config.min_dynamic_patch
344 self.max_num = self.config.max_dynamic_patch
345 self.image_size = self.vision_config.image_size
@@ -348,18 +348,19 @@ def h2ovl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
348 **kwargs):
349 # yapf: disable
350 from vllm.model_executor.models.h2ovl import (
351 - IMG_CONTEXT, IMG_END, IMG_START, image_to_pixel_values)
351 + IMG_CONTEXT, IMG_END, IMG_START, image_to_pixel_values_h2ovl)
352
353 # yapf: enable
354 images = [images] if isinstance(images, Image) else images
355 pixel_values = [
356 - image_to_pixel_values(image,
357 - self.image_size,
358 - self.min_num,
359 - self.max_num,
360 - self.use_thumbnail,
361 - use_MSAC=self.config.use_msac).to(
362 - self.dtype) for image in images
356 + image_to_pixel_values_h2ovl(
357 + image,
358 + input_size=self.image_size,
359 + min_num=self.min_num,
360 + max_num=self.max_num,
361 + use_thumbnail=self.use_thumbnail,
362 + use_msac=self.use_msac,
363 + ) for image in images
364 ]
365 num_patches_list = [
366 pixel_value.shape[0] for pixel_value in pixel_values
@@ -394,7 +395,6 @@ def internvl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
395 def __init__(self, hf_runner: HfRunner):
396 self.num_image_token = hf_runner.model.num_image_token
397 self.tokenizer = hf_runner.tokenizer
397 - self.dtype = hf_runner.model.dtype
398
399 self.config = AutoConfig.from_pretrained(hf_runner.model_name,
400 trust_remote_code=True)
@@ -407,13 +407,17 @@ def internvl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
407 def __call__(self, text: str, images: Union[Image, List[Image]],
408 **kwargs):
409 from vllm.model_executor.models.internvl import (
410 - IMG_CONTEXT, IMG_END, IMG_START, image_to_pixel_values)
410 + IMG_CONTEXT, IMG_END, IMG_START,
411 + image_to_pixel_values_internvl)
412 images = [images] if isinstance(images, Image) else images
413 pixel_values = [
413 - image_to_pixel_values(image, self.image_size, self.min_num,
414 - self.max_num,
415 - self.use_thumbnail).to(self.dtype)
416 - for image in images
414 + image_to_pixel_values_internvl(
415 + image,
416 + input_size=self.image_size,
417 + min_num=self.min_num,
418 + max_num=self.max_num,
419 + use_thumbnail=self.use_thumbnail,
420 + ) for image in images
421 ]
422 num_patches_list = [
423 pixel_value.shape[0] for pixel_value in pixel_values
@@ -448,7 +452,8 @@ def _internvl_generate(
452 ) -> torch.LongTensor:
453 """Generate method for InternVL2 model without fixed use_cache."""
454 assert self.img_context_token_id is not None
451 - vit_embeds = self.extract_feature(pixel_values)
455 + target_dtype = next(self.parameters()).dtype
456 + vit_embeds = self.extract_feature(pixel_values.to(target_dtype))
457 input_embeds = self.language_model.get_input_embeddings()(input_ids)
458 B, N, C = input_embeds.shape
459 input_embeds = input_embeds.reshape(B * N, C)
tests/models/multimodal/processing/test_common.py
+3 -1
@@ -141,13 +141,14 @@ def _test_processing_correctness(
141
142
143 # yapf: disable
144 -# True if the model supports multiple data items of the modality per request
144 @pytest.mark.parametrize("model_id", [
145 "rhymes-ai/Aria",
146 "Salesforce/blip2-opt-2.7b",
147 "facebook/chameleon-7b",
148 "deepseek-ai/deepseek-vl2-tiny",
149 "adept/fuyu-8b",
150 + "h2oai/h2ovl-mississippi-800m",
151 + "OpenGVLab/InternVL2-1B",
152 "llava-hf/llava-1.5-7b-hf",
153 "llava-hf/llava-v1.6-mistral-7b-hf",
154 "llava-hf/LLaVA-NeXT-Video-7B-hf",
@@ -156,6 +157,7 @@ def _test_processing_correctness(
157 "mistral-community/pixtral-12b",
158 "openbmb/MiniCPM-o-2_6",
159 "openbmb/MiniCPM-V-2_6",
160 + "nvidia/NVLM-D-72B",
161 "Qwen/Qwen-VL-Chat",
162 "Qwen/Qwen2-VL-2B-Instruct",
163 "Qwen/Qwen2-Audio-7B-Instruct",
tests/models/multimodal/processing/test_h2ovl.py new
+142
@@ -0,0 +1,142 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +"""Tests for H2OVL's multimodal preprocessing kwargs."""
3 +from typing import Optional
4 +
5 +import pytest
6 +
7 +from vllm.multimodal import MULTIMODAL_REGISTRY
8 +from vllm.multimodal.image import rescale_image_size
9 +from vllm.multimodal.utils import cached_get_tokenizer
10 +
11 +from ....conftest import _ImageAssets
12 +from ...utils import build_model_context
13 +
14 +
15 +@pytest.mark.parametrize("model_id", [
16 + "h2oai/h2ovl-mississippi-800m",
17 + "h2oai/h2ovl-mississippi-2b",
18 +])
19 +@pytest.mark.parametrize(
20 + "size_factors",
21 + [
22 + # Single-scale
23 + [1.0],
24 + # Single-scale, batched
25 + [1.0, 1.0, 1.0],
26 + # Multi-scale
27 + [0.25, 0.5, 1.0],
28 + ],
29 +)
30 +@pytest.mark.parametrize("max_dynamic_patch", [1, 2, 4, 8])
31 +@pytest.mark.parametrize("dynamic_image_size", [True, False])
32 +@pytest.mark.parametrize("num_imgs", [1, 2])
33 +def test_processor_override(
34 + model_id: str,
35 + image_assets: _ImageAssets,
36 + size_factors: list[int],
37 + max_dynamic_patch: int,
38 + dynamic_image_size: Optional[bool],
39 + num_imgs: int,
40 +):
41 + from vllm.model_executor.models.h2ovl import (calculate_h2ovl_targets,
42 + get_h2ovl_target_ratios)
43 +
44 + ctx = build_model_context(
45 + model_name=model_id,
46 + tokenizer_name=model_id,
47 + trust_remote_code=True,
48 + mm_processor_kwargs=None,
49 + limit_mm_per_prompt={"image": num_imgs},
50 + )
51 + tokenizer = cached_get_tokenizer(
52 + ctx.model_config.tokenizer,
53 + trust_remote_code=ctx.model_config.trust_remote_code,
54 + )
55 + processor = MULTIMODAL_REGISTRY.create_processor(
56 + ctx.model_config,
57 + tokenizer=tokenizer,
58 + )
59 +
60 + config = processor.info.get_hf_config()
61 + use_msac = config.use_msac
62 +
63 + mm_processor_kwargs = {
64 + "max_dynamic_patch": max_dynamic_patch,
65 + }
66 + if dynamic_image_size is not None:
67 + mm_processor_kwargs["dynamic_image_size"] = dynamic_image_size
68 +
69 + min_num = config.min_dynamic_patch
70 + max_num = max_dynamic_patch if dynamic_image_size else 1
71 +
72 + # Build the image str / prompt based on the number of images we pass
73 + prompt = "<image>" * num_imgs
74 +
75 + for asset in image_assets:
76 + for factor in size_factors:
77 + image = rescale_image_size(asset.pil_image, factor)
78 + mm_data = {"image": [image] * num_imgs}
79 +
80 + width, height = image.size
81 +
82 + # Calculate the expected number of blocks
83 + if num_imgs == 1 and use_msac:
84 + # First pass
85 + blocks1, _, _, aspect_ratio = calculate_h2ovl_targets(
86 + orig_width=width,
87 + orig_height=height,
88 + target_ratios=get_h2ovl_target_ratios(
89 + min_num,
90 + max_num,
91 + prior_aspect_ratio=None,
92 + ),
93 + image_size=config.vision_config.image_size,
94 + use_thumbnail=False, # Thumbnail is handled separately
95 + )
96 +
97 + # Second pass
98 + blocks2, _, _, _ = calculate_h2ovl_targets(
99 + orig_width=width,
100 + orig_height=height,
101 + target_ratios=get_h2ovl_target_ratios(
102 + min_num,
103 + max_num,
104 + prior_aspect_ratio=aspect_ratio,
105 + ),
106 + image_size=config.vision_config.image_size,
107 + use_thumbnail=False,
108 + )
109 +
110 + # Add thumbnail if use_thumbnail is True and total_blocks > 1
111 + if config.use_thumbnail:
112 + blocks1 += 1 if blocks1 > 1 else 0
113 + blocks2 += 1 if blocks2 > 1 else 0
114 +
115 + # Total blocks is the sum of blocks from both passes minus
116 + # overlapping
117 + total_blocks = blocks1 + blocks2 - 1
118 +
119 + expected_num_patches = total_blocks
120 + else:
121 + blocks, _, _, _ = calculate_h2ovl_targets(
122 + orig_width=width,
123 + orig_height=height,
124 + target_ratios=get_h2ovl_target_ratios(
125 + min_num,
126 + max_num,
127 + prior_aspect_ratio=None,
128 + ),
129 + image_size=config.vision_config.image_size,
130 + use_thumbnail=False,
131 + )
132 + expected_num_patches = blocks
133 +
134 + if config.use_thumbnail and expected_num_patches != 1:
135 + expected_num_patches += 1
136 +
137 + processed_inputs = processor.apply(prompt, mm_data,
138 + mm_processor_kwargs)
139 + pixel_shape = (
140 + processed_inputs["mm_kwargs"]["pixel_values_flat"].shape)
141 +
142 + assert pixel_shape[0] == expected_num_patches * num_imgs
tests/models/multimodal/processing/test_internvl.py
+32 -175
@@ -1,207 +1,64 @@
1 # SPDX-License-Identifier: Apache-2.0
2 """Tests for InternVL's multimodal preprocessing kwargs."""
3 -from typing import Callable, Optional
3 +from typing import Optional
4
5 import pytest
6 -from transformers import AutoTokenizer
6
8 -from vllm.inputs import InputContext, token_inputs
9 -from vllm.multimodal import MultiModalRegistry
7 +from vllm.multimodal import MULTIMODAL_REGISTRY
8 +from vllm.multimodal.utils import cached_get_tokenizer
9
10 from ....conftest import _ImageAssets
11 from ...utils import build_model_context
12
14 -models = ["OpenGVLab/InternVL2-2B"]
13
16 -
17 -# Wrap lazy imports to avoid initializing CUDA during test collection
18 -@pytest.fixture()
19 -def input_processor_for_internvl():
20 - from vllm.model_executor.models.internvl import InternVLInputPipeline
21 -
22 - pipeline = InternVLInputPipeline('<img>', '</img>', '<IMG_CONTEXT>')
23 - return pipeline.input_processor
24 -
25 -
26 -@pytest.fixture()
27 -def dummy_data_for_internvl():
28 - from vllm.model_executor.models.internvl import InternVLInputPipeline
29 -
30 - pipeline = InternVLInputPipeline('<img>', '</img>', '<IMG_CONTEXT>')
31 - return pipeline.dummy_data
32 -
33 -
34 -@pytest.fixture()
35 -def get_max_internvl_image_tokens():
36 - from vllm.model_executor.models.internvl import (
37 - get_max_internvl_image_tokens)
38 - return get_max_internvl_image_tokens
39 -
40 -
41 -@pytest.mark.parametrize("model", models)
14 +@pytest.mark.parametrize("model_id", ["OpenGVLab/InternVL2-2B"])
15 @pytest.mark.parametrize("max_dynamic_patch", [1, 4])
16 @pytest.mark.parametrize("dynamic_image_size", [True, False, None])
44 -def test_input_mapper_override(
45 - model: str,
17 +@pytest.mark.parametrize("num_imgs", [1, 2])
18 +def test_processor_override(
19 + model_id: str,
20 image_assets: _ImageAssets,
21 max_dynamic_patch: int,
22 dynamic_image_size: Optional[bool],
49 -):
50 - mm_processor_kwargs = {
51 - "max_dynamic_patch": max_dynamic_patch,
52 - }
53 - if dynamic_image_size is not None:
54 - mm_processor_kwargs["dynamic_image_size"] = dynamic_image_size
55 -
56 - expected_num_patches = max_dynamic_patch + 1 if max_dynamic_patch > 1 else 1
57 - if dynamic_image_size is False:
58 - expected_num_patches = 1
59 -
60 - ctx = build_model_context(
61 - model_name=model,
62 - tokenizer_name=model,
63 - trust_remote_code=True,
64 - mm_processor_kwargs=mm_processor_kwargs,
65 - )
66 -
67 - mm_registry = MultiModalRegistry()
68 - mm_registry.init_mm_limits_per_prompt(ctx.model_config)
69 -
70 - image = image_assets[0].pil_image.resize((448 * 2, 448 * 2))
71 - vllm_result = mm_registry.map_input(
72 - ctx.model_config,
73 - {"image": image},
74 - )
75 - assert vllm_result["pixel_values"].size(1) == expected_num_patches
76 -
77 -
78 -@pytest.mark.parametrize("model", models)
79 -@pytest.mark.parametrize("max_dynamic_patch", [1, 4, None])
80 -@pytest.mark.parametrize("dynamic_image_size", [True, False, None])
81 -def test_max_tokens_override(
82 - get_max_internvl_image_tokens: Callable,
83 - model: str,
84 - max_dynamic_patch: Optional[int],
85 - dynamic_image_size: Optional[bool],
86 -):
87 - """Ensure get_max_internvl_image_tokens handles mm_processor_kwargs."""
88 - ctx = build_model_context(
89 - model_name=model,
90 - tokenizer_name=model,
91 - trust_remote_code=True,
92 - mm_processor_kwargs=None,
93 - )
94 -
95 - if max_dynamic_patch is None:
96 - max_dynamic_patch = ctx.get_hf_config().max_dynamic_patch
97 - expected_num_patches = max_dynamic_patch + 1 if max_dynamic_patch > 1 else 1
98 - if dynamic_image_size is False:
99 - expected_num_patches = 1
100 - expected_max_tokens = 256 * expected_num_patches
101 -
102 - actual_max_tokens = get_max_internvl_image_tokens(
103 - ctx=InputContext(ctx.model_config),
104 - max_dynamic_patch=max_dynamic_patch,
105 - dynamic_image_size=dynamic_image_size,
106 - )
107 - assert expected_max_tokens == actual_max_tokens
108 -
109 -
110 -@pytest.mark.parametrize("model", models)
111 -@pytest.mark.parametrize("num_imgs", [1, 2])
112 -@pytest.mark.parametrize("max_dynamic_patch", [1, 4, None])
113 -@pytest.mark.parametrize("dynamic_image_size", [True, False, None])
114 -def test_dummy_data_override(
115 - dummy_data_for_internvl: Callable,
116 - model: str,
23 num_imgs: int,
118 - max_dynamic_patch: Optional[int],
119 - dynamic_image_size: Optional[bool],
24 ):
121 - """Ensure dummy_data_for_internvl handles kwargs properly."""
122 - # Same as the previous test - don't initialize mm_processor_kwargs
123 - # in this test and assume that the kwargs will be correctly expanded by
124 - # the partial when calling the dummy data func.
25 ctx = build_model_context(
126 - model_name=model,
127 - tokenizer_name=model,
26 + model_name=model_id,
27 + tokenizer_name=model_id,
28 trust_remote_code=True,
29 mm_processor_kwargs=None,
30 + limit_mm_per_prompt={"image": num_imgs},
31 )
131 -
132 - if max_dynamic_patch is None:
133 - max_dynamic_patch = ctx.get_hf_config().max_dynamic_patch
134 - expected_num_patches = max_dynamic_patch + 1 if max_dynamic_patch > 1 else 1
135 - if dynamic_image_size is False:
136 - expected_num_patches = 1
137 - expected_max_tokens = 256 * expected_num_patches
138 -
139 - dummy_data = dummy_data_for_internvl(
140 - ctx=ctx,
141 - seq_len=8192, # Should be bigger than num_imgs * toks_per_img
142 - mm_counts={"image": num_imgs},
143 - max_dynamic_patch=max_dynamic_patch,
144 - dynamic_image_size=dynamic_image_size,
32 + tokenizer = cached_get_tokenizer(
33 + ctx.model_config.tokenizer,
34 + trust_remote_code=ctx.model_config.trust_remote_code,
35 + )
36 + processor = MULTIMODAL_REGISTRY.create_processor(
37 + ctx.model_config,
38 + tokenizer=tokenizer,
39 )
146 - sequence_data = dummy_data.seq_data
147 -
148 - tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
149 - image_token_id = tokenizer.encode('<IMG_CONTEXT>',
150 - add_special_tokens=False)[0]
40
152 - # Ensure we have the right number of placeholders per size
153 - img_tok_count = sequence_data.get_token_ids().count(image_token_id)
154 - assert img_tok_count == expected_max_tokens * num_imgs
41 + mm_processor_kwargs = {
42 + "max_dynamic_patch": max_dynamic_patch,
43 + }
44 + if dynamic_image_size is not None:
45 + mm_processor_kwargs["dynamic_image_size"] = dynamic_image_size
46
47 + # Build the image str / prompt based on the number of images we pass
48 + prompt = "<image>" * num_imgs
49 + image = image_assets[0].pil_image.resize((448 * 2, 448 * 2))
50 + mm_data = {"image": [image] * num_imgs}
51
157 -@pytest.mark.parametrize("model", models)
158 -@pytest.mark.parametrize("max_dynamic_patch", [1, 4])
159 -@pytest.mark.parametrize("dynamic_image_size", [True, False, None])
160 -@pytest.mark.parametrize("num_imgs", [1, 2])
161 -def test_input_processor_override(
162 - input_processor_for_internvl: Callable,
163 - image_assets: _ImageAssets,
164 - model: str,
165 - num_imgs: int,
166 - max_dynamic_patch: int,
167 - dynamic_image_size: Optional[bool],
168 -):
169 - """Ensure input_processor_for_internvl handles kwargs properly."""
170 - # Same as the previous test - don't initialize mm_processor_kwargs
171 - # in this test and assume that the kwargs will be correctly expanded by
172 - # the partial when calling the custom input processor.
52 expected_num_patches = max_dynamic_patch + 1 if max_dynamic_patch > 1 else 1
53 if dynamic_image_size is False:
54 expected_num_patches = 1
55
177 - ctx = build_model_context(
178 - model_name=model,
179 - tokenizer_name=model,
180 - trust_remote_code=True,
181 - mm_processor_kwargs=None,
182 - )
183 - expected_toks_per_img = 256 * expected_num_patches
184 -
185 - # Build the image str / prompt based on the number of images we pass
186 - tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
187 - placeholders = "<image>" if num_imgs == 1 else "\n".join(
188 - f"Image-{i}: <image>\n" for i in range(1, num_imgs + 1))
189 - prompt = placeholders
190 - images = [image_assets[0].pil_image.resize((448 * 2, 448 * 2))] * num_imgs
191 -
192 - inputs = token_inputs(prompt_token_ids=tokenizer.encode(prompt),
193 - prompt=prompt,
194 - multi_modal_data={"image": images})
195 -
196 - processed_inputs = input_processor_for_internvl(
197 - ctx,
198 - inputs,
199 - max_dynamic_patch=max_dynamic_patch,
200 - dynamic_image_size=dynamic_image_size,
201 - )
56 + processed_inputs = processor.apply(prompt, mm_data, mm_processor_kwargs)
57
58 # Ensure we have the right number of placeholders per num_crops size
204 - image_token_id = tokenizer.encode('<IMG_CONTEXT>',
205 - add_special_tokens=False)[0]
59 + image_token_id = tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
60 img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
207 - assert img_tok_count == expected_toks_per_img * num_imgs
61 + pixel_shape = processed_inputs["mm_kwargs"]["pixel_values_flat"].shape
62 +
63 + assert img_tok_count == 256 * expected_num_patches * num_imgs
64 + assert pixel_shape[0] == expected_num_patches * num_imgs
tests/models/multimodal/processing/test_llava_next.py
+12 -3
@@ -43,7 +43,10 @@ def test_processor_max_tokens(model_id):
43 )
44 processor = MULTIMODAL_REGISTRY.create_processor(
45 ctx.model_config,
46 - tokenizer=cached_get_tokenizer(ctx.model_config.tokenizer),
46 + tokenizer=cached_get_tokenizer(
47 + ctx.model_config.tokenizer,
48 + trust_remote_code=ctx.model_config.trust_remote_code,
49 + ),
50 )
51 info = processor.info
52
@@ -143,7 +146,10 @@ def test_processor_prompt_replacements_regression(model_id, num_imgs):
146 )
147 processor = MULTIMODAL_REGISTRY.create_processor(
148 ctx.model_config,
146 - tokenizer=cached_get_tokenizer(ctx.model_config.tokenizer),
149 + tokenizer=cached_get_tokenizer(
150 + ctx.model_config.tokenizer,
151 + trust_remote_code=ctx.model_config.trust_remote_code,
152 + ),
153 )
154
155 image_ratios = [(171, 152), (184, 161), (198, 176), (333, 296), (369, 328),
@@ -173,7 +179,10 @@ def test_processor_prompt_replacements_all(model_id, num_imgs):
179 )
180 processor = MULTIMODAL_REGISTRY.create_processor(
181 ctx.model_config,
176 - tokenizer=cached_get_tokenizer(ctx.model_config.tokenizer),
182 + tokenizer=cached_get_tokenizer(
183 + ctx.model_config.tokenizer,
184 + trust_remote_code=ctx.model_config.trust_remote_code,
185 + ),
186 )
187
188 seen_aspect_ratios = set[float]()
tests/models/multimodal/processing/test_llava_onevision.py
+12 -3
@@ -44,7 +44,10 @@ def test_processor_max_tokens(model_id):
44 )
45 processor = MULTIMODAL_REGISTRY.create_processor(
46 ctx.model_config,
47 - tokenizer=cached_get_tokenizer(ctx.model_config.tokenizer),
47 + tokenizer=cached_get_tokenizer(
48 + ctx.model_config.tokenizer,
49 + trust_remote_code=ctx.model_config.trust_remote_code,
50 + ),
51 )
52 info = processor.info
53
@@ -143,7 +146,10 @@ def test_processor_prompt_replacements_regression(model_id, num_imgs):
146 )
147 processor = MULTIMODAL_REGISTRY.create_processor(
148 ctx.model_config,
146 - tokenizer=cached_get_tokenizer(ctx.model_config.tokenizer),
149 + tokenizer=cached_get_tokenizer(
150 + ctx.model_config.tokenizer,
151 + trust_remote_code=ctx.model_config.trust_remote_code,
152 + ),
153 )
154
155 image_ratios = [(171, 152), (184, 161), (198, 176), (333, 296), (369, 328),
@@ -174,7 +180,10 @@ def test_processor_prompt_replacements_all(model_id, num_imgs):
180 )
181 processor = MULTIMODAL_REGISTRY.create_processor(
182 ctx.model_config,
177 - tokenizer=cached_get_tokenizer(ctx.model_config.tokenizer),
183 + tokenizer=cached_get_tokenizer(
184 + ctx.model_config.tokenizer,
185 + trust_remote_code=ctx.model_config.trust_remote_code,
186 + ),
187 )
188
189 seen_aspect_ratios = set[float]()
tests/models/multimodal/processing/test_phi3v.py
+4 -1
@@ -38,7 +38,10 @@ def test_processor_override(
38 trust_remote_code=True,
39 limit_mm_per_prompt={"image": num_imgs},
40 )
41 - tokenizer = cached_get_tokenizer(ctx.model_config.tokenizer)
41 + tokenizer = cached_get_tokenizer(
42 + ctx.model_config.tokenizer,
43 + trust_remote_code=ctx.model_config.trust_remote_code,
44 + )
45 processor = MULTIMODAL_REGISTRY.create_processor(
46 ctx.model_config,
47 tokenizer=tokenizer,
tests/models/multimodal/processing/test_qwen2_vl.py
+4 -1
@@ -33,7 +33,10 @@ def test_processor_override(
33 mm_processor_kwargs=None,
34 limit_mm_per_prompt={"image": num_imgs},
35 )
36 - tokenizer = cached_get_tokenizer(ctx.model_config.tokenizer)
36 + tokenizer = cached_get_tokenizer(
37 + ctx.model_config.tokenizer,
38 + trust_remote_code=ctx.model_config.trust_remote_code,
39 + )
40 processor = MULTIMODAL_REGISTRY.create_processor(
41 ctx.model_config,
42 tokenizer=tokenizer,
vllm/model_executor/models/aria.py
+5 -1
@@ -399,7 +399,11 @@ class AriaProcessingInfo(BaseProcessingInfo):
399 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
400 return {"image": None}
401
402 - def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
402 + def get_mm_max_tokens_per_item(
403 + self,
404 + seq_len: int,
405 + mm_counts: Mapping[str, int],
406 + ) -> Mapping[str, int]:
407 return {"image": self.get_num_image_tokens()}
408
409 def get_num_image_tokens(self) -> int:
vllm/model_executor/models/blip2.py
+5 -1
@@ -407,7 +407,11 @@ class Blip2ProcessingInfo(BaseProcessingInfo):
407 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
408 return {"image": 1}
409
410 - def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
410 + def get_mm_max_tokens_per_item(
411 + self,
412 + seq_len: int,
413 + mm_counts: Mapping[str, int],
414 + ) -> Mapping[str, int]:
415 return {"image": self.get_num_image_tokens()}
416
417 def get_num_image_tokens(self) -> int:
vllm/model_executor/models/chameleon.py
+5 -1
@@ -64,7 +64,11 @@ class ChameleonProcessingInfo(BaseProcessingInfo):
64 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
65 return {"image": 1}
66
67 - def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
67 + def get_mm_max_tokens_per_item(
68 + self,
69 + seq_len: int,
70 + mm_counts: Mapping[str, int],
71 + ) -> Mapping[str, int]:
72 return {"image": self.get_num_image_tokens()}
73
74 def get_num_image_tokens(self) -> int:
vllm/model_executor/models/deepseek_vl2.py
+5 -1
@@ -165,7 +165,11 @@ class DeepseekVL2ProcessingInfo(BaseProcessingInfo):
165 image_width=x[1], image_height=x[0]))
166 return ImageSize(width=width, height=height)
167
168 - def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
168 + def get_mm_max_tokens_per_item(
169 + self,
170 + seq_len: int,
171 + mm_counts: Mapping[str, int],
172 + ) -> Mapping[str, int]:
173 max_image_size = self.get_image_size_with_most_features()
174 max_image_tokens = self.get_num_image_tokens(
175 image_height=max_image_size.height,
vllm/model_executor/models/fuyu.py
+5 -1
@@ -80,7 +80,11 @@ class FuyuProcessingInfo(BaseProcessingInfo):
80 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
81 return {"image": 1}
82
83 - def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
83 + def get_mm_max_tokens_per_item(
84 + self,
85 + seq_len: int,
86 + mm_counts: Mapping[str, int],
87 + ) -> Mapping[str, int]:
88 target_width, target_height = self.get_image_size_with_most_features()
89
90 max_ncols, max_nrows = self.get_image_feature_grid_size(
vllm/model_executor/models/h2ovl.py
+386 -235
@@ -7,43 +7,55 @@
7 # Copyright (c) 2024 H2O.AI
8 # Licensed under Apache 2.0 License [see LICENSE for details]
9 # --------------------------------------------------------
10 -from functools import partial
11 -from typing import List, Optional, Tuple
10 +from typing import Mapping, Optional
11
12 import torch
13 from PIL import Image
14 from transformers import PretrainedConfig
15
17 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
18 - token_inputs)
16 +from vllm.logger import init_logger
17 from vllm.model_executor.layers.quantization import QuantizationConfig
20 -from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
21 -from vllm.multimodal.utils import cached_get_tokenizer
22 -from vllm.utils import is_list_of
18 +from vllm.multimodal import MULTIMODAL_REGISTRY
19 +from vllm.multimodal.inputs import MultiModalKwargs
20 +from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
21 + MultiModalDataItems)
22 +from vllm.multimodal.processing import (ProcessingCache, PromptReplacement,
23 + PromptReplacementDetails)
24 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
25 +from vllm.transformers_utils.tokenizer import AnyTokenizer
26
27 from .intern_vit import InternVisionModel
25 -from .internvl import (IMG_CONTEXT, IMG_END, IMG_START, InternVLChatModel,
26 - InternVLInputPipeline, build_transform,
27 - find_closest_aspect_ratio, get_internvl_num_patches)
28 +from .internvl import (IMG_CONTEXT, IMG_END, IMG_START,
29 + BaseInternVLProcessingInfo, BaseInternVLProcessor,
30 + InternVLChatModel, InternVLDummyInputsBuilder,
31 + InternVLMultiModalProcessor, build_transform,
32 + find_closest_aspect_ratio, get_internvl_target_ratios)
33
34 +logger = init_logger(__name__)
35
30 -# modified to include blocks generated in second pass
31 -def calculate_num_blocks(
32 - orig_width: int,
33 - orig_height: int,
34 - min_num: int,
35 - max_num: int,
36 - image_size: int,
36 +
37 +def resolve_h2ovl_min_max_num(
38 + *,
39 + min_dynamic_patch: int,
40 + max_dynamic_patch: int,
41 + dynamic_image_size: bool,
42 use_thumbnail: bool,
38 - prior_aspect_ratio=None,
39 -) -> Tuple[int, int, int, Tuple[int, int]]:
40 - aspect_ratio = orig_width / orig_height
43 +) -> tuple[int, int]:
44 + max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
45 +
46 + if use_thumbnail and max_dynamic_patch != 1:
47 + max_dynamic_patch += 1
48
42 - # calculate the existing image aspect ratio
43 - target_ratios = set((i, j) for n in range(min_num, max_num + 1)
44 - for i in range(1, n + 1) for j in range(1, n + 1)
45 - if i * j <= max_num and i * j >= min_num)
46 - target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
49 + return min_dynamic_patch, max_dynamic_patch
50 +
51 +
52 +def get_h2ovl_target_ratios(
53 + min_num: int,
54 + max_num: int,
55 + *,
56 + prior_aspect_ratio: Optional[tuple[int, int]],
57 +) -> list[tuple[int, int]]:
58 + target_ratios = get_internvl_target_ratios(min_num, max_num)
59
60 # if prior_aspect_ratio is provided, filter the target ratios
61 if prior_aspect_ratio is not None:
@@ -52,44 +64,66 @@ def calculate_num_blocks(
64 ratio[0] != 0 and prior_aspect_ratio[1] % ratio[1] != 0
65 ]
66
67 + return target_ratios
68 +
69 +
70 +# modified to include blocks generated in second pass
71 +def calculate_h2ovl_targets(
72 + *,
73 + orig_width: int,
74 + orig_height: int,
75 + target_ratios: list[tuple[int, int]],
76 + image_size: int,
77 + use_thumbnail: bool,
78 +) -> tuple[int, int, int, tuple[int, int]]:
79 + aspect_ratio = orig_width / orig_height
80 +
81 # find the closest aspect ratio to the target
56 - target_aspect_ratio = find_closest_aspect_ratio(aspect_ratio,
57 - target_ratios, orig_width,
58 - orig_height, image_size)
82 + target_aspect_ratio = find_closest_aspect_ratio(
83 + aspect_ratio,
84 + target_ratios,
85 + width=orig_width,
86 + height=orig_height,
87 + image_size=image_size,
88 + )
89
90 # calculate the target width and height
91 target_width = image_size * target_aspect_ratio[0]
92 target_height = image_size * target_aspect_ratio[1]
93 blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
64 - # add thumbnail image if num_blocks > 1
65 - if use_thumbnail and blocks > 1:
94 +
95 + # add thumbnail image if num_blocks != 1
96 + if use_thumbnail and blocks != 1:
97 blocks += 1
98 +
99 return blocks, target_width, target_height, target_aspect_ratio
100
101
102 # adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
71 -# refactored to handle prior_aspect_ratio as optional
72 -def dynamic_preprocess(
103 +# refactored to handle prior_aspect_ratio
104 +def dynamic_preprocess_h2ovl(
105 image: Image.Image,
74 - min_num: int,
75 - max_num: int,
106 + *,
107 + target_ratios: list[tuple[int, int]],
108 image_size: int,
109 use_thumbnail: bool,
78 - prior_aspect_ratio: Optional[Tuple[int, int]] = None,
79 -) -> Tuple[List[Image.Image], Tuple[int, int]]:
110 +) -> tuple[list[Image.Image], tuple[int, int]]:
111 orig_width, orig_height = image.size
112
82 - # calculate the number of blocks based on prior aspect ratio if available
83 - blocks, target_width, target_height, target_aspect_ratio = (
84 - calculate_num_blocks(
85 - orig_width,
86 - orig_height,
87 - min_num,
88 - max_num,
89 - image_size,
90 - use_thumbnail=False,
91 - prior_aspect_ratio=prior_aspect_ratio,
92 - ))
113 + # calculate the number of blocks without thumbnail
114 + (
115 + blocks,
116 + target_width,
117 + target_height,
118 + target_aspect_ratio,
119 + ) = calculate_h2ovl_targets(
120 + orig_width=orig_width,
121 + orig_height=orig_height,
122 + target_ratios=target_ratios,
123 + image_size=image_size,
124 + use_thumbnail=False,
125 + )
126 +
127 # resize the image
128 resized_img = image.resize((target_width, target_height))
129 processed_images = []
@@ -103,276 +137,393 @@ def dynamic_preprocess(
137 # split the image
138 split_img = resized_img.crop(box)
139 processed_images.append(split_img)
140 +
141 assert len(processed_images) == blocks
142 +
143 if use_thumbnail and len(processed_images) != 1:
144 thumbnail_img = image.resize((image_size, image_size))
145 processed_images.append(thumbnail_img)
146 +
147 return processed_images, target_aspect_ratio
148
149
113 -def load_image(
150 +def _preprocess_image(
151 image: Image.Image,
115 - input_size=448,
116 - min_num=1,
117 - max_num=6,
118 - use_thumbnail=True,
119 - prior_aspect_ratio: Optional[Tuple[int, int]] = None,
120 -) -> Tuple[torch.Tensor, Tuple[int, int]]:
152 + *,
153 + input_size: int,
154 + min_num: int,
155 + max_num: int,
156 + use_thumbnail: bool,
157 + prior_aspect_ratio: Optional[tuple[int, int]],
158 +) -> tuple[torch.Tensor, tuple[int, int]]:
159 + target_ratios = get_h2ovl_target_ratios(
160 + min_num,
161 + max_num,
162 + prior_aspect_ratio=prior_aspect_ratio,
163 + )
164 +
165 transform = build_transform(input_size=input_size)
122 - images, target_aspect_ratio = dynamic_preprocess(
166 + images, target_aspect_ratio = dynamic_preprocess_h2ovl(
167 image,
168 image_size=input_size,
169 use_thumbnail=use_thumbnail,
126 - min_num=min_num,
127 - max_num=max_num,
128 - prior_aspect_ratio=prior_aspect_ratio,
170 + target_ratios=target_ratios,
171 )
130 - pixel_values = [transform(image) for image in images]
131 - pixel_values = torch.stack(pixel_values)
172 +
173 + pixel_values = torch.stack([transform(image) for image in images])
174 return pixel_values, target_aspect_ratio
175
176
135 -# refactored to use the combined load_image function
136 -def image_to_pixel_values(
177 +# refactored to use the _preprocess_image function
178 +def image_to_pixel_values_h2ovl(
179 image: Image.Image,
180 + *,
181 input_size: int,
182 min_num: int,
183 max_num: int,
184 use_thumbnail: bool,
142 - use_MSAC: bool,
185 + use_msac: bool,
186 ) -> torch.Tensor:
187 # when MSAC is turned on, we need to process the image twice
145 - if use_MSAC:
188 + if use_msac:
189 # first pass
147 - pixel_values, target_aspect_ratio = load_image(
190 + pixel_values1, aspect_ratio1 = _preprocess_image(
191 image,
192 input_size=input_size,
193 min_num=min_num,
194 max_num=max_num,
195 use_thumbnail=True,
196 + prior_aspect_ratio=None,
197 )
198 # second pass
155 - pixel_values2, _ = load_image(
199 + pixel_values2, _ = _preprocess_image(
200 image,
201 input_size=input_size,
158 - min_num=min_num,
202 + min_num=3, # Hardcoded value
203 max_num=max_num,
160 - prior_aspect_ratio=target_aspect_ratio,
204 + use_thumbnail=True,
205 + prior_aspect_ratio=aspect_ratio1,
206 )
207 # combine pixel values
208 pixel_values = torch.cat(
164 - [pixel_values2[:-1], pixel_values[:-1], pixel_values2[-1:]], 0)
209 + [pixel_values2[:-1], pixel_values1[:-1], pixel_values2[-1:]], 0)
210
211 else:
167 - pixel_values, _ = load_image(
212 + pixel_values, _ = _preprocess_image(
213 image,
214 input_size=input_size,
215 min_num=min_num,
216 max_num=max_num,
217 use_thumbnail=use_thumbnail,
218 + prior_aspect_ratio=None,
219 )
220
221 return pixel_values
222
223
178 -def image_to_pixel_values_wrapper(hf_config: PretrainedConfig,
179 - max_dynamic_patch: Optional[int] = None,
180 - use_MSAC: Optional[bool] = None):
181 - image_size = hf_config.vision_config.image_size
182 - min_num = hf_config.min_dynamic_patch
183 - if max_dynamic_patch is None:
184 - max_dynamic_patch = hf_config.max_dynamic_patch
185 - if use_MSAC is None:
186 - use_MSAC = hf_config.use_msac
187 - use_thumbnail = hf_config.use_thumbnail
188 - return partial(
189 - image_to_pixel_values,
190 - input_size=image_size,
191 - min_num=min_num,
192 - max_num=max_dynamic_patch,
193 - use_thumbnail=use_thumbnail,
194 - use_MSAC=use_MSAC,
195 - )
196 -
197 -
198 -def get_max_internvl_image_tokens(ctx: InputContext,
199 - *,
200 - max_dynamic_patch: Optional[int] = None):
201 - """
202 - Calculate the maximum number of tokens with/without MSAC and thumbnail
203 - """
204 - hf_config = ctx.get_hf_config()
205 - use_thumbnail = hf_config.use_thumbnail
206 - use_MSAC = hf_config.use_msac
224 +class H2OVLProcessor(BaseInternVLProcessor):
225
208 - if max_dynamic_patch is None:
209 - max_dynamic_patch = hf_config.max_dynamic_patch
226 + def __init__(
227 + self,
228 + config: PretrainedConfig,
229 + tokenizer: AnyTokenizer,
230 + *,
231 + max_dynamic_patch: Optional[int] = None,
232 + dynamic_image_size: Optional[bool] = None,
233 + use_msac: Optional[bool] = None,
234 + ) -> None:
235 + super().__init__(
236 + config,
237 + tokenizer,
238 + max_dynamic_patch=max_dynamic_patch,
239 + dynamic_image_size=dynamic_image_size,
240 + )
241
211 - num_patches = get_internvl_num_patches(hf_config)
242 + if use_msac is None:
243 + use_msac = config.use_msac
244 + assert isinstance(use_msac, bool)
245
213 - coefficient = 2 if use_MSAC else 1
214 - num_blocks = coefficient * max_dynamic_patch + (1 if use_thumbnail else 0)
246 + self.use_msac = use_msac
247
216 - return num_blocks * num_patches
248 + @property
249 + def image_token_id(self) -> int:
250 + return self.tokenizer.get_vocab()[IMG_CONTEXT]
251
252 + def get_image_repl_features(
253 + self,
254 + feature_size: int,
255 + num_patches: Optional[int],
256 + ) -> str:
257 + return IMG_CONTEXT * feature_size
258
219 -class H2OVLInputPipeline(InternVLInputPipeline):
220 - """
221 - Input pipeline for processing image and text data for the H2OVL model.
222 - """
259 + def get_image_repl_full(
260 + self,
261 + feature_size: int,
262 + num_patches: Optional[int],
263 + ) -> str:
264 + features = self.get_image_repl_features(feature_size, num_patches)
265 + return IMG_START + features + IMG_END
266
224 - def input_processor(
267 + def resolve_min_max_num(
268 self,
226 - ctx: InputContext,
227 - inputs: DecoderOnlyInputs,
269 *,
270 max_dynamic_patch: Optional[int] = None,
230 - ) -> DecoderOnlyInputs:
231 - # get multi_modal_data
232 - multi_modal_data = inputs.get("multi_modal_data")
233 - if multi_modal_data is None or "image" not in multi_modal_data:
234 - return inputs
235 -
236 - model_config = ctx.model_config
237 - hf_config = ctx.get_hf_config()
238 - use_MSAC = hf_config.use_msac
239 -
240 - image_data = multi_modal_data["image"]
241 - num_patches = get_internvl_num_patches(hf_config)
242 -
243 - image_pixel_values_mapper = image_to_pixel_values_wrapper(
244 - hf_config, max_dynamic_patch=max_dynamic_patch)
245 -
246 - # single image
247 - if isinstance(image_data, Image.Image):
248 - pixel_values = image_pixel_values_mapper(image_data,
249 - use_MSAC=use_MSAC)
250 - num_blocks = pixel_values.shape[0]
251 - image_feature_sizes = [num_blocks * num_patches]
252 - pixel_values = pixel_values.unsqueeze(0)
253 -
254 - # multi images
255 - elif is_list_of(image_data, Image.Image):
256 - # Do not use MSAC for multi images
257 - image_feature_sizes = []
258 - pixel_values = [
259 - image_pixel_values_mapper(image, use_MSAC=False)
260 - for image in image_data
261 - ]
262 - for pixel_value in pixel_values:
263 - num_blocks = pixel_value.shape[0]
264 - image_feature_sizes.append(num_blocks * num_patches)
265 -
266 - # image embeddings as input
267 - elif isinstance(image_data, torch.Tensor):
268 - _, image_feature_size, _ = image_data.shape
269 - image_feature_sizes = [image_feature_size]
270 - pixel_values = None
271 -
272 - # multi-image image embeddings
273 - elif is_list_of(image_data, torch.Tensor):
274 -
275 - image_feature_sizes = []
276 - for image_embed in image_data:
277 - _, image_feature_size, _ = image_embed.shape
278 - image_feature_sizes.append(image_feature_size)
279 - pixel_values = None
271 + dynamic_image_size: Optional[bool] = None,
272 + use_thumbnail: Optional[bool] = None,
273 + ) -> tuple[int, int]:
274 + min_dynamic_patch = self.min_dynamic_patch
275 + max_dynamic_patch = (self.max_dynamic_patch if max_dynamic_patch
276 + is None else max_dynamic_patch)
277 + dynamic_image_size = (self.dynamic_image_size if dynamic_image_size
278 + is None else dynamic_image_size)
279 + use_thumbnail = (self.use_thumbnail
280 + if use_thumbnail is None else use_thumbnail)
281 +
282 + return resolve_h2ovl_min_max_num(
283 + min_dynamic_patch=min_dynamic_patch,
284 + max_dynamic_patch=max_dynamic_patch,
285 + dynamic_image_size=dynamic_image_size,
286 + use_thumbnail=use_thumbnail,
287 + )
288
281 - else:
282 - raise TypeError(f"Invalid image type: {type(image_data)}")
289 + def resolve_target_ratios(
290 + self,
291 + *,
292 + max_dynamic_patch: Optional[int] = None,
293 + dynamic_image_size: Optional[bool] = None,
294 + use_thumbnail: Optional[bool] = None,
295 + prior_aspect_ratio: Optional[tuple[int, int]] = None,
296 + ) -> list[tuple[int, int]]:
297 + min_num, max_num = self.resolve_min_max_num(
298 + max_dynamic_patch=max_dynamic_patch,
299 + dynamic_image_size=dynamic_image_size,
300 + use_thumbnail=use_thumbnail,
301 + )
302 + if prior_aspect_ratio: # hardcoded value for second pass of use_msac
303 + min_num = 3
304
284 - tokenizer = cached_get_tokenizer(
285 - model_config.tokenizer,
286 - trust_remote_code=model_config.trust_remote_code,
305 + return get_h2ovl_target_ratios(
306 + min_num,
307 + max_num,
308 + prior_aspect_ratio=prior_aspect_ratio,
309 )
310
289 - prompt = inputs.get("prompt")
290 - prompt_token_ids = inputs["prompt_token_ids"]
291 - if prompt is None:
292 - prompt = tokenizer.decode(prompt_token_ids)
293 -
294 - new_prompt = self._expand_image_prompt(prompt, image_feature_sizes,
295 - num_patches)
296 - new_prompt_token_ids = tokenizer.encode(new_prompt)
297 -
298 - # Wrap image processing in input_processor to avoid duplication
299 - image_token_id = tokenizer.encode(
300 - self.img_context_token,
301 - add_special_tokens=False,
302 - return_tensors="pt",
303 - )[0]
304 -
305 - # Update multi_modal_data to return
306 - if pixel_values is not None:
307 - multi_modal_data = {
308 - "image": {
309 - "pixel_values": pixel_values,
310 - "image_token_id": image_token_id,
311 - }
312 - }
311 + def get_num_image_tokens(
312 + self,
313 + *,
314 + image_width: int,
315 + image_height: int,
316 + use_msac: Optional[bool] = None,
317 + ) -> int:
318 + use_msac = (self.use_msac if use_msac is None else use_msac)
319 +
320 + use_thumbnail = self.use_thumbnail
321 +
322 + if use_msac:
323 + target_ratios_1 = self.resolve_target_ratios(
324 + use_thumbnail=False, # Applied in calculate_targets
325 + )
326 + num_patches_1, _, _, aspect_ratio_1 = calculate_h2ovl_targets(
327 + orig_width=image_width,
328 + orig_height=image_height,
329 + image_size=self.image_size,
330 + target_ratios=target_ratios_1,
331 + use_thumbnail=True,
332 + )
333 +
334 + target_ratios_2 = self.resolve_target_ratios(
335 + use_thumbnail=False, # Applied in calculate_targets
336 + prior_aspect_ratio=aspect_ratio_1,
337 + )
338 + num_patches_2, _, _, _ = calculate_h2ovl_targets(
339 + orig_width=image_width,
340 + orig_height=image_height,
341 + image_size=self.image_size,
342 + target_ratios=target_ratios_2,
343 + use_thumbnail=True,
344 + )
345 +
346 + num_patches = num_patches_1 + num_patches_2 - 1
347 else:
314 - multi_modal_data = {"image": {"image_embeds": image_data}}
348 + target_ratios = self.resolve_target_ratios(
349 + use_thumbnail=False, # Applied in calculate_targets
350 + )
351 + num_patches, _, _, _ = calculate_h2ovl_targets(
352 + orig_width=image_width,
353 + orig_height=image_height,
354 + image_size=self.image_size,
355 + target_ratios=target_ratios,
356 + use_thumbnail=use_thumbnail,
357 + )
358 +
359 + return num_patches * self.num_image_token
360
316 - return token_inputs(
317 - prompt=prompt,
318 - prompt_token_ids=new_prompt_token_ids,
319 - multi_modal_data=multi_modal_data,
361 + def _images_to_pixel_values_lst(
362 + self,
363 + images: list[Image.Image],
364 + max_dynamic_patch: Optional[int] = None,
365 + dynamic_image_size: Optional[bool] = None,
366 + ) -> list[torch.Tensor]:
367 + use_msac = self.use_msac if len(images) == 1 else False
368 +
369 + min_num, max_num = self.resolve_min_max_num(
370 + max_dynamic_patch=max_dynamic_patch,
371 + dynamic_image_size=dynamic_image_size,
372 + use_thumbnail=False, # Applied in image_to_pixel_values
373 )
374
322 - def input_mapper(
375 + return [
376 + image_to_pixel_values_h2ovl(
377 + image,
378 + input_size=self.image_size,
379 + min_num=min_num,
380 + max_num=max_num,
381 + use_thumbnail=self.use_thumbnail,
382 + use_msac=use_msac,
383 + ) for image in images
384 + ]
385 +
386 +
387 +class H2OVLProcessingInfo(BaseInternVLProcessingInfo):
388 +
389 + def get_hf_processor(
390 self,
324 - ctx: InputContext,
325 - data: object,
391 *,
392 max_dynamic_patch: Optional[int] = None,
328 - ) -> MultiModalKwargs:
393 + dynamic_image_size: Optional[bool] = None,
394 + ) -> H2OVLProcessor:
395 + return H2OVLProcessor(
396 + self.get_hf_config(),
397 + self.get_tokenizer(),
398 + max_dynamic_patch=max_dynamic_patch,
399 + dynamic_image_size=dynamic_image_size,
400 + )
401 +
402 + def get_mm_max_tokens_per_item(
403 + self,
404 + seq_len: int,
405 + mm_counts: Mapping[str, int],
406 + ) -> Mapping[str, int]:
407 + max_tokens_one_image = self.get_max_image_tokens(use_msac=None)
408 + if mm_counts.get("image", 0) <= 1:
409 + max_tokens_per_image = max_tokens_one_image
410 + else:
411 + max_tokens_per_image = self.get_max_image_tokens(use_msac=False)
412 +
413 + return {"image": max_tokens_per_image}
414
330 - # NOTE: Preprocessing for the image data is done in the
331 - # 'input_processor' function during actual inference.
332 - if isinstance(data, dict):
333 - return MultiModalKwargs(data)
415 + def get_num_image_tokens(
416 + self,
417 + *,
418 + image_width: int,
419 + image_height: int,
420 + processor: Optional[H2OVLProcessor],
421 + use_msac: Optional[bool] = None,
422 + ) -> int:
423 + if processor is None:
424 + processor = self.get_hf_processor()
425 +
426 + return processor.get_num_image_tokens(
427 + image_width=image_width,
428 + image_height=image_height,
429 + use_msac=use_msac,
430 + )
431
335 - # The section below is only used with dummy data during
336 - # memory profiling.
337 - hf_config = ctx.get_hf_config()
432 + def get_max_image_tokens(self, use_msac: Optional[bool] = None) -> int:
433 + target_width, target_height = self.get_image_size_with_most_features()
434
339 - image_pixel_values_mapper = image_to_pixel_values_wrapper(
340 - hf_config, max_dynamic_patch)
435 + return self.get_num_image_tokens(
436 + image_width=target_width,
437 + image_height=target_height,
438 + processor=None,
439 + use_msac=use_msac,
440 + )
441
342 - if isinstance(data, Image.Image):
343 - pixel_values = image_pixel_values_mapper(data)
344 - pixel_values = pixel_values.unsqueeze(0)
442
346 - elif is_list_of(data, Image.Image):
347 - hf_config.use_msac = False
348 - pixel_values = [image_pixel_values_mapper(img) for img in data]
443 +class H2OVLMultiModalProcessor(InternVLMultiModalProcessor[H2OVLProcessingInfo]
444 + ):
445 +
446 + def __init__(self,
447 + info: H2OVLProcessingInfo,
448 + dummy_inputs: "BaseDummyInputsBuilder[H2OVLProcessingInfo]",
449 + *,
450 + cache: Optional[ProcessingCache] = None,
451 + enable_sanity_checks: bool = True) -> None:
452 + super().__init__(
453 + info,
454 + dummy_inputs,
455 + cache=cache,
456 + enable_sanity_checks=enable_sanity_checks,
457 + )
458 +
459 + if self.cache is not None:
460 + # The processor output depends on the number of images passed,
461 + # making it incompatible with processing cache which is supposed
462 + # to be invariant of how many images are passed per prompt
463 + self.cache = None
464 + logger.warning_once(
465 + f"{type(self).__name__} does not support processing cache.")
466
467 + def _get_prompt_replacements(
468 + self,
469 + mm_items: MultiModalDataItems,
470 + hf_processor_mm_kwargs: Mapping[str, object],
471 + out_mm_kwargs: MultiModalKwargs,
472 + ) -> list[PromptReplacement]:
473 + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
474 +
475 + if "image_num_patches" in out_mm_kwargs:
476 + image_num_patches = out_mm_kwargs["image_num_patches"]
477 + assert isinstance(image_num_patches, torch.Tensor)
478 + image_num_patches = image_num_patches.tolist()
479 + elif "image_embeds" in out_mm_kwargs:
480 + # TODO: Use image size information in dictionary embedding inputs
481 + # to compute num_patches (similar to Qwen2-VL)
482 + image_num_patches = [None] * len(out_mm_kwargs["image_embeds"])
483 else:
351 - return MultiModalKwargs({"image_embeds": data})
352 - model_config = ctx.model_config
353 - tokenizer = cached_get_tokenizer(
354 - model_config.tokenizer,
355 - trust_remote_code=model_config.trust_remote_code,
356 - )
357 - image_token_id = tokenizer.encode(
358 - self.img_context_token,
359 - add_special_tokens=False,
360 - return_tensors="pt",
361 - )[0]
484 + image_num_patches = []
485 +
486 + num_images = len(image_num_patches)
487
363 - return MultiModalKwargs({
364 - "pixel_values": pixel_values,
365 - "image_token_id": image_token_id
366 - })
488 + def get_replacement_internvl(item_idx: int):
489 + images = mm_items.get_items(
490 + "image", (ImageEmbeddingItems, ImageProcessorItems))
491
492 + if isinstance(images, ImageEmbeddingItems):
493 + feature_size = images.get_feature_size(item_idx)
494 + else:
495 + image_size = images.get_image_size(item_idx)
496 + feature_size = self.info.get_num_image_tokens(
497 + image_width=image_size.width,
498 + image_height=image_size.height,
499 + processor=hf_processor,
500 + use_msac=None if num_images == 1 else False,
501 + )
502 +
503 + num_patches = image_num_patches[item_idx]
504 + if num_patches is not None:
505 + assert isinstance(num_patches, int)
506 +
507 + return PromptReplacementDetails(
508 + full=hf_processor.get_image_repl_full(feature_size,
509 + num_patches),
510 + features=hf_processor.get_image_repl_features(
511 + feature_size, num_patches),
512 + )
513
369 -input_pipeline = H2OVLInputPipeline(IMG_START, IMG_END, IMG_CONTEXT)
514 + return [
515 + PromptReplacement(
516 + modality="image",
517 + target="<image>",
518 + replacement=get_replacement_internvl,
519 + )
520 + ]
521
522
372 -@MULTIMODAL_REGISTRY.register_image_input_mapper(input_pipeline.input_mapper)
373 -@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_internvl_image_tokens)
374 -@INPUT_REGISTRY.register_dummy_data(input_pipeline.dummy_data)
375 -@INPUT_REGISTRY.register_input_processor(input_pipeline.input_processor)
523 +@MULTIMODAL_REGISTRY.register_processor(
524 + H2OVLMultiModalProcessor,
525 + info=H2OVLProcessingInfo,
526 + dummy_inputs=InternVLDummyInputsBuilder)
527 class H2OVLChatModel(InternVLChatModel):
528
529 def _init_vision_model(
vllm/model_executor/models/internvl.py
+503 -320
@@ -6,35 +6,37 @@
6 # Copyright (c) 2023 OpenGVLab
7 # Licensed under The MIT License [see LICENSE for details]
8 # --------------------------------------------------------
9 -import re
10 -from functools import cached_property, partial
9 +from abc import ABC, abstractmethod
10 +from functools import cached_property
11 from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
12 - TypedDict, Union)
12 + TypedDict, TypeVar, Union)
13
14 import torch
15 import torch.nn as nn
16 import torchvision.transforms as T
17 from PIL import Image
18 -from transformers import PretrainedConfig
18 +from transformers import BatchFeature, PretrainedConfig, TensorType
19
20 from vllm.attention import AttentionMetadata
21 from vllm.config import VllmConfig
22 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
23 - InputContext, token_inputs)
22 from vllm.model_executor.layers.quantization import QuantizationConfig
23 from vllm.model_executor.layers.quantization.awq import AWQConfig
24 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
25 from vllm.model_executor.models.intern_vit import (InternVisionModel,
26 InternVisionPatchModel)
27 from vllm.model_executor.sampling_metadata import SamplingMetadata
30 -from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
31 -from vllm.multimodal.inputs import NestedTensors, PlaceholderRange
32 -from vllm.multimodal.utils import cached_get_tokenizer
28 +from vllm.multimodal import MULTIMODAL_REGISTRY
29 +from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
30 + NestedTensors)
31 +from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
32 + ImageSize, MultiModalDataItems)
33 +from vllm.multimodal.processing import (BaseMultiModalProcessor,
34 + BaseProcessingInfo, PromptReplacement,
35 + PromptReplacementDetails)
36 +from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
37 from vllm.sequence import IntermediateTensors
34 -from vllm.utils import is_list_of
38 +from vllm.transformers_utils.tokenizer import AnyTokenizer
39
36 -from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
37 - get_clip_num_patches)
40 from .interfaces import SupportsMultiModal, SupportsPP
41 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
42 maybe_prefix, merge_multimodal_embeddings)
@@ -75,22 +77,27 @@ InternVLImageInputs = Union[InternVLImagePixelInputs,
77 InternVLImageEmbeddingInputs]
78
79
78 -# copied from https://huggingface.co/OpenGVLab/InternVL2-1B
79 -def build_transform(input_size):
80 +# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
81 +def build_transform(input_size: int):
82 MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
81 - transform = T.Compose([
83 + return T.Compose([
84 T.Lambda(lambda img: img.convert('RGB') if img.mode != 'RGB' else img),
85 T.Resize((input_size, input_size),
86 interpolation=T.InterpolationMode.BICUBIC),
87 T.ToTensor(),
88 T.Normalize(mean=MEAN, std=STD)
89 ])
88 - return transform
90
91
91 -# copied from https://huggingface.co/OpenGVLab/InternVL2-1B
92 -def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height,
93 - image_size):
92 +# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
93 +def find_closest_aspect_ratio(
94 + aspect_ratio: float,
95 + target_ratios: list[tuple[int, int]],
96 + *,
97 + width: int,
98 + height: int,
99 + image_size: int,
100 +) -> tuple[int, int]:
101 best_ratio_diff = float('inf')
102 best_ratio = (1, 1)
103 area = width * height
@@ -106,67 +113,82 @@ def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height,
113 return best_ratio
114
115
109 -def calculate_num_blocks(orig_width: int, orig_height: int, min_num: int,
110 - max_num: int, image_size: int,
111 - use_thumbnail: bool) -> Tuple[int, int, int]:
112 - aspect_ratio = orig_width / orig_height
116 +def resolve_internvl_min_max_num(
117 + *,
118 + min_dynamic_patch: int,
119 + max_dynamic_patch: int,
120 + dynamic_image_size: bool,
121 + use_thumbnail: bool,
122 +) -> tuple[int, int]:
123 + max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
124 +
125 + if use_thumbnail and max_dynamic_patch != 1:
126 + max_dynamic_patch += 1
127 +
128 + return min_dynamic_patch, max_dynamic_patch
129 +
130
114 - # calculate the existing image aspect ratio
115 - target_ratios = set((i, j) for n in range(min_num, max_num + 1)
116 - for i in range(1, n + 1) for j in range(1, n + 1)
117 - if i * j <= max_num and i * j >= min_num)
118 - target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
131 +def get_internvl_target_ratios(
132 + min_num: int,
133 + max_num: int,
134 +) -> list[tuple[int, int]]:
135 + target_ratios = {(i, j)
136 + for n in range(min_num, max_num + 1)
137 + for i in range(1, n + 1)
138 + for j in range(1, n + 1) if min_num <= i * j <= max_num}
139 + return sorted(target_ratios, key=lambda x: x[0] * x[1])
140 +
141 +
142 +def calculate_internvl_targets(
143 + *,
144 + orig_width: int,
145 + orig_height: int,
146 + target_ratios: list[tuple[int, int]],
147 + image_size: int,
148 + use_thumbnail: bool,
149 +) -> tuple[int, int, int]:
150 + aspect_ratio = orig_width / orig_height
151
152 # find the closest aspect ratio to the target
121 - target_aspect_ratio = find_closest_aspect_ratio(aspect_ratio,
122 - target_ratios, orig_width,
123 - orig_height, image_size)
153 + target_aspect_ratio = find_closest_aspect_ratio(
154 + aspect_ratio,
155 + target_ratios,
156 + width=orig_width,
157 + height=orig_height,
158 + image_size=image_size,
159 + )
160
161 # calculate the target width and height
162 target_width = image_size * target_aspect_ratio[0]
163 target_height = image_size * target_aspect_ratio[1]
164 blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
129 - # add thumbnail image if num_blocks > 1
130 - if use_thumbnail and blocks > 1:
131 - blocks += 1
132 - return blocks, target_width, target_height
133 -
165
135 -def calculate_num_blocks_wrapper(
136 - hf_config: PretrainedConfig,
137 - max_dynamic_patch: Optional[int] = None,
138 - dynamic_image_size: Optional[bool] = None,
139 -):
140 - if dynamic_image_size is None:
141 - dynamic_image_size = hf_config.dynamic_image_size
166 + # add thumbnail image if num_blocks != 1
167 + if use_thumbnail and blocks != 1:
168 + blocks += 1
169
143 - max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
144 - if max_dynamic_patch is None:
145 - max_dynamic_patch = hf_config.max_dynamic_patch
146 - min_num = hf_config.min_dynamic_patch
147 - image_size = hf_config.vision_config.image_size
148 - use_thumbnail = hf_config.use_thumbnail
149 - return partial(calculate_num_blocks,
150 - min_num=min_num,
151 - max_num=max_dynamic_patch,
152 - image_size=image_size,
153 - use_thumbnail=use_thumbnail)
170 + return blocks, target_width, target_height
171
172
173 # adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
157 -def dynamic_preprocess(image: Image.Image, min_num: int, max_num: int,
158 - image_size: int,
159 - use_thumbnail: bool) -> List[Image.Image]:
174 +def dynamic_preprocess_internvl(
175 + image: Image.Image,
176 + *,
177 + target_ratios: list[tuple[int, int]],
178 + image_size: int,
179 + use_thumbnail: bool,
180 +) -> list[Image.Image]:
181 orig_width, orig_height = image.size
182
183 # calculate the number of blocks without thumbnail
163 - blocks, target_width, target_height = calculate_num_blocks(
164 - orig_width,
165 - orig_height,
166 - min_num,
167 - max_num,
168 - image_size,
169 - use_thumbnail=False)
184 + blocks, target_width, target_height = calculate_internvl_targets(
185 + orig_width=orig_width,
186 + orig_height=orig_height,
187 + target_ratios=target_ratios,
188 + image_size=image_size,
189 + use_thumbnail=False,
190 + )
191 +
192 # resize the image
193 resized_img = image.resize((target_width, target_height))
194 processed_images = []
@@ -178,301 +200,463 @@ def dynamic_preprocess(image: Image.Image, min_num: int, max_num: int,
200 # split the image
201 split_img = resized_img.crop(box)
202 processed_images.append(split_img)
203 +
204 assert len(processed_images) == blocks
205 +
206 if use_thumbnail and len(processed_images) != 1:
207 thumbnail_img = image.resize((image_size, image_size))
208 processed_images.append(thumbnail_img)
209 +
210 return processed_images
211
212
213 # adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
189 -def image_to_pixel_values(image: Image.Image, input_size: int, min_num: int,
190 - max_num: int, use_thumbnail: bool) -> torch.Tensor:
214 +def image_to_pixel_values_internvl(
215 + image: Image.Image,
216 + *,
217 + input_size: int,
218 + min_num: int,
219 + max_num: int,
220 + use_thumbnail: bool,
221 +) -> torch.Tensor:
222 + target_ratios = get_internvl_target_ratios(min_num, max_num)
223 +
224 transform = build_transform(input_size=input_size)
192 - images = dynamic_preprocess(image,
193 - min_num=min_num,
194 - max_num=max_num,
195 - image_size=input_size,
196 - use_thumbnail=use_thumbnail)
197 - pixel_values = [transform(image) for image in images]
198 - pixel_values = torch.stack(pixel_values)
225 + images = dynamic_preprocess_internvl(
226 + image,
227 + target_ratios=target_ratios,
228 + image_size=input_size,
229 + use_thumbnail=use_thumbnail,
230 + )
231 +
232 + pixel_values = torch.stack([transform(image) for image in images])
233 return pixel_values
234
235
202 -def image_to_pixel_values_wrapper(
203 - hf_config: PretrainedConfig,
204 - max_dynamic_patch: Optional[int] = None,
205 - dynamic_image_size: Optional[bool] = None,
206 -):
207 - image_size = hf_config.vision_config.image_size
208 - min_num = hf_config.min_dynamic_patch
209 - if dynamic_image_size is None:
210 - dynamic_image_size = hf_config.dynamic_image_size
236 +class BaseInternVLProcessor(ABC):
237 + """
238 + This model doesn't define its own HF processor,
239 + so we implement our own one here.
240
212 - max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
213 - if max_dynamic_patch is None:
214 - max_dynamic_patch = hf_config.max_dynamic_patch
215 - use_thumbnail = hf_config.use_thumbnail
216 - return partial(image_to_pixel_values,
217 - input_size=image_size,
218 - min_num=min_num,
219 - max_num=max_dynamic_patch,
220 - use_thumbnail=use_thumbnail)
221 -
222 -
223 -def get_internvl_num_patches(hf_config: PretrainedConfig):
224 - vision_config = hf_config.vision_config
225 - downsample_ratio = hf_config.downsample_ratio
226 - image_size = vision_config.image_size
227 - patch_size = vision_config.patch_size
228 - return int(
229 - get_clip_num_patches(image_size=image_size, patch_size=patch_size) *
230 - (downsample_ratio**2))
231 -
232 -
233 -def get_max_internvl_image_tokens(
234 - ctx: InputContext,
235 - *,
236 - max_dynamic_patch: Optional[int] = None,
237 - dynamic_image_size: Optional[bool] = None,
238 -):
239 - hf_config = ctx.get_hf_config()
240 - if dynamic_image_size is None:
241 - dynamic_image_size = hf_config.dynamic_image_size
241 + The code to insert image tokens is based on:
242 + https://huggingface.co/OpenGVLab/InternVL2-1B/blob/main/modeling_internvl_chat.py#L252
243 + """
244
243 - max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
244 - if max_dynamic_patch is None:
245 - max_dynamic_patch = hf_config.max_dynamic_patch
246 - use_thumbnail = hf_config.use_thumbnail
247 - if use_thumbnail and max_dynamic_patch > 1:
248 - max_dynamic_patch += 1
245 + def __init__(
246 + self,
247 + config: PretrainedConfig,
248 + tokenizer: AnyTokenizer,
249 + *,
250 + max_dynamic_patch: Optional[int] = None,
251 + dynamic_image_size: Optional[bool] = None,
252 + ) -> None:
253 + super().__init__()
254
250 - num_patches = get_internvl_num_patches(hf_config)
251 - return num_patches * max_dynamic_patch
255 + self.config = config
256 + self.tokenizer = tokenizer
257
258 + image_size: int = config.vision_config.image_size
259 + patch_size: int = config.vision_config.patch_size
260
254 -def get_max_internvl_image_size(
255 - ctx: InputContext,
256 - *,
257 - max_dynamic_patch: Optional[int] = None,
258 - dynamic_image_size: Optional[bool] = None,
259 -):
260 - hf_config = ctx.get_hf_config()
261 - image_size = hf_config.vision_config.image_size
262 - if dynamic_image_size is None:
263 - dynamic_image_size = hf_config.dynamic_image_size
261 + if dynamic_image_size is None:
262 + dynamic_image_size = config.dynamic_image_size
263 + assert isinstance(dynamic_image_size, bool)
264
265 - max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
266 - if max_dynamic_patch is None:
267 - max_dynamic_patch = hf_config.max_dynamic_patch
268 - use_thumbnail = hf_config.use_thumbnail
269 - if use_thumbnail and max_dynamic_patch > 1:
270 - max_dynamic_patch += 1
271 - width = image_size * max_dynamic_patch
272 - height = image_size
273 - return width, height
265 + if max_dynamic_patch is None:
266 + max_dynamic_patch = config.max_dynamic_patch
267 + assert isinstance(max_dynamic_patch, int)
268
269 + self.num_image_token = int(
270 + (image_size // patch_size)**2 * (config.downsample_ratio**2))
271 + self.image_size = image_size
272 + self.min_dynamic_patch: int = config.min_dynamic_patch
273 + self.max_dynamic_patch = max_dynamic_patch
274 + self.dynamic_image_size = dynamic_image_size
275 + self.use_thumbnail: bool = config.use_thumbnail
276 +
277 + @property
278 + @abstractmethod
279 + def image_token_id(self) -> int:
280 + raise NotImplementedError
281 +
282 + @abstractmethod
283 + def get_image_repl_features(
284 + self,
285 + feature_size: int,
286 + num_patches: Optional[int],
287 + ) -> str:
288 + raise NotImplementedError
289
276 -class InternVLInputPipeline:
290 + @abstractmethod
291 + def get_image_repl_full(
292 + self,
293 + feature_size: int,
294 + num_patches: Optional[int],
295 + ) -> str:
296 + raise NotImplementedError
297
278 - def __init__(
298 + def resolve_min_max_num(
299 self,
280 - img_start_token: str,
281 - img_end_token: str,
282 - img_context_token: str,
283 - ) -> None:
284 - super().__init__()
300 + *,
301 + max_dynamic_patch: Optional[int] = None,
302 + dynamic_image_size: Optional[bool] = None,
303 + use_thumbnail: Optional[bool] = None,
304 + ) -> tuple[int, int]:
305 + min_dynamic_patch = self.min_dynamic_patch
306 + max_dynamic_patch = (self.max_dynamic_patch if max_dynamic_patch
307 + is None else max_dynamic_patch)
308 + dynamic_image_size = (self.dynamic_image_size if dynamic_image_size
309 + is None else dynamic_image_size)
310 + use_thumbnail = (self.use_thumbnail
311 + if use_thumbnail is None else use_thumbnail)
312 +
313 + return resolve_internvl_min_max_num(
314 + min_dynamic_patch=min_dynamic_patch,
315 + max_dynamic_patch=max_dynamic_patch,
316 + dynamic_image_size=dynamic_image_size,
317 + use_thumbnail=use_thumbnail,
318 + )
319
286 - self.img_start_token = img_start_token
287 - self.img_end_token = img_end_token
288 - self.img_context_token = img_context_token
320 + def resolve_target_ratios(
321 + self,
322 + *,
323 + max_dynamic_patch: Optional[int] = None,
324 + dynamic_image_size: Optional[bool] = None,
325 + use_thumbnail: Optional[bool] = None,
326 + ) -> list[tuple[int, int]]:
327 + min_num, max_num = self.resolve_min_max_num(
328 + max_dynamic_patch=max_dynamic_patch,
329 + dynamic_image_size=dynamic_image_size,
330 + use_thumbnail=use_thumbnail,
331 + )
332
290 - def _create_image_prompt(self, feature_size: int, num_patches: int) -> str:
291 - return (self.img_start_token + self.img_context_token * feature_size +
292 - self.img_end_token)
333 + return get_internvl_target_ratios(min_num, max_num)
334
294 - def _expand_image_prompt(
335 + def get_num_image_tokens(
336 self,
296 - prompt: str,
297 - feature_sizes: List[int],
298 - num_patches: int,
299 - ) -> str:
300 - image_idx = sorted(
301 - map(int, re.findall(r"Image-(\d+): <image>\n", prompt)))
337 + *,
338 + image_width: int,
339 + image_height: int,
340 + ) -> int:
341 + target_ratios = self.resolve_target_ratios(
342 + use_thumbnail=False, # Applied in calculate_targets
343 + )
344 +
345 + num_patches, _, _ = calculate_internvl_targets(
346 + orig_width=image_width,
347 + orig_height=image_height,
348 + image_size=self.image_size,
349 + target_ratios=target_ratios,
350 + use_thumbnail=self.use_thumbnail,
351 + )
352
303 - new_prompt = prompt
304 - for idx, feature_size in enumerate(feature_sizes, start=1):
305 - image_prompt = self._create_image_prompt(feature_size, num_patches)
306 - if not image_idx:
307 - image_prompt = f"Image-{idx}: {image_prompt}"
353 + return num_patches * self.num_image_token
354
309 - new_prompt = new_prompt.replace('<image>', image_prompt, 1)
355 + def _images_to_pixel_values_lst(
356 + self,
357 + images: list[Image.Image],
358 + max_dynamic_patch: Optional[int] = None,
359 + dynamic_image_size: Optional[bool] = None,
360 + ) -> list[torch.Tensor]:
361 + min_num, max_num = self.resolve_min_max_num(
362 + max_dynamic_patch=max_dynamic_patch,
363 + dynamic_image_size=dynamic_image_size,
364 + use_thumbnail=False, # Applied in image_to_pixel_values
365 + )
366
311 - return new_prompt
367 + return [
368 + image_to_pixel_values_internvl(
369 + image,
370 + input_size=self.image_size,
371 + min_num=min_num,
372 + max_num=max_num,
373 + use_thumbnail=self.use_thumbnail,
374 + ) for image in images
375 + ]
376
313 - def input_processor(
377 + def __call__(
378 self,
315 - ctx: InputContext,
316 - inputs: DecoderOnlyInputs,
317 - *,
379 + text: Optional[Union[str, list[str]]] = None,
380 + images: Optional[Union[Image.Image, list[Image.Image]]] = None,
381 max_dynamic_patch: Optional[int] = None,
382 dynamic_image_size: Optional[bool] = None,
320 - ) -> DecoderOnlyInputs:
321 - multi_modal_data = inputs.get("multi_modal_data")
322 - if multi_modal_data is None or "image" not in multi_modal_data:
323 - return inputs
324 -
325 - model_config = ctx.model_config
326 - hf_config = ctx.get_hf_config()
327 -
328 - image_data = multi_modal_data["image"]
329 - num_patches = get_internvl_num_patches(hf_config)
330 - num_blocks_calculator = calculate_num_blocks_wrapper(
331 - hf_config, max_dynamic_patch, dynamic_image_size)
332 - if isinstance(image_data, Image.Image):
333 - width, height = image_data.size
334 - num_blocks, _, _ = num_blocks_calculator(width, height)
335 - image_feature_sizes = [num_blocks * num_patches]
336 - elif is_list_of(image_data, Image.Image):
337 - image_feature_sizes = []
338 - for image in image_data:
339 - width, height = image.size
340 - num_blocks, _, _ = num_blocks_calculator(width, height)
341 - image_feature_sizes.append(num_blocks * num_patches)
342 - elif isinstance(image_data, torch.Tensor):
343 - num_images, image_feature_size, hidden_size = image_data.shape
344 - image_feature_sizes = [image_feature_size]
383 + return_tensors: Optional[Union[str, TensorType]] = None,
384 + ) -> BatchFeature:
385 + if text is None:
386 + text = []
387 + if not isinstance(text, list):
388 + text = [text]
389 + if images is None:
390 + images = []
391 + if not isinstance(images, list):
392 + images = [images]
393 +
394 + if len(images) == 0:
395 + image_inputs = {}
396 else:
346 - raise TypeError(f"Invalid image type: {type(image_data)}")
347 -
348 - tokenizer = cached_get_tokenizer(
349 - model_config.tokenizer,
350 - trust_remote_code=model_config.trust_remote_code)
351 -
352 - prompt = inputs.get("prompt")
353 - prompt_token_ids = inputs["prompt_token_ids"]
354 - if prompt is None:
355 - prompt = tokenizer.decode(prompt_token_ids)
356 -
357 - new_prompt = self._expand_image_prompt(prompt, image_feature_sizes,
358 - num_patches)
359 - new_prompt_token_ids = tokenizer.encode(new_prompt)
360 - img_context_token_id = tokenizer.encode(self.img_context_token,
361 - add_special_tokens=False)
362 - assert len(img_context_token_id) == 1, \
363 - (f"Invalid image token '{self.img_context_token}': A valid image "
364 - f"token encodes to a single token ID, got {img_context_token_id}.")
365 - img_context_token_id = img_context_token_id[0]
366 -
367 - # Get precise tracking of placeholder positions
368 - token_idx = image_idx = 0
369 - placeholder_ranges = []
370 - while token_idx < len(new_prompt_token_ids):
371 - if new_prompt_token_ids[token_idx] == img_context_token_id:
372 - curr_image_featue_size = image_feature_sizes[image_idx]
373 - placeholder_ranges.append(
374 - PlaceholderRange(offset=token_idx,
375 - length=curr_image_featue_size))
376 - image_idx += 1
377 - token_idx += curr_image_featue_size
378 - else:
379 - token_idx += 1
397 + pixel_values_lst = self._images_to_pixel_values_lst(
398 + images,
399 + max_dynamic_patch=max_dynamic_patch,
400 + dynamic_image_size=dynamic_image_size,
401 + )
402 + image_inputs = {
403 + "pixel_values_flat": torch.cat(pixel_values_lst),
404 + "image_num_patches": list(map(len, pixel_values_lst)),
405 + }
406 +
407 + for pixel_values in pixel_values_lst:
408 + num_patches = pixel_values.shape[0]
409 + feature_size = num_patches * self.num_image_token
410 +
411 + image_repl = self.get_image_repl_full(feature_size,
412 + num_patches)
413 + text = [t.replace('<image>', image_repl, 1) for t in text]
414 +
415 + text_inputs = self.tokenizer(text)
416 +
417 + return BatchFeature(
418 + {
419 + **text_inputs,
420 + **image_inputs,
421 + },
422 + tensor_type=return_tensors,
423 + )
424
381 - return token_inputs(
382 - prompt=prompt,
383 - prompt_token_ids=new_prompt_token_ids,
384 - multi_modal_data=multi_modal_data,
385 - multi_modal_placeholders={"image": placeholder_ranges})
425
387 - def input_mapper(
426 +class InternVLProcessor(BaseInternVLProcessor):
427 +
428 + @property
429 + def image_token_id(self) -> int:
430 + return self.tokenizer.get_vocab()[IMG_CONTEXT]
431 +
432 + def get_image_repl_features(
433 + self,
434 + feature_size: int,
435 + num_patches: Optional[int],
436 + ) -> str:
437 + return IMG_CONTEXT * feature_size
438 +
439 + def get_image_repl_full(
440 + self,
441 + feature_size: int,
442 + num_patches: Optional[int],
443 + ) -> str:
444 + features = self.get_image_repl_features(feature_size, num_patches)
445 + return IMG_START + features + IMG_END
446 +
447 +
448 +class BaseInternVLProcessingInfo(BaseProcessingInfo):
449 +
450 + @abstractmethod
451 + def get_hf_processor(
452 self,
389 - ctx: InputContext,
390 - data: object,
453 *,
454 max_dynamic_patch: Optional[int] = None,
455 dynamic_image_size: Optional[bool] = None,
394 - ):
395 - hf_config = ctx.get_hf_config()
396 -
397 - image_pixel_values_mapper = image_to_pixel_values_wrapper(
398 - hf_config, max_dynamic_patch, dynamic_image_size)
399 - if isinstance(data, Image.Image):
400 - data = image_pixel_values_mapper(data)
401 - # Add an N dimension for number of images per prompt (currently 1).
402 - data = data.unsqueeze(0)
403 - elif is_list_of(data, Image.Image):
404 - # we can't stack here because images may have different num_patches
405 - data = [image_pixel_values_mapper(img) for img in data]
406 - else:
407 - return MultiModalKwargs({"image_embeds": data})
408 - model_config = ctx.model_config
409 - tokenizer = cached_get_tokenizer(
410 - model_config.tokenizer,
411 - trust_remote_code=model_config.trust_remote_code)
412 - image_token_id = tokenizer.encode(self.img_context_token,
413 - add_special_tokens=False,
414 - return_tensors="pt")[0]
415 -
416 - return MultiModalKwargs({
417 - "pixel_values": data,
418 - "image_token_id": image_token_id
419 - })
420 -
421 - def dummy_data(
456 + ) -> BaseInternVLProcessor:
457 + raise NotImplementedError
458 +
459 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
460 + return {"image": None}
461 +
462 + def get_mm_max_tokens_per_item(
463 self,
423 - ctx: InputContext,
464 seq_len: int,
465 mm_counts: Mapping[str, int],
466 + ) -> Mapping[str, int]:
467 + return {"image": self.get_max_image_tokens()}
468 +
469 + def get_num_image_tokens(
470 + self,
471 *,
427 - max_dynamic_patch: Optional[int] = None,
428 - dynamic_image_size: Optional[bool] = None,
429 - ):
430 - num_images = mm_counts["image"]
472 + image_width: int,
473 + image_height: int,
474 + processor: Optional[BaseInternVLProcessor],
475 + ) -> int:
476 + if processor is None:
477 + processor = self.get_hf_processor()
478 +
479 + return processor.get_num_image_tokens(
480 + image_width=image_width,
481 + image_height=image_height,
482 + )
483
432 - hf_config = ctx.get_hf_config()
484 + def get_max_image_tokens(self) -> int:
485 + target_width, target_height = self.get_image_size_with_most_features()
486
434 - image_feature_size = get_max_internvl_image_tokens(
435 - ctx,
436 - max_dynamic_patch=max_dynamic_patch,
437 - dynamic_image_size=dynamic_image_size,
487 + return self.get_num_image_tokens(
488 + image_width=target_width,
489 + image_height=target_height,
490 + processor=None,
491 )
439 - model_config = ctx.model_config
440 - tokenizer = cached_get_tokenizer(
441 - model_config.tokenizer,
442 - trust_remote_code=model_config.trust_remote_code)
443 -
444 - seq_data, ranges = dummy_seq_data_for_clip(
445 - hf_config.vision_config,
446 - seq_len,
447 - num_images,
448 - image_token_id=tokenizer.encode(self.img_context_token,
449 - add_special_tokens=False)[0],
450 - image_feature_size_override=image_feature_size,
492 +
493 + def get_image_size_with_most_features(self) -> ImageSize:
494 + processor = self.get_hf_processor()
495 +
496 + base_size = processor.image_size
497 + target_ratios = processor.resolve_target_ratios()
498 +
499 + largest_feature_size, largest_feature_pinpoint = 0, None
500 + for wr, hr in target_ratios:
501 + width, height = base_size * wr, base_size * hr
502 +
503 + feat_size = self.get_num_image_tokens(
504 + image_width=width,
505 + image_height=height,
506 + processor=processor,
507 + )
508 + if feat_size > largest_feature_size:
509 + largest_feature_size = feat_size
510 + largest_feature_pinpoint = ImageSize(width=width,
511 + height=height)
512 +
513 + if largest_feature_size == 0 or largest_feature_pinpoint is None:
514 + raise ValueError("Cannot have a largest feature size of 0!")
515 +
516 + return largest_feature_pinpoint
517 +
518 +
519 +_I = TypeVar("_I", bound=BaseInternVLProcessingInfo)
520 +
521 +
522 +class InternVLDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
523 +
524 + def get_dummy_processor_inputs(
525 + self,
526 + seq_len: int,
527 + mm_counts: Mapping[str, int],
528 + ) -> ProcessorInputs:
529 + target_width, target_height = \
530 + self.info.get_image_size_with_most_features()
531 + num_images = mm_counts.get("image", 0)
532 +
533 + mm_data = {
534 + "image":
535 + self._get_dummy_images(width=target_width,
536 + height=target_height,
537 + num_images=num_images)
538 + }
539 +
540 + return ProcessorInputs(
541 + prompt_text="<image>" * num_images,
542 + mm_data=mm_data,
543 )
544
453 - max_image_width, max_image_height = get_max_internvl_image_size(
454 - ctx,
455 - max_dynamic_patch=max_dynamic_patch,
456 - dynamic_image_size=dynamic_image_size,
545 +
546 +class InternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
547 +
548 + def _call_hf_processor(
549 + self,
550 + prompt: str,
551 + mm_data: Mapping[str, object],
552 + mm_kwargs: Mapping[str, object],
553 + ) -> BatchFeature:
554 + processed_outputs = super()._call_hf_processor(
555 + prompt=prompt,
556 + mm_data=mm_data,
557 + mm_kwargs=mm_kwargs,
558 )
559
459 - mm_data = dummy_image_for_clip(
460 - hf_config.vision_config,
461 - num_images,
462 - image_width_override=max_image_width,
463 - image_height_override=max_image_height,
560 + image_token_id = self.info.get_hf_processor(**mm_kwargs).image_token_id
561 + image_data = mm_data.get("images", [])
562 + assert isinstance(image_data, list)
563 +
564 + # Since there may be extra tokens in the feature placeholders,
565 + # we need to pass the image token ID to the model to select the
566 + # tokens to merge from the vision encoder outputs
567 + processed_outputs["image_token_id"] = [image_token_id
568 + ] * len(image_data)
569 +
570 + return processed_outputs
571 +
572 + def _get_mm_fields_config(
573 + self,
574 + hf_inputs: BatchFeature,
575 + hf_processor_mm_kwargs: Mapping[str, object],
576 + ) -> Mapping[str, MultiModalFieldConfig]:
577 + image_num_patches = hf_inputs.get("image_num_patches", torch.empty(0))
578 +
579 + return dict(
580 + pixel_values_flat=MultiModalFieldConfig.flat_from_sizes(
581 + "image", image_num_patches),
582 + image_num_patches=MultiModalFieldConfig.batched("image"),
583 + image_embeds=MultiModalFieldConfig.batched("image"),
584 + image_token_id=MultiModalFieldConfig.batched("image"),
585 )
586
466 - return DummyData(seq_data, mm_data, ranges)
587 + def _get_prompt_replacements(
588 + self,
589 + mm_items: MultiModalDataItems,
590 + hf_processor_mm_kwargs: Mapping[str, object],
591 + out_mm_kwargs: MultiModalKwargs,
592 + ) -> list[PromptReplacement]:
593 + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
594 +
595 + if "image_num_patches" in out_mm_kwargs:
596 + image_num_patches = out_mm_kwargs["image_num_patches"]
597 + assert isinstance(image_num_patches, torch.Tensor)
598 + image_num_patches = image_num_patches.tolist()
599 + elif "image_embeds" in out_mm_kwargs:
600 + # TODO: Use image size information in dictionary embedding inputs
601 + # to compute num_patches (similar to Qwen2-VL)
602 + image_num_patches = [None] * len(out_mm_kwargs["image_embeds"])
603 + else:
604 + image_num_patches = []
605 +
606 + def get_replacement_internvl(item_idx: int):
607 + images = mm_items.get_items(
608 + "image", (ImageEmbeddingItems, ImageProcessorItems))
609 +
610 + if isinstance(images, ImageEmbeddingItems):
611 + feature_size = images.get_feature_size(item_idx)
612 + else:
613 + image_size = images.get_image_size(item_idx)
614 + feature_size = self.info.get_num_image_tokens(
615 + image_width=image_size.width,
616 + image_height=image_size.height,
617 + processor=hf_processor,
618 + )
619 +
620 + num_patches = image_num_patches[item_idx]
621 + if num_patches is not None:
622 + assert isinstance(num_patches, int)
623 +
624 + return PromptReplacementDetails(
625 + full=hf_processor.get_image_repl_full(feature_size,
626 + num_patches),
627 + features=hf_processor.get_image_repl_features(
628 + feature_size, num_patches),
629 + )
630
631 + return [
632 + PromptReplacement(
633 + modality="image",
634 + target="<image>",
635 + replacement=get_replacement_internvl,
636 + )
637 + ]
638
469 -input_pipeline = InternVLInputPipeline(IMG_START, IMG_END, IMG_CONTEXT)
639
640 +class InternVLProcessingInfo(BaseInternVLProcessingInfo):
641
472 -@MULTIMODAL_REGISTRY.register_image_input_mapper(input_pipeline.input_mapper)
473 -@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_internvl_image_tokens)
474 -@INPUT_REGISTRY.register_dummy_data(input_pipeline.dummy_data)
475 -@INPUT_REGISTRY.register_input_processor(input_pipeline.input_processor)
642 + def get_hf_processor(
643 + self,
644 + *,
645 + max_dynamic_patch: Optional[int] = None,
646 + dynamic_image_size: Optional[bool] = None,
647 + ) -> InternVLProcessor:
648 + return InternVLProcessor(
649 + self.get_hf_config(),
650 + self.get_tokenizer(),
651 + max_dynamic_patch=max_dynamic_patch,
652 + dynamic_image_size=dynamic_image_size,
653 + )
654 +
655 +
656 +@MULTIMODAL_REGISTRY.register_processor(
657 + InternVLMultiModalProcessor,
658 + info=InternVLProcessingInfo,
659 + dummy_inputs=InternVLDummyInputsBuilder)
660 class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
661
662 def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
@@ -621,11 +805,11 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
805
806 def _parse_and_validate_image_input(
807 self, **kwargs: object) -> Optional[InternVLImageInputs]:
624 - pixel_values = kwargs.pop("pixel_values", None)
625 - image_token_id = kwargs.pop("image_token_id", None)
808 + pixel_values_flat = kwargs.pop("pixel_values_flat", None)
809 + image_num_patches = kwargs.pop("image_num_patches", None)
810 image_embeds = kwargs.pop("image_embeds", None)
811
628 - if pixel_values is None and image_embeds is None:
812 + if pixel_values_flat is None and image_embeds is None:
813 return None
814
815 if image_embeds is not None:
@@ -638,31 +822,30 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
822 data=flatten_bn(image_embeds),
823 )
824
641 - self.img_context_token_id = image_token_id[0]
825 + image_token_id = kwargs["image_token_id"]
826 + assert isinstance(image_token_id, torch.Tensor)
827 + self.img_context_token_id = image_token_id.flatten().unique().item()
828
643 - if pixel_values is not None:
644 - if not isinstance(pixel_values, (torch.Tensor, list)):
829 + if pixel_values_flat is not None:
830 + if not isinstance(pixel_values_flat, (torch.Tensor, list)):
831 raise ValueError("Incorrect type of pixel values. "
646 - f"Got type: {type(pixel_values)}")
647 -
648 - patches_per_image = []
649 - for request_pixel_values in pixel_values:
650 - for image_pixel_values in request_pixel_values:
651 - patches_per_image.append(image_pixel_values.shape[0])
652 - # We need to flatten (B, N, P) to (B*N*P),
653 - # so we call flatten_bn twice.
832 + f"Got type: {type(pixel_values_flat)}")
833 +
834 + assert isinstance(image_num_patches, (torch.Tensor, list))
835 +
836 return InternVLImagePixelInputs(
837 type="pixel_values",
838 data=self._validate_pixel_values(
657 - flatten_bn(flatten_bn(pixel_values), concat=True)),
658 - patches_per_image=patches_per_image)
839 + flatten_bn(pixel_values_flat, concat=True)),
840 + patches_per_image=flatten_bn(image_num_patches,
841 + concat=True).tolist())
842
843 raise AssertionError("This line should be unreachable.")
844
845 def _process_image_input(
846 self,
847 image_input: InternVLImageInputs,
665 - ) -> Tuple[torch.Tensor]:
848 + ) -> tuple[torch.Tensor, ...]:
849 if image_input["type"] == "image_embeds":
850 return image_input["data"]
851
@@ -689,7 +872,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
872 image_embeds = image_embeds.split(image_feature_sizes)
873 return image_embeds
874
692 - def _set_visual_token_mask(self, input_ids: torch.Tensor) -> torch.Tensor:
875 + def _set_visual_token_mask(self, input_ids: torch.Tensor) -> None:
876 if self.is_mono:
877 self.visual_token_mask = (
878 input_ids == self.img_context_token_id).reshape(-1, 1)
vllm/model_executor/models/llava.py
+5 -1
@@ -125,7 +125,11 @@ class BaseLlavaProcessingInfo(BaseProcessingInfo):
125 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
126 return {"image": None}
127
128 - def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
128 + def get_mm_max_tokens_per_item(
129 + self,
130 + seq_len: int,
131 + mm_counts: Mapping[str, int],
132 + ) -> Mapping[str, int]:
133 return {"image": self.get_max_image_tokens()}
134
135 def _apply_feature_select_strategy(
vllm/model_executor/models/llava_next_video.py
+5 -1
@@ -62,7 +62,11 @@ class LlavaNextVideoProcessingInfo(BaseProcessingInfo):
62 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
63 return {"video": 1}
64
65 - def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
65 + def get_mm_max_tokens_per_item(
66 + self,
67 + seq_len: int,
68 + mm_counts: Mapping[str, int],
69 + ) -> Mapping[str, int]:
70 target_width, target_height = self.get_image_size_with_most_features()
71
72 max_video_tokens = self.get_num_video_tokens(
vllm/model_executor/models/llava_onevision.py
+5 -1
@@ -103,7 +103,11 @@ class LlavaOnevisionProcessingInfo(LlavaNextProcessingInfo):
103 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
104 return {"image": None, "video": None}
105
106 - def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
106 + def get_mm_max_tokens_per_item(
107 + self,
108 + seq_len: int,
109 + mm_counts: Mapping[str, int],
110 + ) -> Mapping[str, int]:
111 return {
112 "image": self.get_max_image_tokens(),
113 "video": self.get_max_video_tokens(seq_len),
vllm/model_executor/models/minicpmo.py
+13 -15
@@ -23,7 +23,6 @@
23 # limitations under the License.
24 """Inference-only MiniCPM-O model compatible with HuggingFace weights."""
25 from functools import partial
26 -from itertools import accumulate
26 from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional, Set,
27 Tuple, TypedDict, Union)
28
@@ -138,11 +137,15 @@ class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):
137 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
138 return {"image": None, "video": None, "audio": None}
139
141 - def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
140 + def get_mm_max_tokens_per_item(
141 + self,
142 + seq_len: int,
143 + mm_counts: Mapping[str, int],
144 + ) -> Mapping[str, int]:
145 return {
146 "image": self.get_max_image_tokens(),
147 "audio": self.get_max_audio_tokens(),
145 - "video": self.get_max_video_tokens(seq_len)
148 + "video": self.get_max_video_tokens(seq_len),
149 }
150
151 def get_default_audio_pool_step(self) -> int:
@@ -369,23 +372,18 @@ class MiniCPMOMultiModalProcessor(
372 hf_inputs,
373 hf_processor_mm_kwargs: Mapping[str, object],
374 ) -> Mapping[str, MultiModalFieldConfig]:
375 + audio_num_slices = hf_inputs.get("audio_num_slices", torch.empty(0))
376
373 - def get_slices(num_slices: List[int]) -> List[int]:
374 - slice_indices = [0] + list(accumulate(num_slices))
375 - slices = [(slice_indices[i], slice_indices[i + 1])
376 - for i in range(len(num_slices))]
377 - return [slice(*slice_item) for slice_item in slices]
378 -
379 - audio_slices = get_slices(
380 - hf_inputs.get("audio_num_slices", torch.empty(0)))
377 return dict(
378 **super()._get_mm_fields_config(hf_inputs, hf_processor_mm_kwargs),
383 - audio_features=MultiModalFieldConfig.flat("audio", audio_slices),
384 - audio_feature_lens=MultiModalFieldConfig.flat(
385 - "audio", audio_slices),
379 + audio_features=MultiModalFieldConfig.flat_from_sizes(
380 + "audio", audio_num_slices),
381 + audio_feature_lens=MultiModalFieldConfig.flat_from_sizes(
382 + "audio", audio_num_slices),
383 audio_num_slices=MultiModalFieldConfig.batched("audio"),
384 audio_orders_in_mm_data=MultiModalFieldConfig.batched("audio"),
388 - audio_embeds=MultiModalFieldConfig.flat("audio", audio_slices))
385 + audio_embeds=MultiModalFieldConfig.flat_from_sizes(
386 + "audio", audio_num_slices))
387
388
389 class MultiModalProjector(nn.Module):
vllm/model_executor/models/minicpmv.py
+24 -26
@@ -26,7 +26,6 @@ import math
26 import re
27 from collections import Counter
28 from functools import cached_property, partial
29 -from itertools import accumulate
29 from typing import (Any, Callable, Dict, Iterable, List, Literal, Mapping,
30 Optional, Set, Tuple, TypedDict, Union)
31
@@ -365,7 +364,11 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
364 else:
365 return {"image": None}
366
368 - def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
367 + def get_mm_max_tokens_per_item(
368 + self,
369 + seq_len: int,
370 + mm_counts: Mapping[str, int],
371 + ) -> Mapping[str, int]:
372 mm_max_tokens = {"image": self.get_max_image_tokens()}
373 if self.get_model_version() == (2, 6):
374 mm_max_tokens["video"] = self.get_max_video_tokens(seq_len)
@@ -761,30 +764,25 @@ class MiniCPMVMultiModalProcessor(
764 hf_inputs,
765 hf_processor_mm_kwargs: Mapping[str, object],
766 ) -> Mapping[str, MultiModalFieldConfig]:
764 -
765 - def get_slices(num_slices: List[int]) -> List[int]:
766 - slice_indices = [0] + list(accumulate(num_slices))
767 - slices = [(slice_indices[i], slice_indices[i + 1])
768 - for i in range(len(num_slices))]
769 - return [slice(*slice_item) for slice_item in slices]
770 -
771 - image_slices = get_slices(
772 - hf_inputs.get("image_num_slices", torch.empty(0)))
773 - video_slices = get_slices(
774 - hf_inputs.get("video_num_slices", torch.empty(0)))
775 -
776 - return dict(
777 - pixel_values=MultiModalFieldConfig.flat("image", image_slices),
778 - image_sizes=MultiModalFieldConfig.batched("image"),
779 - tgt_sizes=MultiModalFieldConfig.flat("image", image_slices),
780 - image_num_slices=MultiModalFieldConfig.batched("image"),
781 - image_embeds=MultiModalFieldConfig.flat("image", image_slices),
782 - video_pixel_values=MultiModalFieldConfig.flat(
783 - "video", video_slices),
784 - video_image_sizes=MultiModalFieldConfig.batched("video"),
785 - video_tgt_sizes=MultiModalFieldConfig.flat("video", video_slices),
786 - video_embeds=MultiModalFieldConfig.flat("video", video_slices),
787 - video_num_slices=MultiModalFieldConfig.batched("video"))
767 + image_num_slices = hf_inputs.get("image_num_slices", torch.empty(0))
768 + video_num_slices = hf_inputs.get("video_num_slices", torch.empty(0))
769 +
770 + return dict(pixel_values=MultiModalFieldConfig.flat_from_sizes(
771 + "image", image_num_slices),
772 + image_sizes=MultiModalFieldConfig.batched("image"),
773 + tgt_sizes=MultiModalFieldConfig.flat_from_sizes(
774 + "image", image_num_slices),
775 + image_num_slices=MultiModalFieldConfig.batched("image"),
776 + image_embeds=MultiModalFieldConfig.flat_from_sizes(
777 + "image", image_num_slices),
778 + video_pixel_values=MultiModalFieldConfig.flat_from_sizes(
779 + "video", video_num_slices),
780 + video_image_sizes=MultiModalFieldConfig.batched("video"),
781 + video_tgt_sizes=MultiModalFieldConfig.flat_from_sizes(
782 + "video", video_num_slices),
783 + video_embeds=MultiModalFieldConfig.flat_from_sizes(
784 + "video", video_num_slices),
785 + video_num_slices=MultiModalFieldConfig.batched("video"))
786
787 def apply(
788 self,
vllm/model_executor/models/nvlm_d.py
+166 -20
@@ -6,44 +6,190 @@
6 # Copyright (c) 2024 NVIDIA
7 # Licensed under Apache 2.0 License [see LICENSE for details]
8 # --------------------------------------------------------
9 -from typing import Optional
9 +from typing import Mapping, Optional
10
11 +import torch
12 import torch.nn as nn
13 from transformers import PretrainedConfig
14
14 -from vllm.inputs import INPUT_REGISTRY
15 from vllm.model_executor.layers.quantization import QuantizationConfig
16 from vllm.multimodal import MULTIMODAL_REGISTRY
17 +from vllm.multimodal.inputs import MultiModalKwargs
18 +from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
19 + MultiModalDataItems)
20 +from vllm.multimodal.processing import (PromptReplacement,
21 + PromptReplacementDetails)
22 +from vllm.multimodal.profiling import ProcessorInputs
23
24 from .intern_vit import InternVisionModel
19 -from .internvl import (InternVLChatModel, InternVLInputPipeline,
20 - get_max_internvl_image_tokens)
25 +from .internvl import (BaseInternVLProcessingInfo, BaseInternVLProcessor,
26 + InternVLChatModel, InternVLDummyInputsBuilder,
27 + InternVLMultiModalProcessor)
28
22 -IMG_START = '<|vision_start|>'
23 -IMG_END = '<|vision_end|>'
24 -IMG_CONTEXT = '<|vision_pad|>'
29 +IMG_PAD = "<|vision_pad|>"
30
31
27 -class NVLMInputPipeline(InternVLInputPipeline):
32 +class NVLMProcessor(BaseInternVLProcessor):
33 +
34 + @property
35 + def image_token_id(self) -> int:
36 + return self.tokenizer.get_vocab()[IMG_PAD]
37 +
38 + def get_image_repl_features(
39 + self,
40 + feature_size: int,
41 + num_patches: Optional[int],
42 + ) -> str:
43 + if num_patches is None:
44 + raise NotImplementedError("Embedding inputs are not supported")
45 +
46 + tile_pos_identifiers = [f"<tile_{i}>" for i in range(1, num_patches)]
47 + if self.use_thumbnail and num_patches != 1:
48 + tile_pos_identifiers += ["<tile_global_thumbnail>"]
49
29 - def _create_image_prompt(self, feature_size: int, num_patches: int) -> str:
30 - tile_pos_identifiers = ([f"<tile_{i}>"
31 - for i in range(1, num_patches)] +
32 - ["<tile_global_thumbnail>"])
50 context_size = feature_size // num_patches
51 + features = "".join(identifier + IMG_PAD * context_size
52 + for identifier in tile_pos_identifiers)
53 +
54 + # We include the start and end as well because "<Image><tile" is
55 + # tokenized as ["<Image", "><", "tile"], resulting in assertion error
56 + # when trying to find "<tile" as a subsequence of "<Image><tile"
57 + return "<Image>" + features + "</Image>"
58 +
59 + def get_image_repl_full(
60 + self,
61 + feature_size: int,
62 + num_patches: Optional[int],
63 + ) -> str:
64 + return self.get_image_repl_features(feature_size, num_patches)
65 +
66 +
67 +class NVLMProcessingInfo(BaseInternVLProcessingInfo):
68 +
69 + def get_hf_processor(
70 + self,
71 + *,
72 + max_dynamic_patch: Optional[int] = None,
73 + dynamic_image_size: Optional[bool] = None,
74 + ) -> NVLMProcessor:
75 + return NVLMProcessor(
76 + self.get_hf_config(),
77 + self.get_tokenizer(),
78 + max_dynamic_patch=max_dynamic_patch,
79 + dynamic_image_size=dynamic_image_size,
80 + )
81 +
82 + def get_max_image_tokens(self) -> int:
83 + hf_processor = self.get_hf_processor()
84 + tokenizer = hf_processor.tokenizer
85 +
86 + max_num_patches = hf_processor.max_dynamic_patch
87 + # we need +1 here because max_dynamic_patch in config doesn't
88 + # include the thumbnail patch
89 + tile_pos_identifiers = [
90 + f"<tile_{i+1}>" for i in range(max_num_patches)
91 + ]
92 + if hf_processor.use_thumbnail and max_num_patches != 1:
93 + tile_pos_identifiers += ["<tile_global_thumbnail>"]
94 +
95 + # "<Image><tile" is tokenized as ["<Image", "><", "tile"]
96 + # so we include <tile_1> in the start_str
97 + start_str = "<Image>" + tile_pos_identifiers.pop(0)
98 + end_str = "</Image>"
99 + start_token_len = len(tokenizer.encode(start_str))
100 + end_token_len = len(tokenizer.encode(end_str))
101 + tile_token_len = sum(
102 + len(tokenizer.encode(identifier))
103 + for identifier in tile_pos_identifiers)
104 + non_image_tokens_num = start_token_len + end_token_len + tile_token_len
105 + return super().get_max_image_tokens() + non_image_tokens_num
106 +
107 +
108 +class NVLMDummyInputsBuilder(InternVLDummyInputsBuilder[NVLMProcessingInfo]):
109 +
110 + def get_dummy_processor_inputs(
111 + self,
112 + seq_len: int,
113 + mm_counts: Mapping[str, int],
114 + ) -> ProcessorInputs:
115 + target_width, target_height = \
116 + self.info.get_image_size_with_most_features()
117 + num_images = mm_counts.get("image", 0)
118 +
119 + mm_data = {
120 + "image":
121 + self._get_dummy_images(width=target_width,
122 + height=target_height,
123 + num_images=num_images)
124 + }
125 +
126 + return ProcessorInputs(
127 + # The newline is necessary to separate ">" of the current item
128 + # and "<" of the next item
129 + prompt_text="<image>\n" * num_images,
130 + mm_data=mm_data,
131 + )
132 +
133 +
134 +class NVLMMultiModalProcessor(InternVLMultiModalProcessor[NVLMProcessingInfo]):
135
35 - return '<Image>' + ''.join(
36 - tile_pos_identifier + self.img_context_token * context_size
37 - for tile_pos_identifier in tile_pos_identifiers) + '</Image>'
136 + def _get_prompt_replacements(
137 + self,
138 + mm_items: MultiModalDataItems,
139 + hf_processor_mm_kwargs: Mapping[str, object],
140 + out_mm_kwargs: MultiModalKwargs,
141 + ) -> list[PromptReplacement]:
142 + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
143 +
144 + if "image_num_patches" in out_mm_kwargs:
145 + image_num_patches = out_mm_kwargs["image_num_patches"]
146 + assert isinstance(image_num_patches, torch.Tensor)
147 + image_num_patches = image_num_patches.tolist()
148 + elif "image_embeds" in out_mm_kwargs:
149 + # TODO: Use image size information in dictionary embedding inputs
150 + # to compute num_patches (similar to Qwen2-VL)
151 + image_num_patches = [None] * len(out_mm_kwargs["image_embeds"])
152 + else:
153 + image_num_patches = []
154 +
155 + def get_replacement_nvlm(item_idx: int):
156 + images = mm_items.get_items(
157 + "image", (ImageEmbeddingItems, ImageProcessorItems))
158
159 + if isinstance(images, ImageEmbeddingItems):
160 + feature_size = images.get_feature_size(item_idx)
161 + else:
162 + image_size = images.get_image_size(item_idx)
163 + feature_size = self.info.get_num_image_tokens(
164 + image_width=image_size.width,
165 + image_height=image_size.height,
166 + processor=hf_processor,
167 + )
168 +
169 + num_patches = image_num_patches[item_idx]
170 + if num_patches is not None:
171 + assert isinstance(num_patches, int)
172 +
173 + return PromptReplacementDetails(
174 + full=hf_processor.get_image_repl_full(feature_size,
175 + num_patches) + "\n",
176 + features=hf_processor.get_image_repl_features(
177 + feature_size, num_patches) + "\n",
178 + )
179
40 -input_pipeline = NVLMInputPipeline(IMG_START, IMG_END, IMG_CONTEXT)
180 + # See note in dummy data regarding why we have the extra newline
181 + return [
182 + PromptReplacement(
183 + modality="image",
184 + target="<image>\n",
185 + replacement=get_replacement_nvlm,
186 + )
187 + ]
188
189
43 -@MULTIMODAL_REGISTRY.register_image_input_mapper(input_pipeline.input_mapper)
44 -@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_internvl_image_tokens)
45 -@INPUT_REGISTRY.register_dummy_data(input_pipeline.dummy_data)
46 -@INPUT_REGISTRY.register_input_processor(input_pipeline.input_processor)
190 +@MULTIMODAL_REGISTRY.register_processor(NVLMMultiModalProcessor,
191 + info=NVLMProcessingInfo,
192 + dummy_inputs=NVLMDummyInputsBuilder)
193 class NVLM_D_Model(InternVLChatModel):
194
195 def _init_mlp1(self, config: PretrainedConfig) -> nn.Sequential:
vllm/model_executor/models/phi3v.py
+5 -1
@@ -322,7 +322,11 @@ class Phi3VProcessingInfo(BaseProcessingInfo):
322 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
323 return {"image": None}
324
325 - def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
325 + def get_mm_max_tokens_per_item(
326 + self,
327 + seq_len: int,
328 + mm_counts: Mapping[str, int],
329 + ) -> Mapping[str, int]:
330 target_width, target_height = self.get_image_size_with_most_features()
331
332 max_image_tokens = self.get_num_image_tokens(
vllm/model_executor/models/qwen.py
+8 -4
@@ -779,7 +779,11 @@ class QWenVLProcessingInfo(BaseProcessingInfo):
779 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
780 return {"image": None}
781
782 - def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
782 + def get_mm_max_tokens_per_item(
783 + self,
784 + seq_len: int,
785 + mm_counts: Mapping[str, int],
786 + ) -> Mapping[str, int]:
787 return {"image": self.get_num_image_tokens()}
788
789 def get_num_image_tokens(self) -> int:
@@ -799,13 +803,13 @@ class QWenVLDummyInputsBuilder(BaseDummyInputsBuilder[QWenVLProcessingInfo]):
803
804 vision_config = hf_config.visual
805
802 - max_image_size = vision_config["image_size"]
806 + target_width = target_height = vision_config["image_size"]
807 num_images = mm_counts.get("image", 0)
808
809 mm_data = {
810 "image":
807 - self._get_dummy_images(width=max_image_size,
808 - height=max_image_size,
811 + self._get_dummy_images(width=target_width,
812 + height=target_height,
813 num_images=num_images)
814 }
815
vllm/model_executor/models/qwen2_audio.py
+5 -1
@@ -110,7 +110,11 @@ class Qwen2AudioProcessingInfo(BaseProcessingInfo):
110 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
111 return {"audio": None}
112
113 - def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
113 + def get_mm_max_tokens_per_item(
114 + self,
115 + seq_len: int,
116 + mm_counts: Mapping[str, int],
117 + ) -> Mapping[str, int]:
118 hf_config = self.get_hf_config()
119 max_source_positions = hf_config.audio_config.max_source_positions
120 max_output_lengths = (max_source_positions - 2) // 2 + 1
vllm/model_executor/models/qwen2_vl.py
+15 -16
@@ -758,7 +758,11 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
758 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
759 return {"image": None, "video": None}
760
761 - def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
761 + def get_mm_max_tokens_per_item(
762 + self,
763 + seq_len: int,
764 + mm_counts: Mapping[str, int],
765 + ) -> Mapping[str, int]:
766 return {
767 "image": self.get_max_image_tokens(),
768 "video": self.get_max_video_tokens(seq_len),
@@ -989,26 +993,21 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo]
993 hf_processor_mm_kwargs: Mapping[str, object],
994 ) -> Mapping[str, MultiModalFieldConfig]:
995 image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3)))
992 - image_slice_idxs = [0] + image_grid_thw.prod(-1).cumsum_(0).tolist()
993 - image_slices = [
994 - slice(image_slice_idxs[i], image_slice_idxs[i + 1])
995 - for i in range(len(image_grid_thw))
996 - ]
996 + image_grid_sizes = image_grid_thw.prod(-1)
997
998 video_grid_thw = hf_inputs.get("video_grid_thw", torch.empty((0, 3)))
999 - video_slice_idxs = [0] + video_grid_thw.prod(-1).cumsum_(0).tolist()
1000 - video_slices = [
1001 - slice(video_slice_idxs[i], video_slice_idxs[i + 1])
1002 - for i in range(len(video_grid_thw))
1003 - ]
999 + video_grid_sizes = video_grid_thw.prod(-1)
1000
1001 return dict(
1006 - pixel_values=MultiModalFieldConfig.flat("image", image_slices),
1007 - image_embeds=MultiModalFieldConfig.flat("image", image_slices),
1002 + pixel_values=MultiModalFieldConfig.flat_from_sizes(
1003 + "image", image_grid_sizes),
1004 + image_embeds=MultiModalFieldConfig.flat_from_sizes(
1005 + "image", image_grid_sizes),
1006 image_grid_thw=MultiModalFieldConfig.batched("image"),
1009 - pixel_values_videos=MultiModalFieldConfig.flat(
1010 - "video", video_slices),
1011 - video_embeds=MultiModalFieldConfig.flat("video", video_slices),
1007 + pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
1008 + "video", video_grid_sizes),
1009 + video_embeds=MultiModalFieldConfig.flat_from_sizes(
1010 + "video", video_grid_sizes),
1011 video_grid_thw=MultiModalFieldConfig.batched("video"),
1012 )
1013
vllm/model_executor/models/ultravox.py
+5 -1
@@ -92,7 +92,11 @@ class UltravoxProcessingInfo(BaseProcessingInfo):
92 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
93 return {"audio": None}
94
95 - def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
95 + def get_mm_max_tokens_per_item(
96 + self,
97 + seq_len: int,
98 + mm_counts: Mapping[str, int],
99 + ) -> Mapping[str, int]:
100 feature_extractor = self.get_feature_extractor()
101 max_audio_tokens = math.ceil(feature_extractor.chunk_length *
102 _AUDIO_TOKENS_PER_SECOND)
vllm/multimodal/inputs.py
+11
@@ -4,6 +4,7 @@ from abc import ABC, abstractmethod
4 from collections import UserDict, defaultdict
5 from collections.abc import Mapping, Sequence
6 from dataclasses import dataclass
7 +from itertools import accumulate
8 from typing import (TYPE_CHECKING, Any, Literal, Optional, TypedDict, TypeVar,
9 Union, cast, final)
10
@@ -258,6 +259,16 @@ class MultiModalFieldConfig:
259 slices=slices,
260 )
261
262 + @staticmethod
263 + def flat_from_sizes(modality: str, size_per_item: torch.Tensor):
264 + slice_idxs = [0, *accumulate(size_per_item)]
265 + slices = [
266 + slice(slice_idxs[i], slice_idxs[i + 1])
267 + for i in range(len(size_per_item))
268 + ]
269 +
270 + return MultiModalFieldConfig.flat(modality, slices)
271 +
272 def __init__(
273 self,
274 field_cls: type[BaseMultiModalField],
vllm/multimodal/processing.py
+5 -1
@@ -680,7 +680,11 @@ class BaseProcessingInfo:
680 raise NotImplementedError
681
682 @abstractmethod
683 - def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
683 + def get_mm_max_tokens_per_item(
684 + self,
685 + seq_len: int,
686 + mm_counts: Mapping[str, int],
687 + ) -> Mapping[str, int]:
688 """
689 Get the maximum possible number of tokens per data item
690 for each modality.
vllm/multimodal/profiling.py
+2 -1
@@ -151,7 +151,8 @@ class MultiModalProfiler(Generic[_I]):
151 mm_counts = self.get_mm_limits()
152
153 info = self.processing_info
154 - mm_max_tokens_per_item = info.get_mm_max_tokens_per_item(seq_len)
154 + mm_max_tokens_per_item = info.get_mm_max_tokens_per_item(
155 + seq_len, mm_counts)
156
157 if mm_counts.keys() != mm_max_tokens_per_item.keys():
158 raise AssertionError(
vllm/multimodal/registry.py
+3 -1
@@ -264,7 +264,9 @@ class MultiModalRegistry:
264 )
265 processor = self.create_processor(model_config, tokenizer)
266 seq_len = model_config.max_model_len
267 - return processor.info.get_mm_max_tokens_per_item(seq_len)
267 + mm_limits = self.get_mm_limits_per_prompt(model_config)
268 + return processor.info.get_mm_max_tokens_per_item(
269 + seq_len, mm_limits)
270
271 return {
272 key: plugin.get_max_multimodal_tokens(model_config)