A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[VLM] merged multimodal processor and V1 support for idefics3 (#12660)
Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Isotr0py committed
Feb 4, 2025 at 20:00 UTC
815079de8e9dd984d474f7046412d5aedf4350ff
7 files changed
+315
-457
docs/source/models/supported_models.md
+1
-1
@@ -733,7 +733,7 @@ See [this page](#generative-models) for more information on how to use generativ
733
* `HuggingFaceM4/Idefics3-8B-Llama3` etc.
734
* ✅︎
735
*
736
- *
736
+ * ✅︎
737
- * `InternVLChatModel`
738
* InternVL 2.5, Mono-InternVL, InternVL 2.0
739
* T + I<sup>E+</sup>
tests/models/decoder_only/vision_language/test_models.py
+2
-2
@@ -254,14 +254,14 @@ VLM_TEST_SETTINGS = {
254
patch_hf_runner=model_utils.h2ovl_patch_hf_runner,
255
),
256
"idefics3": VLMTestInfo(
257
- models=["HuggingFaceM4/Idefics3-8B-Llama3"],
257
+ models=["HuggingFaceTB/SmolVLM-256M-Instruct"],
258
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
259
prompt_formatter=lambda img_prompt:f"<|begin_of_text|>User:{img_prompt}<end_of_utterance>\nAssistant:", # noqa: E501
260
img_idx_to_prompt=lambda idx: "<image>",
261
max_model_len=8192,
262
max_num_seqs=2,
263
auto_cls=AutoModelForVision2Seq,
264
- marks=[large_gpu_mark(min_gb=48)],
264
+ hf_output_post_proc=model_utils.idefics3_trunc_hf_output,
265
),
266
"intern_vl": VLMTestInfo(
267
models=[
tests/models/decoder_only/vision_language/vlm_utils/model_utils.py
+8
@@ -192,6 +192,14 @@ def deepseekvl2_trunc_hf_output(hf_output: RunnerOutput,
192
return output_ids, output_str, out_logprobs
193
194
195
+def idefics3_trunc_hf_output(hf_output: RunnerOutput,
196
+ model: str) -> RunnerOutput:
197
+ output_ids, output_str, out_logprobs = hf_output
198
+ if output_str.endswith("<end_of_utterance>"):
199
+ output_str = output_str.split("<end_of_utterance>")[0]
200
+ return output_ids, output_str, out_logprobs
201
+
202
+
203
def minicpmv_trunc_hf_output(hf_output: RunnerOutput,
204
model: str) -> RunnerOutput:
205
output_ids, output_str, out_logprobs = hf_output
tests/models/multimodal/processing/test_common.py
+1
@@ -149,6 +149,7 @@ def _test_processing_correctness(
149
"adept/fuyu-8b",
150
"h2oai/h2ovl-mississippi-800m",
151
"OpenGVLab/InternVL2-1B",
152
+ "HuggingFaceM4/Idefics3-8B-Llama3",
153
"llava-hf/llava-1.5-7b-hf",
154
"llava-hf/llava-v1.6-mistral-7b-hf",
155
"llava-hf/LLaVA-NeXT-Video-7B-hf",
tests/models/multimodal/processing/test_idefics3.py
+33
-146
@@ -1,13 +1,10 @@
1
# SPDX-License-Identifier: Apache-2.0
2
"""Tests for Idefics3's multimodal preprocessing kwargs."""
3
-from typing import Optional
4
-
3
import pytest
6
-import torch
7
-from transformers import AutoImageProcessor, AutoTokenizer
4
+from transformers import Idefics3Config
5
9
-from vllm.inputs import InputContext, token_inputs
10
-from vllm.multimodal import MultiModalRegistry
6
+from vllm.multimodal import MULTIMODAL_REGISTRY
7
+from vllm.multimodal.utils import cached_get_tokenizer
8
9
from ....conftest import _ImageAssets
10
from ...utils import build_model_context
@@ -15,163 +12,53 @@ from ...utils import build_model_context
12
models = ["HuggingFaceM4/Idefics3-8B-Llama3"]
13
14
18
-# Wrap lazy imports to avoid initializing CUDA during test collection
19
-@pytest.fixture()
20
-def input_processor_for_idefics3():
21
- from vllm.model_executor.models.idefics3 import (
22
- input_processor_for_idefics3)
23
- return input_processor_for_idefics3
24
-
25
-
26
-@pytest.fixture()
27
-def dummy_data_for_idefics3():
28
- from vllm.model_executor.models.idefics3 import dummy_data_for_idefics3
29
- return dummy_data_for_idefics3
30
-
31
-
32
-@pytest.fixture()
33
-def get_max_idefics3_image_tokens():
34
- from vllm.model_executor.models.idefics3 import (
35
- get_max_idefics3_image_tokens)
36
- return get_max_idefics3_image_tokens
37
-
38
-
39
-@pytest.mark.parametrize("model", models)
40
-@pytest.mark.parametrize("longest_edge", [None, 168, 336, 400, 2 * 336])
41
-def test_input_mapper_override(model: str, image_assets: _ImageAssets,
42
- longest_edge: Optional[int]):
43
- """Ensure that the [default] input mapper handles size properly."""
44
-
45
- mm_processor_kwargs = {
46
- "size": {
47
- "longest_edge": longest_edge
48
- }
49
- } if longest_edge is not None else {}
50
- ctx = build_model_context(
51
- model_name=model,
52
- tokenizer_name=model,
53
- trust_remote_code=True,
54
- mm_processor_kwargs=mm_processor_kwargs,
55
- )
56
-
57
- hf_processor = AutoImageProcessor.from_pretrained(model,
58
- trust_remote_code=True,
59
- **mm_processor_kwargs)
60
-
61
- mm_registry = MultiModalRegistry()
62
- mm_registry.init_mm_limits_per_prompt(ctx.model_config)
63
-
64
- image = image_assets[0].pil_image
65
- hf_result = hf_processor.preprocess(
66
- image,
67
- return_tensors="pt",
68
- )
69
-
70
- vllm_result = mm_registry.map_input(
71
- ctx.model_config,
72
- {"image": image},
73
- )
74
-
75
- assert torch.all(hf_result["pixel_values"] == vllm_result["pixel_values"])
76
-
77
-
78
-@pytest.mark.parametrize("model", models)
79
-@pytest.mark.parametrize("longest_edge, expected_max_tokens", [
80
- (None, 2873),
81
- (168, 169),
82
- (336, 169),
83
- (400, 338),
84
- (672, 338),
85
-])
86
-def test_max_tokens_override(get_max_idefics3_image_tokens, model: str,
87
- longest_edge: Optional[int],
88
- expected_max_tokens: int):
89
- """Ensure get_max_idefics3_image_tokens handles mm_processor_kwargs."""
90
- size = {"longest_edge": longest_edge} if longest_edge is not None else None
91
- ctx = build_model_context(
92
- model_name=model,
93
- tokenizer_name=model,
94
- trust_remote_code=True,
95
- mm_processor_kwargs=None,
96
- )
97
-
98
- actual_max_tokens = get_max_idefics3_image_tokens(
99
- ctx=InputContext(ctx.model_config),
100
- size=size,
101
- )
102
-
103
- assert expected_max_tokens == actual_max_tokens
104
-
105
-
106
-@pytest.mark.parametrize("model", models)
107
-@pytest.mark.parametrize("longest_edge, toks_per_img, num_imgs", [
108
- (168, 169, 1),
109
- (168, 169, 2),
110
- (400, 338, 1),
111
- (400, 338, 2),
112
-])
113
-def test_dummy_data_override(dummy_data_for_idefics3, model: str,
114
- longest_edge: int, toks_per_img: int,
115
- num_imgs: int):
116
- """Ensure dummy_data_for_idefics3 handles num_crops properly."""
117
- # Same as the previous test - don't initialize mm_processor_kwargs
118
- # in this test and assume that the kwargs will be correctly expanded by
119
- # the partial when calling the dummy data func.
120
- size = {"longest_edge": longest_edge} if longest_edge is not None else None
121
- ctx = build_model_context(
122
- model_name=model,
123
- tokenizer_name=model,
124
- trust_remote_code=True,
125
- mm_processor_kwargs=None,
126
- )
127
-
128
- dummy_data = dummy_data_for_idefics3(
129
- ctx=ctx,
130
- seq_len=8192, # Should be bigger than num_imgs * toks_per_img
131
- mm_counts={"image": num_imgs},
132
- size=size)
133
- sequence_data = dummy_data.seq_data
134
- # Ensure we have the right number of placeholders per size
135
- image_token_id = ctx.get_hf_config().image_token_id
136
- img_tok_count = sequence_data.get_token_ids().count(image_token_id)
137
- assert img_tok_count == toks_per_img * num_imgs
138
-
139
-
15
@pytest.mark.parametrize("model", models)
141
-@pytest.mark.parametrize("longest_edge,expected_toks_per_img,num_imgs", [
142
- (336, 169 * (1**2 + 1), 1),
143
- (336, 169 * (1**2 + 1), 2),
144
- (400, 169 * (2**2 + 1), 1),
145
- (400, 169 * (2**2 + 1), 2),
146
-])
147
-def test_input_processor_override(input_processor_for_idefics3,
148
- image_assets: _ImageAssets, model: str,
149
- longest_edge: int,
150
- expected_toks_per_img: int, num_imgs: int):
16
+# yapf: disable
17
+@pytest.mark.parametrize(
18
+ ("mm_processor_kwargs", "expected_toks_per_img"),
19
+ [
20
+ ({"size": {"longest_edge": 364}}, 169),
21
+ ({"size": {"longest_edge": 728}}, 169 * (2**2 + 1)),
22
+ ])
23
+# yapf: enable
24
+@pytest.mark.parametrize("num_imgs", [1, 2])
25
+def test_processor_override(image_assets: _ImageAssets, model: str,
26
+ mm_processor_kwargs: dict[str, object],
27
+ expected_toks_per_img: int, num_imgs: int):
28
"""Ensure input_processor_for_idefics3 handles num_crops properly."""
29
# Same as the previous test - don't initialize mm_processor_kwargs
30
# in this test and assume that the kwargs will be correctly expanded by
31
# the partial when calling the custom input processor.
155
- size = {"longest_edge": longest_edge} if longest_edge is not None else None
32
ctx = build_model_context(
33
model_name=model,
34
tokenizer_name=model,
35
trust_remote_code=True,
36
mm_processor_kwargs=None,
37
+ limit_mm_per_prompt={"image": num_imgs},
38
)
39
+ tokenizer = cached_get_tokenizer(ctx.model_config.tokenizer)
40
+ processor = MULTIMODAL_REGISTRY.create_processor(
41
+ ctx.model_config,
42
+ tokenizer=tokenizer,
43
+ )
44
+ hf_processor = processor.info.get_hf_processor(**mm_processor_kwargs)
45
46
# Build the image str / prompt based on the number of images we pass
164
- tokenizer = AutoTokenizer.from_pretrained(model)
47
placeholders = "<image>" if num_imgs == 1 else "\n".join(
48
f"Image-{i}: <image>\n" for i in range(1, num_imgs + 1))
49
prompt = f"<|begin_of_text|>User:{placeholders}\n<end_of_utterance>\nAssistant:" # noqa: E501
168
- images = [image_assets[0].pil_image.resize((336 * 4, 336 * 4))] * num_imgs
169
-
170
- inputs = token_inputs(prompt_token_ids=tokenizer.encode(prompt),
171
- prompt=prompt,
172
- multi_modal_data={"image": images})
50
174
- processed_inputs = input_processor_for_idefics3(ctx, inputs, size=size)
51
+ # Build mm_data
52
+ image_size = ctx.get_hf_config(Idefics3Config).vision_config.image_size
53
+ dummy_image_size = (image_size * 4, image_size * 4)
54
+ dummy_image = image_assets[0].pil_image.resize(dummy_image_size)
55
+ mm_data = {"image": [dummy_image] * num_imgs}
56
+
57
+ processed_inputs = processor.apply(prompt, mm_data, mm_processor_kwargs)
58
+ # Ensure the placeholders format are correct
59
+ hf_processed_inputs = hf_processor(text=prompt, images=mm_data["image"])
60
+ assert processed_inputs["prompt_token_ids"] == hf_processed_inputs[
61
+ "input_ids"][0]
62
63
# Ensure we have the right number of placeholders per num_crops size
64
image_token_id = ctx.get_hf_config().image_token_id
vllm/inputs/registry.py
+18
@@ -31,6 +31,17 @@ C = TypeVar("C", bound=PretrainedConfig, default=PretrainedConfig)
31
P = TypeVar("P", bound=ProcessorMixin, default=ProcessorMixin)
32
33
34
+class HashableDict(dict):
35
+ """
36
+ A dictionary that can be hashed by lru_cache.
37
+ """
38
+
39
+ # NOTE: pythonic dict is not hashable,
40
+ # we override on it directly for simplicity
41
+ def __hash__(self) -> int: # type: ignore[override]
42
+ return hash(frozenset(self.items()))
43
+
44
+
45
@dataclass(frozen=True)
46
class InputContext:
47
"""
@@ -104,6 +115,13 @@ class InputContext:
115
if isinstance(typ, type):
116
merged_kwargs["processor_cls"] = typ
117
118
+ # NOTE: Pythonic dict is not hashable and will raise unhashable type
119
+ # error when calling `cached_get_processor`, therefore we need to
120
+ # wrap it to a hashable dict.
121
+ for key, value in merged_kwargs.items():
122
+ if isinstance(value, dict):
123
+ merged_kwargs[key] = HashableDict(value)
124
+
125
hf_processor = cached_get_processor(
126
self.model_config.model,
127
trust_remote_code=self.model_config.trust_remote_code,
vllm/model_executor/models/idefics3.py
+252
-308
@@ -16,35 +16,35 @@
16
"""Inference-only Idefics3 model compatible with HuggingFace weights."""
17
18
import math
19
-from typing import (Dict, Iterable, List, Literal, Mapping, NamedTuple,
20
- Optional, Set, Tuple, TypedDict, Union)
19
+from typing import (Dict, Iterable, List, Literal, Mapping, Optional, Set,
20
+ Tuple, TypedDict, Union)
21
22
import torch
23
import torch.utils.checkpoint
24
-from PIL import Image
24
from torch import nn
26
-# Temporary solution for transformers below 4.46.0.
27
-from transformers import PretrainedConfig as Idefics3Config
28
-from transformers import ProcessorMixin as Idefics3ImageProcessor
25
+from transformers import (BatchFeature, Idefics3Config, Idefics3ImageProcessor,
26
+ Idefics3Processor)
27
28
from vllm.attention import AttentionMetadata
29
from vllm.config import VllmConfig
32
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
33
- InputContext, token_inputs)
30
from vllm.logger import init_logger
31
from vllm.model_executor.layers.linear import ReplicatedLinear
32
from vllm.model_executor.layers.logits_processor import LogitsProcessor
33
from vllm.model_executor.layers.quantization import QuantizationConfig
38
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
34
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
35
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
36
from vllm.model_executor.models.module_mapping import MultiModelKeys
37
from vllm.model_executor.sampling_metadata import SamplingMetadata
38
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
43
-from vllm.multimodal.image import cached_get_image_processor
39
from vllm.multimodal.inputs import NestedTensors
45
-from vllm.sequence import IntermediateTensors, SequenceData
46
-from vllm.transformers_utils.processor import cached_get_processor
47
-from vllm.utils import is_list_of
40
+from vllm.multimodal.parse import ImageProcessorItems
41
+from vllm.multimodal.processing import (BaseMultiModalProcessor,
42
+ BaseProcessingInfo,
43
+ MultiModalDataItems,
44
+ MultiModalFieldConfig,
45
+ PromptReplacement)
46
+from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
47
+from vllm.sequence import IntermediateTensors
48
49
# yapf: disable
50
from .idefics2_vision_model import (
@@ -77,307 +77,253 @@ class Idefics3ImageEmbeddingInputs(TypedDict):
77
"""
78
79
80
-class Idefics3ProcessorSize(NamedTuple):
81
- """Hashable wrapper for unhashable `size` dict of Idefics3Processor."""
82
- # NOTE: cached_get_processor/cached_get_image_processor uses lru_cache,
83
- # we need to use NamedTuple instead of TypedDict to avoid hashing issues.
84
- longest_edge: int
85
-
86
- def __contains__(self, key: str) -> bool:
87
- return key in self._asdict() and getattr(self, key) is not None
88
-
89
- def __getitem__(self, key: str) -> int:
90
- return getattr(self, key)
91
-
92
-
80
ImageInputs = Union[Idefics3ImagePixelInputs, Idefics3ImageEmbeddingInputs]
81
82
96
-def get_mm_processor_kwargs(size: Optional[Dict[str, int]] = None) -> Dict:
97
- mm_processor_kwargs = {}
98
- if size:
99
- mm_processor_kwargs["size"] = Idefics3ProcessorSize(**size)
100
- return mm_processor_kwargs
101
-
102
-
103
-def input_mapper_for_idefics3(
104
- ctx: InputContext,
105
- data: object,
106
- *,
107
- size: Optional[Dict[str, int]] = None,
108
-):
109
- model_config = ctx.model_config
110
- mm_processor_kwargs = get_mm_processor_kwargs(size)
111
- image_processor = cached_get_image_processor(
112
- model_config.model,
113
- trust_remote_code=model_config.trust_remote_code,
114
- **mm_processor_kwargs)
115
- if image_processor is None:
116
- raise RuntimeError("No HuggingFace processor is available "
117
- "to process the image object")
118
-
119
- if isinstance(data, Image.Image):
120
- images = [[data]]
121
- elif is_list_of(data, Image.Image):
122
- images = [data]
123
- else:
124
- raise TypeError(f"Invalid image type: {type(data)}")
125
-
126
- try:
127
- batch_data = image_processor(images,
128
- return_tensors="pt",
129
- return_row_col_info=True).data
130
- except Exception:
131
- logger.error("Failed to process image (%s)", data)
132
- raise
133
-
134
- return MultiModalKwargs(batch_data)
135
-
136
-
137
-def _resize_output_size(height: int,
138
- width: int,
139
- max_len: Optional[int] = None,
140
- min_len: Optional[int] = 1,
141
- max_size: Optional[int] = None) -> Tuple[int, int]:
142
- # Set default value for max_len if not provided
143
- max_len = max(height, width) if max_len is None else max_len
144
- aspect_ratio = width / height
145
-
146
- # Handle the maximum size constraint
147
- if max_size is not None:
148
- max_len = min(max_len, max_size)
149
-
150
- # Adjust dimensions according to the aspect ratio
151
- if width >= height:
152
- width = max_len
153
- height = int(width / aspect_ratio)
154
- else:
155
- height = max_len
156
- width = int(height * aspect_ratio)
157
-
158
- # Ensure both width and height are even (if needed)
159
- height += 1 if height % 2 != 0 else 0
160
- width += 1 if width % 2 != 0 else 0
161
-
162
- # Ensure dimensions are not smaller than the minimum length
163
- height = max(height, min_len)
164
- width = max(width, min_len)
165
-
166
- return height, width
167
-
168
-
169
-def _get_resize_output_image_size(
170
- image_size: Tuple[int, int],
171
- resolution_max_side: int,
172
- max_image_size: int = 1820,
173
-) -> Tuple[int, int]:
174
- if resolution_max_side > max_image_size:
175
- raise ValueError(
176
- "`resolution_max_side` cannot be larger than `max_image_size`")
177
-
178
- height, width = image_size
179
-
180
- # Find the output size, when rescaling the longest edge to max_len and
181
- # preserving the aspect ratio
182
- height, width = _resize_output_size(height,
183
- width,
184
- max_len=resolution_max_side)
185
-
186
- return height, width
187
-
188
-
189
-def _prompt_split_image(image_seq_len: int, image_rows: int, image_cols: int,
190
- fake_token_around_image: str, image_token: str,
191
- global_img_token: str) -> str:
192
- """
193
- Prompt with expanded image tokens for when the image is split
194
- into patches.
195
- """
196
- text_split_images = ""
197
- for n_h in range(image_rows):
198
- for n_w in range(image_cols):
199
- text_split_images += (fake_token_around_image +
200
- f"<row_{n_h + 1}_col_{n_w + 1}>" +
201
- image_token * image_seq_len)
202
- text_split_images += "\n"
203
-
204
- text_split_images += "\n" + _prompt_single_image(
205
- image_seq_len=image_seq_len,
206
- fake_token_around_image=fake_token_around_image,
207
- image_token=image_token,
208
- global_img_token=global_img_token)
209
- return text_split_images
210
-
211
-
212
-def _prompt_single_image(image_seq_len: int, fake_token_around_image: str,
213
- image_token: str, global_img_token: str):
214
- """Prompt with expanded image tokens for a single image."""
215
- return (fake_token_around_image + global_img_token +
216
- image_token * image_seq_len + fake_token_around_image)
217
-
218
-
219
-def _get_image_prompt_string(image_rows: int, image_cols: int,
220
- image_seq_len: int, fake_token_around_image: str,
221
- image_token: str, global_img_token: str):
222
- if image_rows == 0 and image_cols == 0:
223
- return _prompt_single_image(
224
- image_seq_len=image_seq_len,
225
- fake_token_around_image=fake_token_around_image,
226
- image_token=image_token,
227
- global_img_token=global_img_token,
228
- )
229
- return _prompt_split_image(image_seq_len, image_rows, image_cols,
230
- fake_token_around_image, image_token,
231
- global_img_token)
232
-
233
-
234
-def input_processor_for_idefics3(ctx: InputContext,
235
- inputs: DecoderOnlyInputs,
236
- *,
237
- size: Optional[Dict[str, int]] = None):
238
- multi_modal_data = inputs.get("multi_modal_data")
239
- if multi_modal_data is None or "image" not in multi_modal_data:
240
- return inputs
241
-
242
- model_config = ctx.model_config
243
- mm_processor_kwargs = get_mm_processor_kwargs(size)
244
- processor = cached_get_processor(model_config.model, **mm_processor_kwargs)
245
- image_processor = processor.image_processor
246
- tokenizer = processor.tokenizer
247
- size = image_processor.size['longest_edge']
248
- max_image_size = image_processor.max_image_size['longest_edge']
249
-
250
- image_data = multi_modal_data["image"]
251
- if isinstance(image_data, Image.Image):
252
- image_list = [image_data]
253
- elif is_list_of(image_data, Image.Image):
254
- image_list = image_data
255
- else:
256
- raise TypeError(f"Invalid image type: {type(image_data)}")
257
-
258
- image_rows = []
259
- image_cols = []
260
- for image in image_list:
261
- height, width = _get_resize_output_image_size(image.size, size)
262
-
263
- rows = math.ceil(height / max_image_size)
264
- cols = math.ceil(width / max_image_size)
265
- image_rows.append(rows)
266
- image_cols.append(cols)
267
- image_rows = [image_rows]
268
- image_cols = [image_cols]
269
-
270
- n_images_in_text = []
271
-
272
- text = inputs.get("prompt")
273
- if text is None:
274
- prompt_token_ids = inputs.get("prompt_token_ids", [])
275
- assert prompt_token_ids
276
- text = tokenizer.decode(prompt_token_ids)
277
-
278
- if isinstance(text, str):
279
- text = [text]
280
- elif not isinstance(text, list) and not isinstance(text[0], str):
281
- raise ValueError("Invalid input text. Please provide a string, "
282
- "or a list of strings")
283
-
284
- fake_image_token = processor.fake_image_token.content
285
- image_token = processor.image_token.content
286
- global_img_token = processor.global_image_tag
287
-
288
- prompt_strings = []
289
- for sample, sample_rows, sample_cols in zip(text, image_rows, image_cols):
290
- n_images_in_text.append(sample.count(image_token))
291
-
292
- # Replace the image token with fake tokens around the expanded
293
- # image token sequence of length `image_seq_len`
294
- image_prompt_strings = []
295
- for n_rows, n_cols in zip(sample_rows, sample_cols):
296
- image_prompt_string = _get_image_prompt_string(
297
- n_rows,
298
- n_cols,
299
- processor.image_seq_len,
300
- image_token=image_token,
301
- fake_token_around_image=fake_image_token,
302
- global_img_token=global_img_token,
303
- )
304
- image_prompt_strings.append(image_prompt_string)
305
-
306
- split_sample = sample.split(image_token)
307
- if len(split_sample) == 0:
308
- raise ValueError("The image token should be present in the text.")
309
-
310
- # Place in the image prompt strings where the image tokens are
311
- sample = split_sample[0]
312
- for i, image_prompt_string in enumerate(image_prompt_strings):
313
- sample += image_prompt_string + split_sample[i + 1]
314
- prompt_strings.append(sample)
83
+class Idefics3ProcessingInfo(BaseProcessingInfo):
84
316
- prompt_token_ids = tokenizer(text=prompt_strings[0]).input_ids
85
+ def get_hf_processor(
86
+ self,
87
+ *,
88
+ size: Optional[Dict[str, int]] = None) -> Idefics3Processor:
89
+ if size is not None:
90
+ return self.ctx.get_hf_processor(Idefics3Processor, size=size)
91
318
- return token_inputs(
319
- prompt_token_ids=prompt_token_ids,
320
- prompt=prompt_strings[0],
321
- multi_modal_data=multi_modal_data,
322
- )
92
+ return self.ctx.get_hf_processor(Idefics3Processor)
93
94
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
95
+ return {"image": None}
96
325
-def _get_max_num_image_patch(image_processor: Idefics3ImageProcessor) -> int:
326
- size = image_processor.size['longest_edge']
327
- max_image_size = image_processor.max_image_size['longest_edge']
328
- resized_height, resized_width = size, size
97
+ def get_mm_max_tokens_per_item(
98
+ self,
99
+ seq_len: int,
100
+ mm_counts: Mapping[str, int],
101
+ ) -> Mapping[str, int]:
102
+ hf_processor = self.get_hf_processor()
103
+ image_processor: Idefics3ImageProcessor = hf_processor.image_processor
104
+ grid_w, grid_h = self._get_image_feature_grid_size(
105
+ image_width=image_processor.size['longest_edge'],
106
+ image_height=image_processor.size['longest_edge'],
107
+ )
108
+ num_image_token = (grid_w * grid_h + 1) * hf_processor.image_seq_len
109
+ # Calculate Non-image-token length
110
+ # NOTE: <row_1_col_1> and <global-img> are special token for SmolVLM
111
+ # but not for Idefic3, so we need to tokenize them to get actual length.
112
+ tokenizer = self.get_tokenizer()
113
+ tile_token_len = len(tokenizer.tokenize("<row_1_col_1>"))
114
+ glob_token_len = len(tokenizer.tokenize(hf_processor.global_image_tag))
115
+ # linebreak and <fake_token_around_image> always cost 1 token
116
+ fake_token_len = lb_len = 1
117
+ non_image_token = (grid_w * grid_h) * (
118
+ tile_token_len + fake_token_len) + glob_token_len + (
119
+ grid_h + 1) * lb_len + fake_token_len
120
+ return {"image": num_image_token + non_image_token}
121
+
122
+ def _resize_output_size(self,
123
+ *,
124
+ height: int,
125
+ width: int,
126
+ max_len: Optional[int] = None,
127
+ min_len: Optional[int] = 1,
128
+ max_size: Optional[int] = None) -> tuple[int, int]:
129
+ # Set default value for max_len if not provided
130
+ max_len = max(height, width) if max_len is None else max_len
131
+ aspect_ratio = width / height
132
+
133
+ # Handle the maximum size constraint
134
+ if max_size is not None:
135
+ max_len = min(max_len, max_size)
136
+
137
+ # Adjust dimensions according to the aspect ratio
138
+ if width >= height:
139
+ width = max_len
140
+ height = int(width / aspect_ratio)
141
+ else:
142
+ height = max_len
143
+ width = int(height * aspect_ratio)
144
330
- grid_h = resized_height // max_image_size
331
- grid_w = resized_width // max_image_size
332
- return (grid_h * grid_w + 1)
145
+ # Ensure both width and height are even (if needed)
146
+ height += height % 2
147
+ width += width % 2
148
149
+ # Ensure dimensions are not smaller than the minimum length
150
+ height = max(height, min_len)
151
+ width = max(width, min_len)
152
335
-def get_max_idefics3_image_tokens(ctx: InputContext,
336
- *,
337
- size: Optional[Dict[str,
338
- int]] = None) -> int:
339
- model_config = ctx.model_config
340
- mm_processor_kwargs = get_mm_processor_kwargs(size)
341
- processor = cached_get_processor(model_config.model, **mm_processor_kwargs)
342
- image_seq_len = processor.image_seq_len
343
- image_processor = processor.image_processor
153
+ return height, width
154
345
- max_num_image_patches = _get_max_num_image_patch(image_processor)
155
+ def _get_resize_output_image_size(
156
+ self,
157
+ *,
158
+ image_width: int,
159
+ image_height: int,
160
+ resolution_max_side: int,
161
+ ) -> tuple[int, int]:
162
+ hf_processor = self.get_hf_processor()
163
+ image_processor: Idefics3ImageProcessor = hf_processor.image_processor
164
+ max_image_size = image_processor.size['longest_edge']
165
+ if resolution_max_side > max_image_size:
166
+ raise ValueError(
167
+ "`resolution_max_side` cannot be larger than `max_image_size`")
168
+
169
+ height, width = image_height, image_width
170
+
171
+ # Find the output size, when rescaling the longest edge to max_len and
172
+ # preserving the aspect ratio
173
+ height, width = self._resize_output_size(height=height,
174
+ width=width,
175
+ max_len=resolution_max_side)
176
+ return height, width
177
+
178
+ def _get_image_feature_grid_size(
179
+ self,
180
+ *,
181
+ image_width: int,
182
+ image_height: int,
183
+ size: Optional[dict[str, object]] = None,
184
+ ) -> tuple[int, int]:
185
+ hf_processor = self.get_hf_processor(size=size)
186
+ image_processor: Idefics3ImageProcessor = hf_processor.image_processor
187
+ max_image_size = image_processor.max_image_size['longest_edge']
188
+ size = image_processor.size['longest_edge']
189
+ assert size % max_image_size == 0, (
190
+ "`longest_edge` in image_processor's `size` must be divisible by "
191
+ "`longest_edge` in `max_image_size`, this may be caused by "
192
+ "incorrect mm_kwargs override.")
193
+
194
+ resized_height, resized_width = self._get_resize_output_image_size(
195
+ image_width=image_width,
196
+ image_height=image_height,
197
+ resolution_max_side=size,
198
+ )
199
+ if resized_height > max_image_size or resized_width > max_image_size:
200
+ grid_h = math.ceil(resized_height / max_image_size)
201
+ grid_w = math.ceil(resized_width / max_image_size)
202
+ else:
203
+ grid_h = grid_w = 0
204
+ return grid_w, grid_h
205
347
- return max_num_image_patches * image_seq_len
206
207
+class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo]
208
+ ):
209
350
-def dummy_data_for_idefics3(
351
- ctx: InputContext,
210
+ def get_dummy_processor_inputs(
211
+ self,
212
seq_len: int,
213
mm_counts: Mapping[str, int],
354
- *,
355
- size: Optional[Dict[str, int]] = None) -> DummyData:
356
- hf_config = ctx.get_hf_config()
357
- num_images = mm_counts["image"]
214
+ ) -> ProcessorInputs:
215
+ num_images = mm_counts.get("image", 0)
216
+ hf_processor = self.info.get_hf_processor()
217
+ image_processor: Idefics3ImageProcessor = hf_processor.image_processor
218
+ longest_edge = image_processor.max_image_size['longest_edge']
219
+ image_token: str = hf_processor.image_token.content
220
+
221
+ mm_data = {
222
+ "image":
223
+ self._get_dummy_images(width=longest_edge,
224
+ height=longest_edge,
225
+ num_images=num_images)
226
+ }
227
+
228
+ return ProcessorInputs(
229
+ prompt_text=image_token * num_images,
230
+ mm_data=mm_data,
231
+ )
232
359
- mm_processor_kwargs = get_mm_processor_kwargs(size)
360
- processor = cached_get_processor(ctx.model_config.model,
361
- **mm_processor_kwargs)
362
- max_num_image_patches = _get_max_num_image_patch(processor.image_processor)
363
- image_seq_len = processor.image_seq_len
364
- max_llm_image_tokens = max_num_image_patches * image_seq_len * num_images
233
366
- if seq_len - max_llm_image_tokens < 0:
367
- raise RuntimeError(
368
- f"Idefics3 cannot process {num_images} images in a prompt, "
369
- "please increase max_model_len or reduce image limit by "
370
- "--limit-mm-per-prompt.")
234
+class Idefics3MultimodalProcessor(
235
+ BaseMultiModalProcessor[Idefics3ProcessingInfo]):
236
372
- seq_data = SequenceData.from_prompt_token_counts(
373
- (hf_config.image_token_id, max_llm_image_tokens),
374
- (0, seq_len - max_llm_image_tokens))
237
+ def _call_hf_processor(
238
+ self,
239
+ prompt: str,
240
+ mm_data: Mapping[str, object],
241
+ mm_kwargs: Mapping[str, object],
242
+ ) -> BatchFeature:
243
+ if mm_data:
244
+ processed_outputs = super()._call_hf_processor(
245
+ prompt, mm_data, mm_kwargs)
246
+ image_grids = [
247
+ self.info._get_image_feature_grid_size(
248
+ image_width=img.width,
249
+ image_height=img.height,
250
+ **mm_kwargs,
251
+ ) for img in mm_data["images"]
252
+ ]
253
+ image_patches = list(map(lambda x: math.prod(x) + 1, image_grids))
254
+ for key in ("pixel_values", "pixel_attention_mask"):
255
+ data = processed_outputs.pop(key)
256
+ data = data.flatten(0, 1).split(image_patches)
257
+ processed_outputs[key] = data
258
+ else:
259
+ tokenizer = self.info.get_tokenizer()
260
+ processed_outputs = tokenizer(prompt,
261
+ add_special_tokens=True,
262
+ return_tensors="pt")
263
+ return processed_outputs
264
376
- width = height = hf_config.vision_config.image_size
377
- image = Image.new("RGB", (width, height), color=0)
378
- mm_data = {"image": [image] if num_images == 1 else [image] * num_images}
265
+ def _get_mm_fields_config(
266
+ self,
267
+ hf_inputs: BatchFeature,
268
+ hf_processor_mm_kwargs: Mapping[str, object],
269
+ ) -> Mapping[str, MultiModalFieldConfig]:
270
+ return dict(
271
+ pixel_values=MultiModalFieldConfig.batched("image"),
272
+ pixel_attention_mask=MultiModalFieldConfig.batched("image"),
273
+ image_embeds=MultiModalFieldConfig.batched("image"),
274
+ )
275
380
- return DummyData(seq_data, mm_data)
276
+ def _get_prompt_replacements(
277
+ self,
278
+ mm_items: MultiModalDataItems,
279
+ hf_processor_mm_kwargs: Mapping[str, object],
280
+ out_mm_kwargs: MultiModalKwargs,
281
+ ) -> list[PromptReplacement]:
282
+ hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
283
+
284
+ image_token = hf_processor.image_token.content
285
+ fake_image_token = hf_processor.fake_image_token.content
286
+ global_img_token = hf_processor.global_image_tag
287
+ image_seq_len = hf_processor.image_seq_len
288
+ grid_placeholder = "<row_{n_h}_col_{n_w}>"
289
+
290
+ p_img = image_token * image_seq_len
291
+ global_img_placeholder = fake_image_token + global_img_token + p_img
292
+ tile_img_placeholder = fake_image_token + grid_placeholder + p_img
293
+
294
+ def get_replacement_idefics3(item_idx: int) -> str:
295
+ images = mm_items.get_items("image", ImageProcessorItems)
296
+
297
+ image_size = images.get_image_size(item_idx)
298
+ grid_w, grid_h = self.info._get_image_feature_grid_size(
299
+ image_width=image_size.width,
300
+ image_height=image_size.height,
301
+ **hf_processor_mm_kwargs,
302
+ )
303
+ if grid_w == 0 and grid_h == 0:
304
+ image_placeholder = global_img_placeholder
305
+ else:
306
+ tiles_placeholder = list[str]()
307
+ for i in range(grid_h):
308
+ for j in range(grid_w):
309
+ placeholder_per_tile = tile_img_placeholder.format(
310
+ n_h=i + 1, n_w=j + 1)
311
+ tiles_placeholder.append(placeholder_per_tile)
312
+ # Add line break if it is the last tile in the row
313
+ if j == grid_w - 1:
314
+ tiles_placeholder.append("\n")
315
+
316
+ image_placeholder = "".join(
317
+ [*tiles_placeholder, "\n", global_img_placeholder])
318
+ return image_placeholder + fake_image_token
319
+
320
+ return [
321
+ PromptReplacement(
322
+ modality="image",
323
+ target=image_token,
324
+ replacement=get_replacement_idefics3,
325
+ )
326
+ ]
327
328
329
class Idefics3SimpleMLP(nn.Module):
@@ -453,7 +399,7 @@ class Idefics3Model(nn.Module):
399
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
400
super().__init__()
401
456
- config = vllm_config.model_config.hf_config
402
+ config: Idefics3Config = vllm_config.model_config.hf_config
403
quant_config = vllm_config.quant_config
404
405
self.config = config
@@ -541,15 +487,13 @@ class Idefics3Model(nn.Module):
487
self,
488
pixel_values: torch.Tensor,
489
pixel_attention_mask: Optional[torch.BoolTensor] = None,
544
- ) -> torch.Tensor:
490
+ ) -> NestedTensors:
491
# NOTE: we skip the step to select the vision feature layer since
492
# this is already done inside the vision tower
547
- batch_size, num_images, num_channels, height, width = pixel_values.shape
493
+ num_patches = [x.size(0) for x in pixel_values]
494
pixel_values = pixel_values.to(
495
dtype=self.vision_model.embeddings.patch_embedding.weight.dtype
496
) # fp16 compatibility
551
- pixel_values = pixel_values.view(batch_size * num_images,
552
- *pixel_values.shape[2:])
497
498
# Remove padding images - padding images are full 0.
499
nb_values_per_image = pixel_values.shape[1:].numel()
@@ -567,8 +511,6 @@ class Idefics3Model(nn.Module):
511
)
512
else:
513
# Remove padding images from the mask
570
- pixel_attention_mask = pixel_attention_mask.view(
571
- batch_size * num_images, *pixel_attention_mask.shape[2:])
514
pixel_attention_mask = pixel_attention_mask[
515
real_images_inds].contiguous()
516
@@ -587,10 +529,10 @@ class Idefics3Model(nn.Module):
529
patch_attention_mask=patch_attention_mask,
530
)
531
590
- return image_hidden_states
532
+ return image_hidden_states.split(num_patches)
533
534
def _process_image_pixels(
593
- self, inputs: Idefics3ImagePixelInputs) -> torch.Tensor:
535
+ self, inputs: Idefics3ImagePixelInputs) -> NestedTensors:
536
assert self.vision_model is not None
537
538
pixel_values = inputs["data"]
@@ -605,7 +547,9 @@ class Idefics3Model(nn.Module):
547
548
assert self.vision_model is not None
549
image_features = self._process_image_pixels(image_input)
608
- return self.connector(image_features)
550
+ num_patches = [x.size(0) for x in image_features]
551
+ image_features = torch.cat(image_features)
552
+ return self.connector(image_features).split(num_patches)
553
554
def get_input_embeddings(
555
self,
@@ -634,10 +578,10 @@ class Idefics3Model(nn.Module):
578
return hidden_states
579
580
637
-@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_idefics3)
638
-@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_idefics3_image_tokens)
639
-@INPUT_REGISTRY.register_dummy_data(dummy_data_for_idefics3)
640
-@INPUT_REGISTRY.register_input_processor(input_processor_for_idefics3)
581
+@MULTIMODAL_REGISTRY.register_processor(
582
+ Idefics3MultimodalProcessor,
583
+ info=Idefics3ProcessingInfo,
584
+ dummy_inputs=Idefics3DummyInputsBuilder)
585
class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
586
SupportsLoRA):
587
packed_modules_mapping = {
@@ -689,7 +633,7 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
633
if self.config.text_config.tie_word_embeddings:
634
self.lm_head.weight = self.model.text_model.wte.weight
635
self.logits_processor = LogitsProcessor(config.text_config.vocab_size)
692
- self.sampler = Sampler()
636
+ self.sampler = get_sampler()
637
638
def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
639
image_input = self.model._parse_and_validate_image_input(**kwargs)