@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 815079de8e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[VLM] merged multimodal processor and V1 support for idefics3 (#12660)

Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

Isotr0py committed Feb 4, 2025 at 20:00 UTC 815079de8e9dd984d474f7046412d5aedf4350ff
7 files changed +315 -457
docs/source/models/supported_models.md
+1 -1
@@ -733,7 +733,7 @@ See [this page](#generative-models) for more information on how to use generativ
733 * `HuggingFaceM4/Idefics3-8B-Llama3` etc.
734 * ✅︎
735 *
736 - *
736 + * ✅︎
737 - * `InternVLChatModel`
738 * InternVL 2.5, Mono-InternVL, InternVL 2.0
739 * T + I<sup>E+</sup>
tests/models/decoder_only/vision_language/test_models.py
+2 -2
@@ -254,14 +254,14 @@ VLM_TEST_SETTINGS = {
254 patch_hf_runner=model_utils.h2ovl_patch_hf_runner,
255 ),
256 "idefics3": VLMTestInfo(
257 - models=["HuggingFaceM4/Idefics3-8B-Llama3"],
257 + models=["HuggingFaceTB/SmolVLM-256M-Instruct"],
258 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
259 prompt_formatter=lambda img_prompt:f"<|begin_of_text|>User:{img_prompt}<end_of_utterance>\nAssistant:", # noqa: E501
260 img_idx_to_prompt=lambda idx: "<image>",
261 max_model_len=8192,
262 max_num_seqs=2,
263 auto_cls=AutoModelForVision2Seq,
264 - marks=[large_gpu_mark(min_gb=48)],
264 + hf_output_post_proc=model_utils.idefics3_trunc_hf_output,
265 ),
266 "intern_vl": VLMTestInfo(
267 models=[
tests/models/decoder_only/vision_language/vlm_utils/model_utils.py
+8
@@ -192,6 +192,14 @@ def deepseekvl2_trunc_hf_output(hf_output: RunnerOutput,
192 return output_ids, output_str, out_logprobs
193
194
195 +def idefics3_trunc_hf_output(hf_output: RunnerOutput,
196 + model: str) -> RunnerOutput:
197 + output_ids, output_str, out_logprobs = hf_output
198 + if output_str.endswith("<end_of_utterance>"):
199 + output_str = output_str.split("<end_of_utterance>")[0]
200 + return output_ids, output_str, out_logprobs
201 +
202 +
203 def minicpmv_trunc_hf_output(hf_output: RunnerOutput,
204 model: str) -> RunnerOutput:
205 output_ids, output_str, out_logprobs = hf_output
tests/models/multimodal/processing/test_common.py
+1
@@ -149,6 +149,7 @@ def _test_processing_correctness(
149 "adept/fuyu-8b",
150 "h2oai/h2ovl-mississippi-800m",
151 "OpenGVLab/InternVL2-1B",
152 + "HuggingFaceM4/Idefics3-8B-Llama3",
153 "llava-hf/llava-1.5-7b-hf",
154 "llava-hf/llava-v1.6-mistral-7b-hf",
155 "llava-hf/LLaVA-NeXT-Video-7B-hf",
tests/models/multimodal/processing/test_idefics3.py
+33 -146
@@ -1,13 +1,10 @@
1 # SPDX-License-Identifier: Apache-2.0
2 """Tests for Idefics3's multimodal preprocessing kwargs."""
3 -from typing import Optional
4 -
3 import pytest
6 -import torch
7 -from transformers import AutoImageProcessor, AutoTokenizer
4 +from transformers import Idefics3Config
5
9 -from vllm.inputs import InputContext, token_inputs
10 -from vllm.multimodal import MultiModalRegistry
6 +from vllm.multimodal import MULTIMODAL_REGISTRY
7 +from vllm.multimodal.utils import cached_get_tokenizer
8
9 from ....conftest import _ImageAssets
10 from ...utils import build_model_context
@@ -15,163 +12,53 @@ from ...utils import build_model_context
12 models = ["HuggingFaceM4/Idefics3-8B-Llama3"]
13
14
18 -# Wrap lazy imports to avoid initializing CUDA during test collection
19 -@pytest.fixture()
20 -def input_processor_for_idefics3():
21 - from vllm.model_executor.models.idefics3 import (
22 - input_processor_for_idefics3)
23 - return input_processor_for_idefics3
24 -
25 -
26 -@pytest.fixture()
27 -def dummy_data_for_idefics3():
28 - from vllm.model_executor.models.idefics3 import dummy_data_for_idefics3
29 - return dummy_data_for_idefics3
30 -
31 -
32 -@pytest.fixture()
33 -def get_max_idefics3_image_tokens():
34 - from vllm.model_executor.models.idefics3 import (
35 - get_max_idefics3_image_tokens)
36 - return get_max_idefics3_image_tokens
37 -
38 -
39 -@pytest.mark.parametrize("model", models)
40 -@pytest.mark.parametrize("longest_edge", [None, 168, 336, 400, 2 * 336])
41 -def test_input_mapper_override(model: str, image_assets: _ImageAssets,
42 - longest_edge: Optional[int]):
43 - """Ensure that the [default] input mapper handles size properly."""
44 -
45 - mm_processor_kwargs = {
46 - "size": {
47 - "longest_edge": longest_edge
48 - }
49 - } if longest_edge is not None else {}
50 - ctx = build_model_context(
51 - model_name=model,
52 - tokenizer_name=model,
53 - trust_remote_code=True,
54 - mm_processor_kwargs=mm_processor_kwargs,
55 - )
56 -
57 - hf_processor = AutoImageProcessor.from_pretrained(model,
58 - trust_remote_code=True,
59 - **mm_processor_kwargs)
60 -
61 - mm_registry = MultiModalRegistry()
62 - mm_registry.init_mm_limits_per_prompt(ctx.model_config)
63 -
64 - image = image_assets[0].pil_image
65 - hf_result = hf_processor.preprocess(
66 - image,
67 - return_tensors="pt",
68 - )
69 -
70 - vllm_result = mm_registry.map_input(
71 - ctx.model_config,
72 - {"image": image},
73 - )
74 -
75 - assert torch.all(hf_result["pixel_values"] == vllm_result["pixel_values"])
76 -
77 -
78 -@pytest.mark.parametrize("model", models)
79 -@pytest.mark.parametrize("longest_edge, expected_max_tokens", [
80 - (None, 2873),
81 - (168, 169),
82 - (336, 169),
83 - (400, 338),
84 - (672, 338),
85 -])
86 -def test_max_tokens_override(get_max_idefics3_image_tokens, model: str,
87 - longest_edge: Optional[int],
88 - expected_max_tokens: int):
89 - """Ensure get_max_idefics3_image_tokens handles mm_processor_kwargs."""
90 - size = {"longest_edge": longest_edge} if longest_edge is not None else None
91 - ctx = build_model_context(
92 - model_name=model,
93 - tokenizer_name=model,
94 - trust_remote_code=True,
95 - mm_processor_kwargs=None,
96 - )
97 -
98 - actual_max_tokens = get_max_idefics3_image_tokens(
99 - ctx=InputContext(ctx.model_config),
100 - size=size,
101 - )
102 -
103 - assert expected_max_tokens == actual_max_tokens
104 -
105 -
106 -@pytest.mark.parametrize("model", models)
107 -@pytest.mark.parametrize("longest_edge, toks_per_img, num_imgs", [
108 - (168, 169, 1),
109 - (168, 169, 2),
110 - (400, 338, 1),
111 - (400, 338, 2),
112 -])
113 -def test_dummy_data_override(dummy_data_for_idefics3, model: str,
114 - longest_edge: int, toks_per_img: int,
115 - num_imgs: int):
116 - """Ensure dummy_data_for_idefics3 handles num_crops properly."""
117 - # Same as the previous test - don't initialize mm_processor_kwargs
118 - # in this test and assume that the kwargs will be correctly expanded by
119 - # the partial when calling the dummy data func.
120 - size = {"longest_edge": longest_edge} if longest_edge is not None else None
121 - ctx = build_model_context(
122 - model_name=model,
123 - tokenizer_name=model,
124 - trust_remote_code=True,
125 - mm_processor_kwargs=None,
126 - )
127 -
128 - dummy_data = dummy_data_for_idefics3(
129 - ctx=ctx,
130 - seq_len=8192, # Should be bigger than num_imgs * toks_per_img
131 - mm_counts={"image": num_imgs},
132 - size=size)
133 - sequence_data = dummy_data.seq_data
134 - # Ensure we have the right number of placeholders per size
135 - image_token_id = ctx.get_hf_config().image_token_id
136 - img_tok_count = sequence_data.get_token_ids().count(image_token_id)
137 - assert img_tok_count == toks_per_img * num_imgs
138 -
139 -
15 @pytest.mark.parametrize("model", models)
141 -@pytest.mark.parametrize("longest_edge,expected_toks_per_img,num_imgs", [
142 - (336, 169 * (1**2 + 1), 1),
143 - (336, 169 * (1**2 + 1), 2),
144 - (400, 169 * (2**2 + 1), 1),
145 - (400, 169 * (2**2 + 1), 2),
146 -])
147 -def test_input_processor_override(input_processor_for_idefics3,
148 - image_assets: _ImageAssets, model: str,
149 - longest_edge: int,
150 - expected_toks_per_img: int, num_imgs: int):
16 +# yapf: disable
17 +@pytest.mark.parametrize(
18 + ("mm_processor_kwargs", "expected_toks_per_img"),
19 + [
20 + ({"size": {"longest_edge": 364}}, 169),
21 + ({"size": {"longest_edge": 728}}, 169 * (2**2 + 1)),
22 + ])
23 +# yapf: enable
24 +@pytest.mark.parametrize("num_imgs", [1, 2])
25 +def test_processor_override(image_assets: _ImageAssets, model: str,
26 + mm_processor_kwargs: dict[str, object],
27 + expected_toks_per_img: int, num_imgs: int):
28 """Ensure input_processor_for_idefics3 handles num_crops properly."""
29 # Same as the previous test - don't initialize mm_processor_kwargs
30 # in this test and assume that the kwargs will be correctly expanded by
31 # the partial when calling the custom input processor.
155 - size = {"longest_edge": longest_edge} if longest_edge is not None else None
32 ctx = build_model_context(
33 model_name=model,
34 tokenizer_name=model,
35 trust_remote_code=True,
36 mm_processor_kwargs=None,
37 + limit_mm_per_prompt={"image": num_imgs},
38 )
39 + tokenizer = cached_get_tokenizer(ctx.model_config.tokenizer)
40 + processor = MULTIMODAL_REGISTRY.create_processor(
41 + ctx.model_config,
42 + tokenizer=tokenizer,
43 + )
44 + hf_processor = processor.info.get_hf_processor(**mm_processor_kwargs)
45
46 # Build the image str / prompt based on the number of images we pass
164 - tokenizer = AutoTokenizer.from_pretrained(model)
47 placeholders = "<image>" if num_imgs == 1 else "\n".join(
48 f"Image-{i}: <image>\n" for i in range(1, num_imgs + 1))
49 prompt = f"<|begin_of_text|>User:{placeholders}\n<end_of_utterance>\nAssistant:" # noqa: E501
168 - images = [image_assets[0].pil_image.resize((336 * 4, 336 * 4))] * num_imgs
169 -
170 - inputs = token_inputs(prompt_token_ids=tokenizer.encode(prompt),
171 - prompt=prompt,
172 - multi_modal_data={"image": images})
50
174 - processed_inputs = input_processor_for_idefics3(ctx, inputs, size=size)
51 + # Build mm_data
52 + image_size = ctx.get_hf_config(Idefics3Config).vision_config.image_size
53 + dummy_image_size = (image_size * 4, image_size * 4)
54 + dummy_image = image_assets[0].pil_image.resize(dummy_image_size)
55 + mm_data = {"image": [dummy_image] * num_imgs}
56 +
57 + processed_inputs = processor.apply(prompt, mm_data, mm_processor_kwargs)
58 + # Ensure the placeholders format are correct
59 + hf_processed_inputs = hf_processor(text=prompt, images=mm_data["image"])
60 + assert processed_inputs["prompt_token_ids"] == hf_processed_inputs[
61 + "input_ids"][0]
62
63 # Ensure we have the right number of placeholders per num_crops size
64 image_token_id = ctx.get_hf_config().image_token_id
vllm/inputs/registry.py
+18
@@ -31,6 +31,17 @@ C = TypeVar("C", bound=PretrainedConfig, default=PretrainedConfig)
31 P = TypeVar("P", bound=ProcessorMixin, default=ProcessorMixin)
32
33
34 +class HashableDict(dict):
35 + """
36 + A dictionary that can be hashed by lru_cache.
37 + """
38 +
39 + # NOTE: pythonic dict is not hashable,
40 + # we override on it directly for simplicity
41 + def __hash__(self) -> int: # type: ignore[override]
42 + return hash(frozenset(self.items()))
43 +
44 +
45 @dataclass(frozen=True)
46 class InputContext:
47 """
@@ -104,6 +115,13 @@ class InputContext:
115 if isinstance(typ, type):
116 merged_kwargs["processor_cls"] = typ
117
118 + # NOTE: Pythonic dict is not hashable and will raise unhashable type
119 + # error when calling `cached_get_processor`, therefore we need to
120 + # wrap it to a hashable dict.
121 + for key, value in merged_kwargs.items():
122 + if isinstance(value, dict):
123 + merged_kwargs[key] = HashableDict(value)
124 +
125 hf_processor = cached_get_processor(
126 self.model_config.model,
127 trust_remote_code=self.model_config.trust_remote_code,
vllm/model_executor/models/idefics3.py
+252 -308
@@ -16,35 +16,35 @@
16 """Inference-only Idefics3 model compatible with HuggingFace weights."""
17
18 import math
19 -from typing import (Dict, Iterable, List, Literal, Mapping, NamedTuple,
20 - Optional, Set, Tuple, TypedDict, Union)
19 +from typing import (Dict, Iterable, List, Literal, Mapping, Optional, Set,
20 + Tuple, TypedDict, Union)
21
22 import torch
23 import torch.utils.checkpoint
24 -from PIL import Image
24 from torch import nn
26 -# Temporary solution for transformers below 4.46.0.
27 -from transformers import PretrainedConfig as Idefics3Config
28 -from transformers import ProcessorMixin as Idefics3ImageProcessor
25 +from transformers import (BatchFeature, Idefics3Config, Idefics3ImageProcessor,
26 + Idefics3Processor)
27
28 from vllm.attention import AttentionMetadata
29 from vllm.config import VllmConfig
32 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
33 - InputContext, token_inputs)
30 from vllm.logger import init_logger
31 from vllm.model_executor.layers.linear import ReplicatedLinear
32 from vllm.model_executor.layers.logits_processor import LogitsProcessor
33 from vllm.model_executor.layers.quantization import QuantizationConfig
38 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
34 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
35 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
36 from vllm.model_executor.models.module_mapping import MultiModelKeys
37 from vllm.model_executor.sampling_metadata import SamplingMetadata
38 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
43 -from vllm.multimodal.image import cached_get_image_processor
39 from vllm.multimodal.inputs import NestedTensors
45 -from vllm.sequence import IntermediateTensors, SequenceData
46 -from vllm.transformers_utils.processor import cached_get_processor
47 -from vllm.utils import is_list_of
40 +from vllm.multimodal.parse import ImageProcessorItems
41 +from vllm.multimodal.processing import (BaseMultiModalProcessor,
42 + BaseProcessingInfo,
43 + MultiModalDataItems,
44 + MultiModalFieldConfig,
45 + PromptReplacement)
46 +from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
47 +from vllm.sequence import IntermediateTensors
48
49 # yapf: disable
50 from .idefics2_vision_model import (
@@ -77,307 +77,253 @@ class Idefics3ImageEmbeddingInputs(TypedDict):
77 """
78
79
80 -class Idefics3ProcessorSize(NamedTuple):
81 - """Hashable wrapper for unhashable `size` dict of Idefics3Processor."""
82 - # NOTE: cached_get_processor/cached_get_image_processor uses lru_cache,
83 - # we need to use NamedTuple instead of TypedDict to avoid hashing issues.
84 - longest_edge: int
85 -
86 - def __contains__(self, key: str) -> bool:
87 - return key in self._asdict() and getattr(self, key) is not None
88 -
89 - def __getitem__(self, key: str) -> int:
90 - return getattr(self, key)
91 -
92 -
80 ImageInputs = Union[Idefics3ImagePixelInputs, Idefics3ImageEmbeddingInputs]
81
82
96 -def get_mm_processor_kwargs(size: Optional[Dict[str, int]] = None) -> Dict:
97 - mm_processor_kwargs = {}
98 - if size:
99 - mm_processor_kwargs["size"] = Idefics3ProcessorSize(**size)
100 - return mm_processor_kwargs
101 -
102 -
103 -def input_mapper_for_idefics3(
104 - ctx: InputContext,
105 - data: object,
106 - *,
107 - size: Optional[Dict[str, int]] = None,
108 -):
109 - model_config = ctx.model_config
110 - mm_processor_kwargs = get_mm_processor_kwargs(size)
111 - image_processor = cached_get_image_processor(
112 - model_config.model,
113 - trust_remote_code=model_config.trust_remote_code,
114 - **mm_processor_kwargs)
115 - if image_processor is None:
116 - raise RuntimeError("No HuggingFace processor is available "
117 - "to process the image object")
118 -
119 - if isinstance(data, Image.Image):
120 - images = [[data]]
121 - elif is_list_of(data, Image.Image):
122 - images = [data]
123 - else:
124 - raise TypeError(f"Invalid image type: {type(data)}")
125 -
126 - try:
127 - batch_data = image_processor(images,
128 - return_tensors="pt",
129 - return_row_col_info=True).data
130 - except Exception:
131 - logger.error("Failed to process image (%s)", data)
132 - raise
133 -
134 - return MultiModalKwargs(batch_data)
135 -
136 -
137 -def _resize_output_size(height: int,
138 - width: int,
139 - max_len: Optional[int] = None,
140 - min_len: Optional[int] = 1,
141 - max_size: Optional[int] = None) -> Tuple[int, int]:
142 - # Set default value for max_len if not provided
143 - max_len = max(height, width) if max_len is None else max_len
144 - aspect_ratio = width / height
145 -
146 - # Handle the maximum size constraint
147 - if max_size is not None:
148 - max_len = min(max_len, max_size)
149 -
150 - # Adjust dimensions according to the aspect ratio
151 - if width >= height:
152 - width = max_len
153 - height = int(width / aspect_ratio)
154 - else:
155 - height = max_len
156 - width = int(height * aspect_ratio)
157 -
158 - # Ensure both width and height are even (if needed)
159 - height += 1 if height % 2 != 0 else 0
160 - width += 1 if width % 2 != 0 else 0
161 -
162 - # Ensure dimensions are not smaller than the minimum length
163 - height = max(height, min_len)
164 - width = max(width, min_len)
165 -
166 - return height, width
167 -
168 -
169 -def _get_resize_output_image_size(
170 - image_size: Tuple[int, int],
171 - resolution_max_side: int,
172 - max_image_size: int = 1820,
173 -) -> Tuple[int, int]:
174 - if resolution_max_side > max_image_size:
175 - raise ValueError(
176 - "`resolution_max_side` cannot be larger than `max_image_size`")
177 -
178 - height, width = image_size
179 -
180 - # Find the output size, when rescaling the longest edge to max_len and
181 - # preserving the aspect ratio
182 - height, width = _resize_output_size(height,
183 - width,
184 - max_len=resolution_max_side)
185 -
186 - return height, width
187 -
188 -
189 -def _prompt_split_image(image_seq_len: int, image_rows: int, image_cols: int,
190 - fake_token_around_image: str, image_token: str,
191 - global_img_token: str) -> str:
192 - """
193 - Prompt with expanded image tokens for when the image is split
194 - into patches.
195 - """
196 - text_split_images = ""
197 - for n_h in range(image_rows):
198 - for n_w in range(image_cols):
199 - text_split_images += (fake_token_around_image +
200 - f"<row_{n_h + 1}_col_{n_w + 1}>" +
201 - image_token * image_seq_len)
202 - text_split_images += "\n"
203 -
204 - text_split_images += "\n" + _prompt_single_image(
205 - image_seq_len=image_seq_len,
206 - fake_token_around_image=fake_token_around_image,
207 - image_token=image_token,
208 - global_img_token=global_img_token)
209 - return text_split_images
210 -
211 -
212 -def _prompt_single_image(image_seq_len: int, fake_token_around_image: str,
213 - image_token: str, global_img_token: str):
214 - """Prompt with expanded image tokens for a single image."""
215 - return (fake_token_around_image + global_img_token +
216 - image_token * image_seq_len + fake_token_around_image)
217 -
218 -
219 -def _get_image_prompt_string(image_rows: int, image_cols: int,
220 - image_seq_len: int, fake_token_around_image: str,
221 - image_token: str, global_img_token: str):
222 - if image_rows == 0 and image_cols == 0:
223 - return _prompt_single_image(
224 - image_seq_len=image_seq_len,
225 - fake_token_around_image=fake_token_around_image,
226 - image_token=image_token,
227 - global_img_token=global_img_token,
228 - )
229 - return _prompt_split_image(image_seq_len, image_rows, image_cols,
230 - fake_token_around_image, image_token,
231 - global_img_token)
232 -
233 -
234 -def input_processor_for_idefics3(ctx: InputContext,
235 - inputs: DecoderOnlyInputs,
236 - *,
237 - size: Optional[Dict[str, int]] = None):
238 - multi_modal_data = inputs.get("multi_modal_data")
239 - if multi_modal_data is None or "image" not in multi_modal_data:
240 - return inputs
241 -
242 - model_config = ctx.model_config
243 - mm_processor_kwargs = get_mm_processor_kwargs(size)
244 - processor = cached_get_processor(model_config.model, **mm_processor_kwargs)
245 - image_processor = processor.image_processor
246 - tokenizer = processor.tokenizer
247 - size = image_processor.size['longest_edge']
248 - max_image_size = image_processor.max_image_size['longest_edge']
249 -
250 - image_data = multi_modal_data["image"]
251 - if isinstance(image_data, Image.Image):
252 - image_list = [image_data]
253 - elif is_list_of(image_data, Image.Image):
254 - image_list = image_data
255 - else:
256 - raise TypeError(f"Invalid image type: {type(image_data)}")
257 -
258 - image_rows = []
259 - image_cols = []
260 - for image in image_list:
261 - height, width = _get_resize_output_image_size(image.size, size)
262 -
263 - rows = math.ceil(height / max_image_size)
264 - cols = math.ceil(width / max_image_size)
265 - image_rows.append(rows)
266 - image_cols.append(cols)
267 - image_rows = [image_rows]
268 - image_cols = [image_cols]
269 -
270 - n_images_in_text = []
271 -
272 - text = inputs.get("prompt")
273 - if text is None:
274 - prompt_token_ids = inputs.get("prompt_token_ids", [])
275 - assert prompt_token_ids
276 - text = tokenizer.decode(prompt_token_ids)
277 -
278 - if isinstance(text, str):
279 - text = [text]
280 - elif not isinstance(text, list) and not isinstance(text[0], str):
281 - raise ValueError("Invalid input text. Please provide a string, "
282 - "or a list of strings")
283 -
284 - fake_image_token = processor.fake_image_token.content
285 - image_token = processor.image_token.content
286 - global_img_token = processor.global_image_tag
287 -
288 - prompt_strings = []
289 - for sample, sample_rows, sample_cols in zip(text, image_rows, image_cols):
290 - n_images_in_text.append(sample.count(image_token))
291 -
292 - # Replace the image token with fake tokens around the expanded
293 - # image token sequence of length `image_seq_len`
294 - image_prompt_strings = []
295 - for n_rows, n_cols in zip(sample_rows, sample_cols):
296 - image_prompt_string = _get_image_prompt_string(
297 - n_rows,
298 - n_cols,
299 - processor.image_seq_len,
300 - image_token=image_token,
301 - fake_token_around_image=fake_image_token,
302 - global_img_token=global_img_token,
303 - )
304 - image_prompt_strings.append(image_prompt_string)
305 -
306 - split_sample = sample.split(image_token)
307 - if len(split_sample) == 0:
308 - raise ValueError("The image token should be present in the text.")
309 -
310 - # Place in the image prompt strings where the image tokens are
311 - sample = split_sample[0]
312 - for i, image_prompt_string in enumerate(image_prompt_strings):
313 - sample += image_prompt_string + split_sample[i + 1]
314 - prompt_strings.append(sample)
83 +class Idefics3ProcessingInfo(BaseProcessingInfo):
84
316 - prompt_token_ids = tokenizer(text=prompt_strings[0]).input_ids
85 + def get_hf_processor(
86 + self,
87 + *,
88 + size: Optional[Dict[str, int]] = None) -> Idefics3Processor:
89 + if size is not None:
90 + return self.ctx.get_hf_processor(Idefics3Processor, size=size)
91
318 - return token_inputs(
319 - prompt_token_ids=prompt_token_ids,
320 - prompt=prompt_strings[0],
321 - multi_modal_data=multi_modal_data,
322 - )
92 + return self.ctx.get_hf_processor(Idefics3Processor)
93
94 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
95 + return {"image": None}
96
325 -def _get_max_num_image_patch(image_processor: Idefics3ImageProcessor) -> int:
326 - size = image_processor.size['longest_edge']
327 - max_image_size = image_processor.max_image_size['longest_edge']
328 - resized_height, resized_width = size, size
97 + def get_mm_max_tokens_per_item(
98 + self,
99 + seq_len: int,
100 + mm_counts: Mapping[str, int],
101 + ) -> Mapping[str, int]:
102 + hf_processor = self.get_hf_processor()
103 + image_processor: Idefics3ImageProcessor = hf_processor.image_processor
104 + grid_w, grid_h = self._get_image_feature_grid_size(
105 + image_width=image_processor.size['longest_edge'],
106 + image_height=image_processor.size['longest_edge'],
107 + )
108 + num_image_token = (grid_w * grid_h + 1) * hf_processor.image_seq_len
109 + # Calculate Non-image-token length
110 + # NOTE: <row_1_col_1> and <global-img> are special token for SmolVLM
111 + # but not for Idefic3, so we need to tokenize them to get actual length.
112 + tokenizer = self.get_tokenizer()
113 + tile_token_len = len(tokenizer.tokenize("<row_1_col_1>"))
114 + glob_token_len = len(tokenizer.tokenize(hf_processor.global_image_tag))
115 + # linebreak and <fake_token_around_image> always cost 1 token
116 + fake_token_len = lb_len = 1
117 + non_image_token = (grid_w * grid_h) * (
118 + tile_token_len + fake_token_len) + glob_token_len + (
119 + grid_h + 1) * lb_len + fake_token_len
120 + return {"image": num_image_token + non_image_token}
121 +
122 + def _resize_output_size(self,
123 + *,
124 + height: int,
125 + width: int,
126 + max_len: Optional[int] = None,
127 + min_len: Optional[int] = 1,
128 + max_size: Optional[int] = None) -> tuple[int, int]:
129 + # Set default value for max_len if not provided
130 + max_len = max(height, width) if max_len is None else max_len
131 + aspect_ratio = width / height
132 +
133 + # Handle the maximum size constraint
134 + if max_size is not None:
135 + max_len = min(max_len, max_size)
136 +
137 + # Adjust dimensions according to the aspect ratio
138 + if width >= height:
139 + width = max_len
140 + height = int(width / aspect_ratio)
141 + else:
142 + height = max_len
143 + width = int(height * aspect_ratio)
144
330 - grid_h = resized_height // max_image_size
331 - grid_w = resized_width // max_image_size
332 - return (grid_h * grid_w + 1)
145 + # Ensure both width and height are even (if needed)
146 + height += height % 2
147 + width += width % 2
148
149 + # Ensure dimensions are not smaller than the minimum length
150 + height = max(height, min_len)
151 + width = max(width, min_len)
152
335 -def get_max_idefics3_image_tokens(ctx: InputContext,
336 - *,
337 - size: Optional[Dict[str,
338 - int]] = None) -> int:
339 - model_config = ctx.model_config
340 - mm_processor_kwargs = get_mm_processor_kwargs(size)
341 - processor = cached_get_processor(model_config.model, **mm_processor_kwargs)
342 - image_seq_len = processor.image_seq_len
343 - image_processor = processor.image_processor
153 + return height, width
154
345 - max_num_image_patches = _get_max_num_image_patch(image_processor)
155 + def _get_resize_output_image_size(
156 + self,
157 + *,
158 + image_width: int,
159 + image_height: int,
160 + resolution_max_side: int,
161 + ) -> tuple[int, int]:
162 + hf_processor = self.get_hf_processor()
163 + image_processor: Idefics3ImageProcessor = hf_processor.image_processor
164 + max_image_size = image_processor.size['longest_edge']
165 + if resolution_max_side > max_image_size:
166 + raise ValueError(
167 + "`resolution_max_side` cannot be larger than `max_image_size`")
168 +
169 + height, width = image_height, image_width
170 +
171 + # Find the output size, when rescaling the longest edge to max_len and
172 + # preserving the aspect ratio
173 + height, width = self._resize_output_size(height=height,
174 + width=width,
175 + max_len=resolution_max_side)
176 + return height, width
177 +
178 + def _get_image_feature_grid_size(
179 + self,
180 + *,
181 + image_width: int,
182 + image_height: int,
183 + size: Optional[dict[str, object]] = None,
184 + ) -> tuple[int, int]:
185 + hf_processor = self.get_hf_processor(size=size)
186 + image_processor: Idefics3ImageProcessor = hf_processor.image_processor
187 + max_image_size = image_processor.max_image_size['longest_edge']
188 + size = image_processor.size['longest_edge']
189 + assert size % max_image_size == 0, (
190 + "`longest_edge` in image_processor's `size` must be divisible by "
191 + "`longest_edge` in `max_image_size`, this may be caused by "
192 + "incorrect mm_kwargs override.")
193 +
194 + resized_height, resized_width = self._get_resize_output_image_size(
195 + image_width=image_width,
196 + image_height=image_height,
197 + resolution_max_side=size,
198 + )
199 + if resized_height > max_image_size or resized_width > max_image_size:
200 + grid_h = math.ceil(resized_height / max_image_size)
201 + grid_w = math.ceil(resized_width / max_image_size)
202 + else:
203 + grid_h = grid_w = 0
204 + return grid_w, grid_h
205
347 - return max_num_image_patches * image_seq_len
206
207 +class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo]
208 + ):
209
350 -def dummy_data_for_idefics3(
351 - ctx: InputContext,
210 + def get_dummy_processor_inputs(
211 + self,
212 seq_len: int,
213 mm_counts: Mapping[str, int],
354 - *,
355 - size: Optional[Dict[str, int]] = None) -> DummyData:
356 - hf_config = ctx.get_hf_config()
357 - num_images = mm_counts["image"]
214 + ) -> ProcessorInputs:
215 + num_images = mm_counts.get("image", 0)
216 + hf_processor = self.info.get_hf_processor()
217 + image_processor: Idefics3ImageProcessor = hf_processor.image_processor
218 + longest_edge = image_processor.max_image_size['longest_edge']
219 + image_token: str = hf_processor.image_token.content
220 +
221 + mm_data = {
222 + "image":
223 + self._get_dummy_images(width=longest_edge,
224 + height=longest_edge,
225 + num_images=num_images)
226 + }
227 +
228 + return ProcessorInputs(
229 + prompt_text=image_token * num_images,
230 + mm_data=mm_data,
231 + )
232
359 - mm_processor_kwargs = get_mm_processor_kwargs(size)
360 - processor = cached_get_processor(ctx.model_config.model,
361 - **mm_processor_kwargs)
362 - max_num_image_patches = _get_max_num_image_patch(processor.image_processor)
363 - image_seq_len = processor.image_seq_len
364 - max_llm_image_tokens = max_num_image_patches * image_seq_len * num_images
233
366 - if seq_len - max_llm_image_tokens < 0:
367 - raise RuntimeError(
368 - f"Idefics3 cannot process {num_images} images in a prompt, "
369 - "please increase max_model_len or reduce image limit by "
370 - "--limit-mm-per-prompt.")
234 +class Idefics3MultimodalProcessor(
235 + BaseMultiModalProcessor[Idefics3ProcessingInfo]):
236
372 - seq_data = SequenceData.from_prompt_token_counts(
373 - (hf_config.image_token_id, max_llm_image_tokens),
374 - (0, seq_len - max_llm_image_tokens))
237 + def _call_hf_processor(
238 + self,
239 + prompt: str,
240 + mm_data: Mapping[str, object],
241 + mm_kwargs: Mapping[str, object],
242 + ) -> BatchFeature:
243 + if mm_data:
244 + processed_outputs = super()._call_hf_processor(
245 + prompt, mm_data, mm_kwargs)
246 + image_grids = [
247 + self.info._get_image_feature_grid_size(
248 + image_width=img.width,
249 + image_height=img.height,
250 + **mm_kwargs,
251 + ) for img in mm_data["images"]
252 + ]
253 + image_patches = list(map(lambda x: math.prod(x) + 1, image_grids))
254 + for key in ("pixel_values", "pixel_attention_mask"):
255 + data = processed_outputs.pop(key)
256 + data = data.flatten(0, 1).split(image_patches)
257 + processed_outputs[key] = data
258 + else:
259 + tokenizer = self.info.get_tokenizer()
260 + processed_outputs = tokenizer(prompt,
261 + add_special_tokens=True,
262 + return_tensors="pt")
263 + return processed_outputs
264
376 - width = height = hf_config.vision_config.image_size
377 - image = Image.new("RGB", (width, height), color=0)
378 - mm_data = {"image": [image] if num_images == 1 else [image] * num_images}
265 + def _get_mm_fields_config(
266 + self,
267 + hf_inputs: BatchFeature,
268 + hf_processor_mm_kwargs: Mapping[str, object],
269 + ) -> Mapping[str, MultiModalFieldConfig]:
270 + return dict(
271 + pixel_values=MultiModalFieldConfig.batched("image"),
272 + pixel_attention_mask=MultiModalFieldConfig.batched("image"),
273 + image_embeds=MultiModalFieldConfig.batched("image"),
274 + )
275
380 - return DummyData(seq_data, mm_data)
276 + def _get_prompt_replacements(
277 + self,
278 + mm_items: MultiModalDataItems,
279 + hf_processor_mm_kwargs: Mapping[str, object],
280 + out_mm_kwargs: MultiModalKwargs,
281 + ) -> list[PromptReplacement]:
282 + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
283 +
284 + image_token = hf_processor.image_token.content
285 + fake_image_token = hf_processor.fake_image_token.content
286 + global_img_token = hf_processor.global_image_tag
287 + image_seq_len = hf_processor.image_seq_len
288 + grid_placeholder = "<row_{n_h}_col_{n_w}>"
289 +
290 + p_img = image_token * image_seq_len
291 + global_img_placeholder = fake_image_token + global_img_token + p_img
292 + tile_img_placeholder = fake_image_token + grid_placeholder + p_img
293 +
294 + def get_replacement_idefics3(item_idx: int) -> str:
295 + images = mm_items.get_items("image", ImageProcessorItems)
296 +
297 + image_size = images.get_image_size(item_idx)
298 + grid_w, grid_h = self.info._get_image_feature_grid_size(
299 + image_width=image_size.width,
300 + image_height=image_size.height,
301 + **hf_processor_mm_kwargs,
302 + )
303 + if grid_w == 0 and grid_h == 0:
304 + image_placeholder = global_img_placeholder
305 + else:
306 + tiles_placeholder = list[str]()
307 + for i in range(grid_h):
308 + for j in range(grid_w):
309 + placeholder_per_tile = tile_img_placeholder.format(
310 + n_h=i + 1, n_w=j + 1)
311 + tiles_placeholder.append(placeholder_per_tile)
312 + # Add line break if it is the last tile in the row
313 + if j == grid_w - 1:
314 + tiles_placeholder.append("\n")
315 +
316 + image_placeholder = "".join(
317 + [*tiles_placeholder, "\n", global_img_placeholder])
318 + return image_placeholder + fake_image_token
319 +
320 + return [
321 + PromptReplacement(
322 + modality="image",
323 + target=image_token,
324 + replacement=get_replacement_idefics3,
325 + )
326 + ]
327
328
329 class Idefics3SimpleMLP(nn.Module):
@@ -453,7 +399,7 @@ class Idefics3Model(nn.Module):
399 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
400 super().__init__()
401
456 - config = vllm_config.model_config.hf_config
402 + config: Idefics3Config = vllm_config.model_config.hf_config
403 quant_config = vllm_config.quant_config
404
405 self.config = config
@@ -541,15 +487,13 @@ class Idefics3Model(nn.Module):
487 self,
488 pixel_values: torch.Tensor,
489 pixel_attention_mask: Optional[torch.BoolTensor] = None,
544 - ) -> torch.Tensor:
490 + ) -> NestedTensors:
491 # NOTE: we skip the step to select the vision feature layer since
492 # this is already done inside the vision tower
547 - batch_size, num_images, num_channels, height, width = pixel_values.shape
493 + num_patches = [x.size(0) for x in pixel_values]
494 pixel_values = pixel_values.to(
495 dtype=self.vision_model.embeddings.patch_embedding.weight.dtype
496 ) # fp16 compatibility
551 - pixel_values = pixel_values.view(batch_size * num_images,
552 - *pixel_values.shape[2:])
497
498 # Remove padding images - padding images are full 0.
499 nb_values_per_image = pixel_values.shape[1:].numel()
@@ -567,8 +511,6 @@ class Idefics3Model(nn.Module):
511 )
512 else:
513 # Remove padding images from the mask
570 - pixel_attention_mask = pixel_attention_mask.view(
571 - batch_size * num_images, *pixel_attention_mask.shape[2:])
514 pixel_attention_mask = pixel_attention_mask[
515 real_images_inds].contiguous()
516
@@ -587,10 +529,10 @@ class Idefics3Model(nn.Module):
529 patch_attention_mask=patch_attention_mask,
530 )
531
590 - return image_hidden_states
532 + return image_hidden_states.split(num_patches)
533
534 def _process_image_pixels(
593 - self, inputs: Idefics3ImagePixelInputs) -> torch.Tensor:
535 + self, inputs: Idefics3ImagePixelInputs) -> NestedTensors:
536 assert self.vision_model is not None
537
538 pixel_values = inputs["data"]
@@ -605,7 +547,9 @@ class Idefics3Model(nn.Module):
547
548 assert self.vision_model is not None
549 image_features = self._process_image_pixels(image_input)
608 - return self.connector(image_features)
550 + num_patches = [x.size(0) for x in image_features]
551 + image_features = torch.cat(image_features)
552 + return self.connector(image_features).split(num_patches)
553
554 def get_input_embeddings(
555 self,
@@ -634,10 +578,10 @@ class Idefics3Model(nn.Module):
578 return hidden_states
579
580
637 -@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_idefics3)
638 -@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_idefics3_image_tokens)
639 -@INPUT_REGISTRY.register_dummy_data(dummy_data_for_idefics3)
640 -@INPUT_REGISTRY.register_input_processor(input_processor_for_idefics3)
581 +@MULTIMODAL_REGISTRY.register_processor(
582 + Idefics3MultimodalProcessor,
583 + info=Idefics3ProcessingInfo,
584 + dummy_inputs=Idefics3DummyInputsBuilder)
585 class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
586 SupportsLoRA):
587 packed_modules_mapping = {
@@ -689,7 +633,7 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
633 if self.config.text_config.tie_word_embeddings:
634 self.lm_head.weight = self.model.text_model.wte.weight
635 self.logits_processor = LogitsProcessor(config.text_config.vocab_size)
692 - self.sampler = Sampler()
636 + self.sampler = get_sampler()
637
638 def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
639 image_input = self.model._parse_and_validate_image_input(**kwargs)