A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Model] Add Idefics3 support (#9767)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: B-201 <Joy25810@foxmail.com> Co-authored-by: B-201 <Joy25810@foxmail.com>
Jee Jee Li committed
Nov 6, 2024 at 19:41 UTC
a5bba7d234b4e0d82e6a64de82a8497760ed44cf
8 files changed
+723
-1
docs/source/models/supported_models.rst
+6
@@ -446,6 +446,12 @@ Text Generation
446
- :code:`h2oai/h2ovl-mississippi-800m`, :code:`h2oai/h2ovl-mississippi-2b`, etc.
447
-
448
- ✅︎
449
+ * - :code:`Idefics3ForConditionalGeneration`
450
+ - Idefics3
451
+ - T + I
452
+ - :code:`HuggingFaceM4/Idefics3-8B-Llama3` etc.
453
+ -
454
+ -
455
* - :code:`InternVLChatModel`
456
- InternVL2
457
- T + I\ :sup:`E+`
examples/offline_inference_vision_language.py
+17
@@ -377,6 +377,22 @@ def run_glm4v(question: str, modality: str):
377
return llm, prompt, stop_token_ids
378
379
380
+# Idefics3-8B-Llama3
381
+def run_idefics3(question: str, modality: str):
382
+ assert modality == "image"
383
+ model_name = "HuggingFaceM4/Idefics3-8B-Llama3"
384
+
385
+ llm = LLM(model=model_name,
386
+ max_model_len=8192,
387
+ max_num_seqs=2,
388
+ enforce_eager=True)
389
+ prompt = (
390
+ f"<|begin_of_text|>User:<image>{question}<end_of_utterance>\nAssistant:"
391
+ )
392
+ stop_token_ids = None
393
+ return llm, prompt, stop_token_ids
394
+
395
+
396
model_example_map = {
397
"llava": run_llava,
398
"llava-next": run_llava_next,
@@ -397,6 +413,7 @@ model_example_map = {
413
"mllama": run_mllama,
414
"molmo": run_molmo,
415
"glm4v": run_glm4v,
416
+ "idefics3": run_idefics3,
417
}
418
419
examples/offline_inference_vision_language_multi_image.py
+25
@@ -290,6 +290,30 @@ def load_mllama(question, image_urls: List[str]) -> ModelRequestData:
290
)
291
292
293
+def load_idefics3(question, image_urls: List[str]) -> ModelRequestData:
294
+ model_name = "HuggingFaceM4/Idefics3-8B-Llama3"
295
+
296
+ # The configuration below has been confirmed to launch on a single L40 GPU.
297
+ llm = LLM(
298
+ model=model_name,
299
+ max_model_len=8192,
300
+ max_num_seqs=16,
301
+ enforce_eager=True,
302
+ limit_mm_per_prompt={"image": len(image_urls)},
303
+ )
304
+
305
+ placeholders = "\n".join(f"Image-{i}: <image>\n"
306
+ for i, _ in enumerate(image_urls, start=1))
307
+ prompt = f"<|begin_of_text|>User:{placeholders}\n{question}<end_of_utterance>\nAssistant:" # noqa: E501
308
+ return ModelRequestData(
309
+ llm=llm,
310
+ prompt=prompt,
311
+ stop_token_ids=None,
312
+ image_data=[fetch_image(url) for url in image_urls],
313
+ chat_template=None,
314
+ )
315
+
316
+
317
model_example_map = {
318
"phi3_v": load_phi3v,
319
"h2ovl_chat": load_h2onvl,
@@ -298,6 +322,7 @@ model_example_map = {
322
"qwen2_vl": load_qwen2_vl,
323
"qwen_vl_chat": load_qwenvl_chat,
324
"mllama": load_mllama,
325
+ "idefics3": load_idefics3,
326
}
327
328
tests/models/decoder_only/vision_language/test_models.py
+16
@@ -327,6 +327,22 @@ VLM_TEST_SETTINGS = {
327
vllm_output_post_proc=model_utils.qwen_vllm_to_hf_output,
328
prompt_path_encoder=model_utils.qwen_prompt_path_encoder,
329
),
330
+ "idefics3": VLMTestInfo(
331
+ models=["HuggingFaceM4/Idefics3-8B-Llama3"],
332
+ test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
333
+ prompt_formatter=lambda img_prompt:f"<|begin_of_text|>User:{img_prompt}<end_of_utterance>\nAssistant:", # noqa: E501
334
+ img_idx_to_prompt=lambda idx: "<image>",
335
+ max_model_len=8192,
336
+ max_num_seqs=2,
337
+ auto_cls=AutoModelForVision2Seq,
338
+ marks=[
339
+ pytest.mark.skipif(
340
+ transformers.__version__ < "4.46.0",
341
+ reason="Model introduced in HF >= 4.46.0"
342
+ ),
343
+ large_gpu_mark(min_gb=48),
344
+ ],
345
+ ),
346
### Tensor parallel / multi-gpu broadcast tests
347
"broadcast-chameleon": VLMTestInfo(
348
models=["facebook/chameleon-7b"],
vllm/entrypoints/chat_utils.py
+2
@@ -187,6 +187,8 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
187
return "<|vision_start|><|image_pad|><|vision_end|>"
188
if model_type == "molmo":
189
return ""
190
+ if model_type == "idefics3":
191
+ return "<image>"
192
193
raise TypeError(f"Unknown {modality} model type: {model_type}")
194
elif modality == "audio":
vllm/model_executor/models/idefics2_vision_model.py
+24
-1
@@ -15,7 +15,7 @@
15
# limitations under the License.
16
"""PyTorch Idefics2 model."""
17
18
-from typing import Optional
18
+from typing import Iterable, Optional, Tuple
19
20
import torch
21
from torch import nn
@@ -29,6 +29,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
29
QKVParallelLinear,
30
RowParallelLinear)
31
from vllm.model_executor.layers.quantization import QuantizationConfig
32
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
33
34
35
class Idefics2VisionEmbeddings(nn.Module):
@@ -329,3 +330,25 @@ class Idefics2VisionTransformer(nn.Module):
330
encoder_outputs = self.encoder(hidden_states)
331
last_hidden_state = self.post_layernorm(encoder_outputs)
332
return last_hidden_state
333
+
334
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
335
+ stacked_params_mapping = [
336
+ # (param_name, shard_name, shard_id)
337
+ ("qkv_proj", "q_proj", "q"),
338
+ ("qkv_proj", "k_proj", "k"),
339
+ ("qkv_proj", "v_proj", "v"),
340
+ ]
341
+ params_dict = dict(self.named_parameters())
342
+ for name, loaded_weight in weights:
343
+ for param_name, weight_name, shard_id in stacked_params_mapping:
344
+ if weight_name not in name:
345
+ continue
346
+ param = params_dict[name.replace(weight_name, param_name)]
347
+ weight_loader = param.weight_loader
348
+ weight_loader(param, loaded_weight, shard_id)
349
+ break
350
+ else:
351
+ param = params_dict[name]
352
+ weight_loader = getattr(param, "weight_loader",
353
+ default_weight_loader)
354
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/idefics3.py
new
+632
@@ -0,0 +1,632 @@
1
+# Copyright 2024 the HuggingFace Inc. team. All rights reserved.
2
+#
3
+# Licensed under the Apache License, Version 2.0 (the "License");
4
+# you may not use this file except in compliance with the License.
5
+# You may obtain a copy of the License at
6
+#
7
+# http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+# Unless required by applicable law or agreed to in writing, software
10
+# distributed under the License is distributed on an "AS IS" BASIS,
11
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+# See the License for the specific language governing permissions and
13
+# limitations under the License.
14
+"""Inference-only Idefics3 model compatible with HuggingFace weights."""
15
+
16
+import math
17
+from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
18
+ TypedDict, Union)
19
+
20
+import torch
21
+import torch.utils.checkpoint
22
+from PIL import Image
23
+from torch import nn
24
+# Temporary solution for transformers below 4.46.0.
25
+from transformers import PretrainedConfig as Idefics3Config
26
+
27
+from vllm.attention import AttentionMetadata
28
+from vllm.config import CacheConfig, MultiModalConfig
29
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
30
+ InputContext, token_inputs)
31
+from vllm.logger import init_logger
32
+from vllm.model_executor.layers.linear import ReplicatedLinear
33
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
34
+from vllm.model_executor.layers.quantization import QuantizationConfig
35
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
36
+from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
37
+from vllm.model_executor.sampling_metadata import SamplingMetadata
38
+from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalInputs
39
+from vllm.multimodal.image import cached_get_image_processor
40
+from vllm.sequence import IntermediateTensors, SequenceData
41
+from vllm.transformers_utils.processor import cached_get_processor
42
+from vllm.utils import is_list_of
43
+
44
+# yapf: disable
45
+from .idefics2_vision_model import (
46
+ Idefics2VisionTransformer as Idefics3VisionTransformer)
47
+# yapf: enable
48
+from .interfaces import SupportsMultiModal
49
+from .llama import LlamaModel
50
+from .utils import AutoWeightsLoader, flatten_bn, merge_multimodal_embeddings
51
+
52
+logger = init_logger(__name__)
53
+
54
+
55
+class Idefics3ImagePixelInputs(TypedDict):
56
+ type: Literal["pixel_values"]
57
+ data: torch.Tensor
58
+ """
59
+ Shape: `(batch_size * num_images, num_channels, height, width)`
60
+ """
61
+ rows: List[int]
62
+ cols: List[int]
63
+ pixel_attention_mask: Optional[torch.BoolTensor]
64
+
65
+
66
+class Idefics3ImageEmbeddingInputs(TypedDict):
67
+ type: Literal["image_embeds"]
68
+ data: torch.Tensor
69
+ """
70
+ Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
71
+ `hidden_size` must match the hidden size of language model backbone.
72
+ """
73
+
74
+
75
+ImageInputs = Union[Idefics3ImagePixelInputs, Idefics3ImageEmbeddingInputs]
76
+
77
+
78
+def input_mapper_for_idefics3(
79
+ ctx: InputContext,
80
+ data: object,
81
+):
82
+ model_config = ctx.model_config
83
+ image_processor = cached_get_image_processor(
84
+ model_config.model, trust_remote_code=model_config.trust_remote_code)
85
+ if image_processor is None:
86
+ raise RuntimeError("No HuggingFace processor is available "
87
+ "to process the image object")
88
+
89
+ if isinstance(data, Image.Image):
90
+ images = [[data]]
91
+ elif is_list_of(data, Image.Image):
92
+ images = [data]
93
+ else:
94
+ raise TypeError(f"Invalid image type: {type(data)}")
95
+
96
+ try:
97
+ batch_data = image_processor(images,
98
+ return_tensors="pt",
99
+ return_row_col_info=True).data
100
+ except Exception:
101
+ logger.error("Failed to process image (%s)", data)
102
+ raise
103
+
104
+ return MultiModalInputs(batch_data)
105
+
106
+
107
+def _resize_output_size(height: int,
108
+ width: int,
109
+ max_len: Optional[int] = None,
110
+ min_len: Optional[int] = 1,
111
+ max_size: Optional[int] = None) -> Tuple[int, int]:
112
+ # Set default value for max_len if not provided
113
+ max_len = max(height, width) if max_len is None else max_len
114
+ aspect_ratio = width / height
115
+
116
+ # Handle the maximum size constraint
117
+ if max_size is not None:
118
+ max_len = min(max_len, max_size)
119
+
120
+ # Adjust dimensions according to the aspect ratio
121
+ if width >= height:
122
+ width = max_len
123
+ height = int(width / aspect_ratio)
124
+ else:
125
+ height = max_len
126
+ width = int(height * aspect_ratio)
127
+
128
+ # Ensure both width and height are even (if needed)
129
+ height += 1 if height % 2 != 0 else 0
130
+ width += 1 if width % 2 != 0 else 0
131
+
132
+ # Ensure dimensions are not smaller than the minimum length
133
+ height = max(height, min_len)
134
+ width = max(width, min_len)
135
+
136
+ return height, width
137
+
138
+
139
+def _get_resize_output_image_size(
140
+ image_size: Tuple[int, int],
141
+ resolution_max_side: int,
142
+ max_image_size: int = 1820,
143
+) -> Tuple[int, int]:
144
+ if resolution_max_side > max_image_size:
145
+ raise ValueError(
146
+ "`resolution_max_side` cannot be larger than `max_image_size`")
147
+
148
+ height, width = image_size
149
+
150
+ # Find the output size, when rescaling the longest edge to max_len and
151
+ # preserving the aspect ratio
152
+ height, width = _resize_output_size(height,
153
+ width,
154
+ max_len=resolution_max_side)
155
+
156
+ return height, width
157
+
158
+
159
+def _prompt_split_image(image_seq_len: int, image_rows: int, image_cols: int,
160
+ fake_token_around_image: str, image_token: str,
161
+ global_img_token: str) -> str:
162
+ """
163
+ Prompt with expanded image tokens for when the image is split
164
+ into patches.
165
+ """
166
+ text_split_images = ""
167
+ for n_h in range(image_rows):
168
+ for n_w in range(image_cols):
169
+ text_split_images += (fake_token_around_image +
170
+ f"<row_{n_h + 1}_col_{n_w + 1}>" +
171
+ image_token * image_seq_len)
172
+ text_split_images += "\n"
173
+
174
+ text_split_images += "\n" + _prompt_single_image(
175
+ image_seq_len=image_seq_len,
176
+ fake_token_around_image=fake_token_around_image,
177
+ image_token=image_token,
178
+ global_img_token=global_img_token)
179
+ return text_split_images
180
+
181
+
182
+def _prompt_single_image(image_seq_len: int, fake_token_around_image: str,
183
+ image_token: str, global_img_token: str):
184
+ """Prompt with expanded image tokens for a single image."""
185
+ return (fake_token_around_image + global_img_token +
186
+ image_token * image_seq_len + fake_token_around_image)
187
+
188
+
189
+def _get_image_prompt_string(image_rows: int, image_cols: int,
190
+ image_seq_len: int, fake_token_around_image: str,
191
+ image_token: str, global_img_token: str):
192
+ if image_rows == 0 and image_cols == 0:
193
+ return _prompt_single_image(
194
+ image_seq_len=image_seq_len,
195
+ fake_token_around_image=fake_token_around_image,
196
+ image_token=image_token,
197
+ global_img_token=global_img_token,
198
+ )
199
+ return _prompt_split_image(image_seq_len, image_rows, image_cols,
200
+ fake_token_around_image, image_token,
201
+ global_img_token)
202
+
203
+
204
+def input_processor_for_idefics3(ctx: InputContext, inputs: DecoderOnlyInputs):
205
+ multi_modal_data = inputs.get("multi_modal_data")
206
+ if multi_modal_data is None or "image" not in multi_modal_data:
207
+ return inputs
208
+
209
+ model_config = ctx.model_config
210
+ processor = cached_get_processor(model_config.model)
211
+ image_processor = processor.image_processor
212
+ tokenizer = processor.tokenizer
213
+ size = image_processor.size['longest_edge']
214
+ max_image_size = image_processor.max_image_size['longest_edge']
215
+
216
+ image_data = multi_modal_data["image"]
217
+ if isinstance(image_data, Image.Image):
218
+ image_list = [image_data]
219
+ elif is_list_of(image_data, Image.Image):
220
+ image_list = image_data
221
+ else:
222
+ raise TypeError(f"Invalid image type: {type(image_data)}")
223
+
224
+ image_rows = []
225
+ image_cols = []
226
+ for image in image_list:
227
+ height, width = _get_resize_output_image_size(image.size, size)
228
+
229
+ rows = math.ceil(height / max_image_size)
230
+ cols = math.ceil(width / max_image_size)
231
+ image_rows.append(rows)
232
+ image_cols.append(cols)
233
+ image_rows = [image_rows]
234
+ image_cols = [image_cols]
235
+
236
+ n_images_in_text = []
237
+
238
+ text = inputs.get("prompt")
239
+ if text is not None:
240
+ if isinstance(text, str):
241
+ text = [text]
242
+ elif not isinstance(text, list) and not isinstance(text[0], str):
243
+ raise ValueError("Invalid input text. Please provide a string, "
244
+ "or a list of strings")
245
+
246
+ fake_image_token = processor.fake_image_token.content
247
+ image_token = processor.image_token.content
248
+ global_img_token = processor.global_image_tag
249
+
250
+ prompt_strings = []
251
+ for sample, sample_rows, sample_cols in zip(text, image_rows,
252
+ image_cols):
253
+ n_images_in_text.append(sample.count(image_token))
254
+
255
+ # Replace the image token with fake tokens around the expanded
256
+ # image token sequence of length `image_seq_len`
257
+ image_prompt_strings = []
258
+ for n_rows, n_cols in zip(sample_rows, sample_cols):
259
+ image_prompt_string = _get_image_prompt_string(
260
+ n_rows,
261
+ n_cols,
262
+ processor.image_seq_len,
263
+ image_token=image_token,
264
+ fake_token_around_image=fake_image_token,
265
+ global_img_token=global_img_token,
266
+ )
267
+ image_prompt_strings.append(image_prompt_string)
268
+
269
+ split_sample = sample.split(image_token)
270
+ if len(split_sample) == 0:
271
+ raise ValueError(
272
+ "The image token should be present in the text.")
273
+
274
+ # Place in the image prompt strings where the image tokens are
275
+ sample = split_sample[0]
276
+ for i, image_prompt_string in enumerate(image_prompt_strings):
277
+ sample += image_prompt_string + split_sample[i + 1]
278
+ prompt_strings.append(sample)
279
+
280
+ prompt_token_ids = tokenizer(text=prompt_strings[0]).input_ids
281
+
282
+ return token_inputs(
283
+ prompt_token_ids=prompt_token_ids,
284
+ prompt=prompt_strings[0],
285
+ multi_modal_data=multi_modal_data,
286
+ )
287
+
288
+
289
+def get_max_idefics3_image_tokens(ctx: InputContext,
290
+ *,
291
+ num_crops: Optional[int] = None):
292
+ model_config = ctx.model_config
293
+ processor = cached_get_processor(model_config.model)
294
+ image_seq_len = processor.image_seq_len
295
+ image_processor = processor.image_processor
296
+
297
+ size = image_processor.size['longest_edge']
298
+ max_image_size = image_processor.max_image_size['longest_edge']
299
+ resized_height, resized_width = size, size
300
+
301
+ grid_h = resized_height // max_image_size
302
+ grid_w = resized_width // max_image_size
303
+
304
+ return (grid_h * grid_w + 1) * image_seq_len
305
+
306
+
307
+def dummy_data_for_idefics3(ctx: InputContext, seq_len: int,
308
+ mm_counts: Mapping[str, int]) -> DummyData:
309
+ hf_config = ctx.get_hf_config()
310
+ num_images = mm_counts["image"]
311
+
312
+ processor = cached_get_processor(ctx.model_config.model)
313
+ image_seq_len = processor.image_seq_len
314
+ max_llm_image_tokens = 17 * image_seq_len * num_images
315
+
316
+ seq_data = SequenceData.from_prompt_token_counts(
317
+ (hf_config.image_token_id, max_llm_image_tokens), (0, seq_len))
318
+
319
+ width = height = hf_config.vision_config.image_size
320
+ image = Image.new("RGB", (width, height), color=0)
321
+ mm_data = {"image": [image] if num_images == 1 else [image] * num_images}
322
+
323
+ return DummyData(seq_data, mm_data)
324
+
325
+
326
+class Idefics3SimpleMLP(nn.Module):
327
+
328
+ def __init__(self, config):
329
+ super().__init__()
330
+ input_size = config.vision_config.hidden_size * (config.scale_factor**
331
+ 2)
332
+ output_size = config.text_config.hidden_size
333
+ self.proj = ReplicatedLinear(input_size, output_size, bias=False)
334
+
335
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
336
+ out, _ = self.proj(x)
337
+ return out
338
+
339
+
340
+class Idefics3Connector(nn.Module):
341
+
342
+ def __init__(self, config):
343
+ super().__init__()
344
+ self.scale_factor = config.scale_factor
345
+ self.modality_projection = Idefics3SimpleMLP(config)
346
+
347
+ def pixel_shuffle(self,
348
+ x: torch.Tensor,
349
+ scale_factor: int = 2) -> torch.Tensor:
350
+ bsz, seq, embed_dim = x.size()
351
+ height = width = int(seq**0.5)
352
+ x = x.view(bsz, height, width, embed_dim)
353
+ x = x.view(bsz, height, int(width / scale_factor),
354
+ embed_dim * scale_factor)
355
+ x = x.permute(0, 2, 1, 3)
356
+ x = x.reshape(
357
+ bsz,
358
+ int(width / scale_factor),
359
+ int(height / scale_factor),
360
+ embed_dim * (scale_factor**2),
361
+ )
362
+ x = x.permute(0, 2, 1, 3)
363
+ x = x.reshape(bsz, int(seq / (scale_factor**2)),
364
+ embed_dim * (scale_factor**2))
365
+ return x
366
+
367
+ def forward(self, image_hidden_states: torch.Tensor) -> torch.Tensor:
368
+ image_hidden_states = self.pixel_shuffle(image_hidden_states,
369
+ self.scale_factor)
370
+ image_hidden_states = self.modality_projection(image_hidden_states)
371
+ return image_hidden_states
372
+
373
+
374
+class Idefics3Model(nn.Module):
375
+
376
+ def __init__(
377
+ self,
378
+ config: Idefics3Config,
379
+ cache_config: Optional[CacheConfig] = None,
380
+ quant_config: Optional[QuantizationConfig] = None,
381
+ ):
382
+ super().__init__()
383
+ self.config = config
384
+ self.padding_idx = self.config.text_config.pad_token_id
385
+ self.vocab_size = self.config.text_config.vocab_size
386
+
387
+ self.vision_model = Idefics3VisionTransformer(config.vision_config,
388
+ quant_config)
389
+ self.connector = Idefics3Connector(config)
390
+ self.text_model = LlamaModel(config.text_config, cache_config,
391
+ quant_config)
392
+
393
+ self.image_seq_len = int(
394
+ ((config.vision_config.image_size //
395
+ config.vision_config.patch_size)**2) / (config.scale_factor**2))
396
+ self.image_token_id = self.config.image_token_id
397
+
398
+ def _validate_pixel_values(
399
+ self, data: Union[torch.Tensor, List[torch.Tensor]]
400
+ ) -> Union[torch.Tensor, List[torch.Tensor]]:
401
+
402
+ h = w = self.config.vision_config.image_size
403
+ expected_dims = (3, h, w)
404
+
405
+ def _validate_shape(d: torch.Tensor):
406
+ actual_dims = tuple(d.shape[1:])
407
+
408
+ if actual_dims != expected_dims:
409
+ expected_expr = ("num_patches", *map(str, expected_dims))
410
+ raise ValueError(
411
+ "The expected shape of pixel values per image per batch "
412
+ f"is {expected_expr}. You supplied {tuple(d.shape)}.")
413
+
414
+ for d in data:
415
+ _validate_shape(d)
416
+
417
+ return data
418
+
419
+ def _parse_and_validate_image_input(
420
+ self, **kwargs: object) -> Optional[ImageInputs]:
421
+ pixel_values = kwargs.pop("pixel_values", None)
422
+ image_embeds = kwargs.pop("image_embeds", None)
423
+ rows = kwargs.pop("rows", None)
424
+ cols = kwargs.pop("cols", None)
425
+ pixel_attention_mask = kwargs.pop("pixel_attention_mask", None)
426
+
427
+ if pixel_values is None and image_embeds is None:
428
+ return None
429
+
430
+ if image_embeds is not None:
431
+ if not isinstance(image_embeds, (torch.Tensor, list)):
432
+ raise ValueError("Incorrect type of image embeddings. "
433
+ f"Got type: {type(image_embeds)}")
434
+
435
+ return Idefics3ImageEmbeddingInputs(
436
+ type="image_embeds",
437
+ data=flatten_bn(image_embeds, concat=True),
438
+ )
439
+
440
+ if pixel_values is not None:
441
+ if not isinstance(pixel_values, (torch.Tensor, list)):
442
+ raise ValueError("Incorrect type of pixel values. "
443
+ f"Got type: {type(pixel_values)}")
444
+
445
+ return Idefics3ImagePixelInputs(type="pixel_values",
446
+ data=self._validate_pixel_values(
447
+ flatten_bn(pixel_values,
448
+ concat=True)),
449
+ rows=rows,
450
+ cols=cols,
451
+ pixel_attention_mask=flatten_bn(
452
+ pixel_attention_mask,
453
+ concat=True))
454
+
455
+ raise AssertionError("This line should be unreachable.")
456
+
457
+ def _image_pixels_to_features(
458
+ self,
459
+ pixel_values: torch.Tensor,
460
+ pixel_attention_mask: Optional[torch.BoolTensor] = None,
461
+ ) -> torch.Tensor:
462
+ # NOTE: we skip the step to select the vision feature layer since
463
+ # this is already done inside the vision tower
464
+ batch_size, num_images, num_channels, height, width = pixel_values.shape
465
+ pixel_values = pixel_values.to(
466
+ dtype=self.vision_model.embeddings.patch_embedding.weight.dtype
467
+ ) # fp16 compatibility
468
+ pixel_values = pixel_values.view(batch_size * num_images,
469
+ *pixel_values.shape[2:])
470
+
471
+ # Remove padding images - padding images are full 0.
472
+ nb_values_per_image = pixel_values.shape[1:].numel()
473
+ real_images_inds = (pixel_values == 0.0).sum(
474
+ dim=(-1, -2, -3)) != nb_values_per_image
475
+ pixel_values = pixel_values[real_images_inds].contiguous()
476
+
477
+ # Handle the vision attention mask
478
+ if pixel_attention_mask is None:
479
+ pixel_attention_mask = torch.ones(
480
+ size=(pixel_values.size(0), pixel_values.size(2),
481
+ pixel_values.size(3)),
482
+ dtype=torch.bool,
483
+ device=pixel_values.device,
484
+ )
485
+ else:
486
+ # Remove padding images from the mask
487
+ pixel_attention_mask = pixel_attention_mask.view(
488
+ batch_size * num_images, *pixel_attention_mask.shape[2:])
489
+ pixel_attention_mask = pixel_attention_mask[
490
+ real_images_inds].contiguous()
491
+
492
+ patch_size = self.config.vision_config.patch_size
493
+ patches_subgrid = pixel_attention_mask.unfold(dimension=1,
494
+ size=patch_size,
495
+ step=patch_size)
496
+ patches_subgrid = patches_subgrid.unfold(dimension=2,
497
+ size=patch_size,
498
+ step=patch_size)
499
+ patch_attention_mask = (patches_subgrid.sum(dim=(-1, -2)) > 0).bool()
500
+
501
+ # Get sequence from the vision encoder
502
+ image_hidden_states = self.vision_model(
503
+ pixel_values=pixel_values,
504
+ patch_attention_mask=patch_attention_mask,
505
+ )
506
+
507
+ return image_hidden_states
508
+
509
+ def _process_image_pixels(
510
+ self, inputs: Idefics3ImagePixelInputs) -> torch.Tensor:
511
+ assert self.vision_model is not None
512
+
513
+ pixel_values = inputs["data"]
514
+ pixel_attention_mask = inputs["pixel_attention_mask"]
515
+
516
+ return self._image_pixels_to_features(pixel_values,
517
+ pixel_attention_mask)
518
+
519
+ def _process_image_input(self, image_input: ImageInputs) -> torch.Tensor:
520
+ if image_input["type"] == "image_embeds":
521
+ return image_input["data"]
522
+
523
+ assert self.vision_model is not None
524
+ image_features = self._process_image_pixels(image_input)
525
+ return self.connector(image_features)
526
+
527
+ def forward(
528
+ self,
529
+ input_ids: torch.Tensor,
530
+ positions: torch.Tensor,
531
+ kv_caches: List[torch.Tensor],
532
+ attn_metadata: AttentionMetadata,
533
+ intermediate_tensors: Optional[IntermediateTensors] = None,
534
+ **kwargs: object,
535
+ ) -> Union[torch.Tensor, IntermediateTensors]:
536
+ if intermediate_tensors is not None:
537
+ input_ids = None
538
+ inputs_embeds = None
539
+ else:
540
+ # always pass the input via `inputs_embeds`
541
+ # to make sure the computation graph is consistent
542
+ image_input = self._parse_and_validate_image_input(**kwargs)
543
+
544
+ if image_input is not None:
545
+ vision_embeddings = self._process_image_input(image_input)
546
+ inputs_embeds = self.text_model.get_input_embeddings(input_ids)
547
+
548
+ inputs_embeds = merge_multimodal_embeddings(
549
+ input_ids, inputs_embeds, vision_embeddings,
550
+ self.image_token_id)
551
+ else:
552
+ inputs_embeds = self.text_model.get_input_embeddings(input_ids)
553
+ input_ids = None
554
+
555
+ hidden_states = self.text_model(
556
+ input_ids,
557
+ positions,
558
+ kv_caches,
559
+ attn_metadata,
560
+ intermediate_tensors,
561
+ inputs_embeds=inputs_embeds,
562
+ )
563
+ return hidden_states
564
+
565
+
566
+@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_idefics3)
567
+@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_idefics3_image_tokens)
568
+@INPUT_REGISTRY.register_dummy_data(dummy_data_for_idefics3)
569
+@INPUT_REGISTRY.register_input_processor(input_processor_for_idefics3)
570
+class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal):
571
+
572
+ def __init__(
573
+ self,
574
+ config: Idefics3Config,
575
+ multimodal_config: MultiModalConfig,
576
+ cache_config: Optional[CacheConfig] = None,
577
+ quant_config: Optional[QuantizationConfig] = None,
578
+ ) -> None:
579
+ super().__init__()
580
+
581
+ self.config = config
582
+ self.multimodal_config = multimodal_config
583
+
584
+ self.model = Idefics3Model(config, cache_config, quant_config)
585
+ self.image_token_id = self.config.image_token_id
586
+
587
+ self.lm_head = ParallelLMHead(
588
+ config.text_config.vocab_size,
589
+ config.text_config.hidden_size,
590
+ quant_config=quant_config,
591
+ )
592
+ if self.config.text_config.tie_word_embeddings:
593
+ self.lm_head.weight = self.model.text_model.wte.weight
594
+ self.logits_processor = LogitsProcessor(config.text_config.vocab_size)
595
+ self.sampler = Sampler()
596
+
597
+ def forward(
598
+ self,
599
+ input_ids: torch.Tensor,
600
+ positions: torch.Tensor,
601
+ kv_caches: List[torch.Tensor],
602
+ attn_metadata: AttentionMetadata,
603
+ intermediate_tensors: Optional[IntermediateTensors] = None,
604
+ **kwargs: object,
605
+ ) -> Union[torch.Tensor, IntermediateTensors]:
606
+ hidden_states = self.model(
607
+ input_ids,
608
+ positions,
609
+ kv_caches,
610
+ attn_metadata,
611
+ intermediate_tensors,
612
+ **kwargs,
613
+ )
614
+ return hidden_states
615
+
616
+ def compute_logits(self, hidden_states: torch.Tensor,
617
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
618
+ logits = self.logits_processor(self.lm_head, hidden_states,
619
+ sampling_metadata)
620
+ return logits
621
+
622
+ def sample(
623
+ self,
624
+ logits: torch.Tensor,
625
+ sampling_metadata: SamplingMetadata,
626
+ ) -> Optional[SamplerOutput]:
627
+ next_tokens = self.sampler(logits, sampling_metadata)
628
+ return next_tokens
629
+
630
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
631
+ loader = AutoWeightsLoader(self)
632
+ loader.load_weights(weights)
vllm/model_executor/models/registry.py
+1
@@ -120,6 +120,7 @@ _MULTIMODAL_MODELS = {
120
"FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
121
"H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
122
"InternVLChatModel": ("internvl", "InternVLChatModel"),
123
+ "Idefics3ForConditionalGeneration":("idefics3","Idefics3ForConditionalGeneration"),
124
"LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"),
125
"LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"), # noqa: E501
126
"LlavaNextVideoForConditionalGeneration": ("llava_next_video", "LlavaNextVideoForConditionalGeneration"), # noqa: E501