@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a5bba7d234
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Model] Add Idefics3 support (#9767)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: B-201 <Joy25810@foxmail.com> Co-authored-by: B-201 <Joy25810@foxmail.com>

Jee Jee Li committed Nov 6, 2024 at 19:41 UTC a5bba7d234b4e0d82e6a64de82a8497760ed44cf
8 files changed +723 -1
docs/source/models/supported_models.rst
+6
@@ -446,6 +446,12 @@ Text Generation
446 - :code:`h2oai/h2ovl-mississippi-800m`, :code:`h2oai/h2ovl-mississippi-2b`, etc.
447 -
448 - ✅︎
449 + * - :code:`Idefics3ForConditionalGeneration`
450 + - Idefics3
451 + - T + I
452 + - :code:`HuggingFaceM4/Idefics3-8B-Llama3` etc.
453 + -
454 + -
455 * - :code:`InternVLChatModel`
456 - InternVL2
457 - T + I\ :sup:`E+`
examples/offline_inference_vision_language.py
+17
@@ -377,6 +377,22 @@ def run_glm4v(question: str, modality: str):
377 return llm, prompt, stop_token_ids
378
379
380 +# Idefics3-8B-Llama3
381 +def run_idefics3(question: str, modality: str):
382 + assert modality == "image"
383 + model_name = "HuggingFaceM4/Idefics3-8B-Llama3"
384 +
385 + llm = LLM(model=model_name,
386 + max_model_len=8192,
387 + max_num_seqs=2,
388 + enforce_eager=True)
389 + prompt = (
390 + f"<|begin_of_text|>User:<image>{question}<end_of_utterance>\nAssistant:"
391 + )
392 + stop_token_ids = None
393 + return llm, prompt, stop_token_ids
394 +
395 +
396 model_example_map = {
397 "llava": run_llava,
398 "llava-next": run_llava_next,
@@ -397,6 +413,7 @@ model_example_map = {
413 "mllama": run_mllama,
414 "molmo": run_molmo,
415 "glm4v": run_glm4v,
416 + "idefics3": run_idefics3,
417 }
418
419
examples/offline_inference_vision_language_multi_image.py
+25
@@ -290,6 +290,30 @@ def load_mllama(question, image_urls: List[str]) -> ModelRequestData:
290 )
291
292
293 +def load_idefics3(question, image_urls: List[str]) -> ModelRequestData:
294 + model_name = "HuggingFaceM4/Idefics3-8B-Llama3"
295 +
296 + # The configuration below has been confirmed to launch on a single L40 GPU.
297 + llm = LLM(
298 + model=model_name,
299 + max_model_len=8192,
300 + max_num_seqs=16,
301 + enforce_eager=True,
302 + limit_mm_per_prompt={"image": len(image_urls)},
303 + )
304 +
305 + placeholders = "\n".join(f"Image-{i}: <image>\n"
306 + for i, _ in enumerate(image_urls, start=1))
307 + prompt = f"<|begin_of_text|>User:{placeholders}\n{question}<end_of_utterance>\nAssistant:" # noqa: E501
308 + return ModelRequestData(
309 + llm=llm,
310 + prompt=prompt,
311 + stop_token_ids=None,
312 + image_data=[fetch_image(url) for url in image_urls],
313 + chat_template=None,
314 + )
315 +
316 +
317 model_example_map = {
318 "phi3_v": load_phi3v,
319 "h2ovl_chat": load_h2onvl,
@@ -298,6 +322,7 @@ model_example_map = {
322 "qwen2_vl": load_qwen2_vl,
323 "qwen_vl_chat": load_qwenvl_chat,
324 "mllama": load_mllama,
325 + "idefics3": load_idefics3,
326 }
327
328
tests/models/decoder_only/vision_language/test_models.py
+16
@@ -327,6 +327,22 @@ VLM_TEST_SETTINGS = {
327 vllm_output_post_proc=model_utils.qwen_vllm_to_hf_output,
328 prompt_path_encoder=model_utils.qwen_prompt_path_encoder,
329 ),
330 + "idefics3": VLMTestInfo(
331 + models=["HuggingFaceM4/Idefics3-8B-Llama3"],
332 + test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
333 + prompt_formatter=lambda img_prompt:f"<|begin_of_text|>User:{img_prompt}<end_of_utterance>\nAssistant:", # noqa: E501
334 + img_idx_to_prompt=lambda idx: "<image>",
335 + max_model_len=8192,
336 + max_num_seqs=2,
337 + auto_cls=AutoModelForVision2Seq,
338 + marks=[
339 + pytest.mark.skipif(
340 + transformers.__version__ < "4.46.0",
341 + reason="Model introduced in HF >= 4.46.0"
342 + ),
343 + large_gpu_mark(min_gb=48),
344 + ],
345 + ),
346 ### Tensor parallel / multi-gpu broadcast tests
347 "broadcast-chameleon": VLMTestInfo(
348 models=["facebook/chameleon-7b"],
vllm/entrypoints/chat_utils.py
+2
@@ -187,6 +187,8 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
187 return "<|vision_start|><|image_pad|><|vision_end|>"
188 if model_type == "molmo":
189 return ""
190 + if model_type == "idefics3":
191 + return "<image>"
192
193 raise TypeError(f"Unknown {modality} model type: {model_type}")
194 elif modality == "audio":
vllm/model_executor/models/idefics2_vision_model.py
+24 -1
@@ -15,7 +15,7 @@
15 # limitations under the License.
16 """PyTorch Idefics2 model."""
17
18 -from typing import Optional
18 +from typing import Iterable, Optional, Tuple
19
20 import torch
21 from torch import nn
@@ -29,6 +29,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
29 QKVParallelLinear,
30 RowParallelLinear)
31 from vllm.model_executor.layers.quantization import QuantizationConfig
32 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
33
34
35 class Idefics2VisionEmbeddings(nn.Module):
@@ -329,3 +330,25 @@ class Idefics2VisionTransformer(nn.Module):
330 encoder_outputs = self.encoder(hidden_states)
331 last_hidden_state = self.post_layernorm(encoder_outputs)
332 return last_hidden_state
333 +
334 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
335 + stacked_params_mapping = [
336 + # (param_name, shard_name, shard_id)
337 + ("qkv_proj", "q_proj", "q"),
338 + ("qkv_proj", "k_proj", "k"),
339 + ("qkv_proj", "v_proj", "v"),
340 + ]
341 + params_dict = dict(self.named_parameters())
342 + for name, loaded_weight in weights:
343 + for param_name, weight_name, shard_id in stacked_params_mapping:
344 + if weight_name not in name:
345 + continue
346 + param = params_dict[name.replace(weight_name, param_name)]
347 + weight_loader = param.weight_loader
348 + weight_loader(param, loaded_weight, shard_id)
349 + break
350 + else:
351 + param = params_dict[name]
352 + weight_loader = getattr(param, "weight_loader",
353 + default_weight_loader)
354 + weight_loader(param, loaded_weight)
vllm/model_executor/models/idefics3.py new
+632
@@ -0,0 +1,632 @@
1 +# Copyright 2024 the HuggingFace Inc. team. All rights reserved.
2 +#
3 +# Licensed under the Apache License, Version 2.0 (the "License");
4 +# you may not use this file except in compliance with the License.
5 +# You may obtain a copy of the License at
6 +#
7 +# http://www.apache.org/licenses/LICENSE-2.0
8 +#
9 +# Unless required by applicable law or agreed to in writing, software
10 +# distributed under the License is distributed on an "AS IS" BASIS,
11 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 +# See the License for the specific language governing permissions and
13 +# limitations under the License.
14 +"""Inference-only Idefics3 model compatible with HuggingFace weights."""
15 +
16 +import math
17 +from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
18 + TypedDict, Union)
19 +
20 +import torch
21 +import torch.utils.checkpoint
22 +from PIL import Image
23 +from torch import nn
24 +# Temporary solution for transformers below 4.46.0.
25 +from transformers import PretrainedConfig as Idefics3Config
26 +
27 +from vllm.attention import AttentionMetadata
28 +from vllm.config import CacheConfig, MultiModalConfig
29 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
30 + InputContext, token_inputs)
31 +from vllm.logger import init_logger
32 +from vllm.model_executor.layers.linear import ReplicatedLinear
33 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
34 +from vllm.model_executor.layers.quantization import QuantizationConfig
35 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
36 +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
37 +from vllm.model_executor.sampling_metadata import SamplingMetadata
38 +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalInputs
39 +from vllm.multimodal.image import cached_get_image_processor
40 +from vllm.sequence import IntermediateTensors, SequenceData
41 +from vllm.transformers_utils.processor import cached_get_processor
42 +from vllm.utils import is_list_of
43 +
44 +# yapf: disable
45 +from .idefics2_vision_model import (
46 + Idefics2VisionTransformer as Idefics3VisionTransformer)
47 +# yapf: enable
48 +from .interfaces import SupportsMultiModal
49 +from .llama import LlamaModel
50 +from .utils import AutoWeightsLoader, flatten_bn, merge_multimodal_embeddings
51 +
52 +logger = init_logger(__name__)
53 +
54 +
55 +class Idefics3ImagePixelInputs(TypedDict):
56 + type: Literal["pixel_values"]
57 + data: torch.Tensor
58 + """
59 + Shape: `(batch_size * num_images, num_channels, height, width)`
60 + """
61 + rows: List[int]
62 + cols: List[int]
63 + pixel_attention_mask: Optional[torch.BoolTensor]
64 +
65 +
66 +class Idefics3ImageEmbeddingInputs(TypedDict):
67 + type: Literal["image_embeds"]
68 + data: torch.Tensor
69 + """
70 + Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
71 + `hidden_size` must match the hidden size of language model backbone.
72 + """
73 +
74 +
75 +ImageInputs = Union[Idefics3ImagePixelInputs, Idefics3ImageEmbeddingInputs]
76 +
77 +
78 +def input_mapper_for_idefics3(
79 + ctx: InputContext,
80 + data: object,
81 +):
82 + model_config = ctx.model_config
83 + image_processor = cached_get_image_processor(
84 + model_config.model, trust_remote_code=model_config.trust_remote_code)
85 + if image_processor is None:
86 + raise RuntimeError("No HuggingFace processor is available "
87 + "to process the image object")
88 +
89 + if isinstance(data, Image.Image):
90 + images = [[data]]
91 + elif is_list_of(data, Image.Image):
92 + images = [data]
93 + else:
94 + raise TypeError(f"Invalid image type: {type(data)}")
95 +
96 + try:
97 + batch_data = image_processor(images,
98 + return_tensors="pt",
99 + return_row_col_info=True).data
100 + except Exception:
101 + logger.error("Failed to process image (%s)", data)
102 + raise
103 +
104 + return MultiModalInputs(batch_data)
105 +
106 +
107 +def _resize_output_size(height: int,
108 + width: int,
109 + max_len: Optional[int] = None,
110 + min_len: Optional[int] = 1,
111 + max_size: Optional[int] = None) -> Tuple[int, int]:
112 + # Set default value for max_len if not provided
113 + max_len = max(height, width) if max_len is None else max_len
114 + aspect_ratio = width / height
115 +
116 + # Handle the maximum size constraint
117 + if max_size is not None:
118 + max_len = min(max_len, max_size)
119 +
120 + # Adjust dimensions according to the aspect ratio
121 + if width >= height:
122 + width = max_len
123 + height = int(width / aspect_ratio)
124 + else:
125 + height = max_len
126 + width = int(height * aspect_ratio)
127 +
128 + # Ensure both width and height are even (if needed)
129 + height += 1 if height % 2 != 0 else 0
130 + width += 1 if width % 2 != 0 else 0
131 +
132 + # Ensure dimensions are not smaller than the minimum length
133 + height = max(height, min_len)
134 + width = max(width, min_len)
135 +
136 + return height, width
137 +
138 +
139 +def _get_resize_output_image_size(
140 + image_size: Tuple[int, int],
141 + resolution_max_side: int,
142 + max_image_size: int = 1820,
143 +) -> Tuple[int, int]:
144 + if resolution_max_side > max_image_size:
145 + raise ValueError(
146 + "`resolution_max_side` cannot be larger than `max_image_size`")
147 +
148 + height, width = image_size
149 +
150 + # Find the output size, when rescaling the longest edge to max_len and
151 + # preserving the aspect ratio
152 + height, width = _resize_output_size(height,
153 + width,
154 + max_len=resolution_max_side)
155 +
156 + return height, width
157 +
158 +
159 +def _prompt_split_image(image_seq_len: int, image_rows: int, image_cols: int,
160 + fake_token_around_image: str, image_token: str,
161 + global_img_token: str) -> str:
162 + """
163 + Prompt with expanded image tokens for when the image is split
164 + into patches.
165 + """
166 + text_split_images = ""
167 + for n_h in range(image_rows):
168 + for n_w in range(image_cols):
169 + text_split_images += (fake_token_around_image +
170 + f"<row_{n_h + 1}_col_{n_w + 1}>" +
171 + image_token * image_seq_len)
172 + text_split_images += "\n"
173 +
174 + text_split_images += "\n" + _prompt_single_image(
175 + image_seq_len=image_seq_len,
176 + fake_token_around_image=fake_token_around_image,
177 + image_token=image_token,
178 + global_img_token=global_img_token)
179 + return text_split_images
180 +
181 +
182 +def _prompt_single_image(image_seq_len: int, fake_token_around_image: str,
183 + image_token: str, global_img_token: str):
184 + """Prompt with expanded image tokens for a single image."""
185 + return (fake_token_around_image + global_img_token +
186 + image_token * image_seq_len + fake_token_around_image)
187 +
188 +
189 +def _get_image_prompt_string(image_rows: int, image_cols: int,
190 + image_seq_len: int, fake_token_around_image: str,
191 + image_token: str, global_img_token: str):
192 + if image_rows == 0 and image_cols == 0:
193 + return _prompt_single_image(
194 + image_seq_len=image_seq_len,
195 + fake_token_around_image=fake_token_around_image,
196 + image_token=image_token,
197 + global_img_token=global_img_token,
198 + )
199 + return _prompt_split_image(image_seq_len, image_rows, image_cols,
200 + fake_token_around_image, image_token,
201 + global_img_token)
202 +
203 +
204 +def input_processor_for_idefics3(ctx: InputContext, inputs: DecoderOnlyInputs):
205 + multi_modal_data = inputs.get("multi_modal_data")
206 + if multi_modal_data is None or "image" not in multi_modal_data:
207 + return inputs
208 +
209 + model_config = ctx.model_config
210 + processor = cached_get_processor(model_config.model)
211 + image_processor = processor.image_processor
212 + tokenizer = processor.tokenizer
213 + size = image_processor.size['longest_edge']
214 + max_image_size = image_processor.max_image_size['longest_edge']
215 +
216 + image_data = multi_modal_data["image"]
217 + if isinstance(image_data, Image.Image):
218 + image_list = [image_data]
219 + elif is_list_of(image_data, Image.Image):
220 + image_list = image_data
221 + else:
222 + raise TypeError(f"Invalid image type: {type(image_data)}")
223 +
224 + image_rows = []
225 + image_cols = []
226 + for image in image_list:
227 + height, width = _get_resize_output_image_size(image.size, size)
228 +
229 + rows = math.ceil(height / max_image_size)
230 + cols = math.ceil(width / max_image_size)
231 + image_rows.append(rows)
232 + image_cols.append(cols)
233 + image_rows = [image_rows]
234 + image_cols = [image_cols]
235 +
236 + n_images_in_text = []
237 +
238 + text = inputs.get("prompt")
239 + if text is not None:
240 + if isinstance(text, str):
241 + text = [text]
242 + elif not isinstance(text, list) and not isinstance(text[0], str):
243 + raise ValueError("Invalid input text. Please provide a string, "
244 + "or a list of strings")
245 +
246 + fake_image_token = processor.fake_image_token.content
247 + image_token = processor.image_token.content
248 + global_img_token = processor.global_image_tag
249 +
250 + prompt_strings = []
251 + for sample, sample_rows, sample_cols in zip(text, image_rows,
252 + image_cols):
253 + n_images_in_text.append(sample.count(image_token))
254 +
255 + # Replace the image token with fake tokens around the expanded
256 + # image token sequence of length `image_seq_len`
257 + image_prompt_strings = []
258 + for n_rows, n_cols in zip(sample_rows, sample_cols):
259 + image_prompt_string = _get_image_prompt_string(
260 + n_rows,
261 + n_cols,
262 + processor.image_seq_len,
263 + image_token=image_token,
264 + fake_token_around_image=fake_image_token,
265 + global_img_token=global_img_token,
266 + )
267 + image_prompt_strings.append(image_prompt_string)
268 +
269 + split_sample = sample.split(image_token)
270 + if len(split_sample) == 0:
271 + raise ValueError(
272 + "The image token should be present in the text.")
273 +
274 + # Place in the image prompt strings where the image tokens are
275 + sample = split_sample[0]
276 + for i, image_prompt_string in enumerate(image_prompt_strings):
277 + sample += image_prompt_string + split_sample[i + 1]
278 + prompt_strings.append(sample)
279 +
280 + prompt_token_ids = tokenizer(text=prompt_strings[0]).input_ids
281 +
282 + return token_inputs(
283 + prompt_token_ids=prompt_token_ids,
284 + prompt=prompt_strings[0],
285 + multi_modal_data=multi_modal_data,
286 + )
287 +
288 +
289 +def get_max_idefics3_image_tokens(ctx: InputContext,
290 + *,
291 + num_crops: Optional[int] = None):
292 + model_config = ctx.model_config
293 + processor = cached_get_processor(model_config.model)
294 + image_seq_len = processor.image_seq_len
295 + image_processor = processor.image_processor
296 +
297 + size = image_processor.size['longest_edge']
298 + max_image_size = image_processor.max_image_size['longest_edge']
299 + resized_height, resized_width = size, size
300 +
301 + grid_h = resized_height // max_image_size
302 + grid_w = resized_width // max_image_size
303 +
304 + return (grid_h * grid_w + 1) * image_seq_len
305 +
306 +
307 +def dummy_data_for_idefics3(ctx: InputContext, seq_len: int,
308 + mm_counts: Mapping[str, int]) -> DummyData:
309 + hf_config = ctx.get_hf_config()
310 + num_images = mm_counts["image"]
311 +
312 + processor = cached_get_processor(ctx.model_config.model)
313 + image_seq_len = processor.image_seq_len
314 + max_llm_image_tokens = 17 * image_seq_len * num_images
315 +
316 + seq_data = SequenceData.from_prompt_token_counts(
317 + (hf_config.image_token_id, max_llm_image_tokens), (0, seq_len))
318 +
319 + width = height = hf_config.vision_config.image_size
320 + image = Image.new("RGB", (width, height), color=0)
321 + mm_data = {"image": [image] if num_images == 1 else [image] * num_images}
322 +
323 + return DummyData(seq_data, mm_data)
324 +
325 +
326 +class Idefics3SimpleMLP(nn.Module):
327 +
328 + def __init__(self, config):
329 + super().__init__()
330 + input_size = config.vision_config.hidden_size * (config.scale_factor**
331 + 2)
332 + output_size = config.text_config.hidden_size
333 + self.proj = ReplicatedLinear(input_size, output_size, bias=False)
334 +
335 + def forward(self, x: torch.Tensor) -> torch.Tensor:
336 + out, _ = self.proj(x)
337 + return out
338 +
339 +
340 +class Idefics3Connector(nn.Module):
341 +
342 + def __init__(self, config):
343 + super().__init__()
344 + self.scale_factor = config.scale_factor
345 + self.modality_projection = Idefics3SimpleMLP(config)
346 +
347 + def pixel_shuffle(self,
348 + x: torch.Tensor,
349 + scale_factor: int = 2) -> torch.Tensor:
350 + bsz, seq, embed_dim = x.size()
351 + height = width = int(seq**0.5)
352 + x = x.view(bsz, height, width, embed_dim)
353 + x = x.view(bsz, height, int(width / scale_factor),
354 + embed_dim * scale_factor)
355 + x = x.permute(0, 2, 1, 3)
356 + x = x.reshape(
357 + bsz,
358 + int(width / scale_factor),
359 + int(height / scale_factor),
360 + embed_dim * (scale_factor**2),
361 + )
362 + x = x.permute(0, 2, 1, 3)
363 + x = x.reshape(bsz, int(seq / (scale_factor**2)),
364 + embed_dim * (scale_factor**2))
365 + return x
366 +
367 + def forward(self, image_hidden_states: torch.Tensor) -> torch.Tensor:
368 + image_hidden_states = self.pixel_shuffle(image_hidden_states,
369 + self.scale_factor)
370 + image_hidden_states = self.modality_projection(image_hidden_states)
371 + return image_hidden_states
372 +
373 +
374 +class Idefics3Model(nn.Module):
375 +
376 + def __init__(
377 + self,
378 + config: Idefics3Config,
379 + cache_config: Optional[CacheConfig] = None,
380 + quant_config: Optional[QuantizationConfig] = None,
381 + ):
382 + super().__init__()
383 + self.config = config
384 + self.padding_idx = self.config.text_config.pad_token_id
385 + self.vocab_size = self.config.text_config.vocab_size
386 +
387 + self.vision_model = Idefics3VisionTransformer(config.vision_config,
388 + quant_config)
389 + self.connector = Idefics3Connector(config)
390 + self.text_model = LlamaModel(config.text_config, cache_config,
391 + quant_config)
392 +
393 + self.image_seq_len = int(
394 + ((config.vision_config.image_size //
395 + config.vision_config.patch_size)**2) / (config.scale_factor**2))
396 + self.image_token_id = self.config.image_token_id
397 +
398 + def _validate_pixel_values(
399 + self, data: Union[torch.Tensor, List[torch.Tensor]]
400 + ) -> Union[torch.Tensor, List[torch.Tensor]]:
401 +
402 + h = w = self.config.vision_config.image_size
403 + expected_dims = (3, h, w)
404 +
405 + def _validate_shape(d: torch.Tensor):
406 + actual_dims = tuple(d.shape[1:])
407 +
408 + if actual_dims != expected_dims:
409 + expected_expr = ("num_patches", *map(str, expected_dims))
410 + raise ValueError(
411 + "The expected shape of pixel values per image per batch "
412 + f"is {expected_expr}. You supplied {tuple(d.shape)}.")
413 +
414 + for d in data:
415 + _validate_shape(d)
416 +
417 + return data
418 +
419 + def _parse_and_validate_image_input(
420 + self, **kwargs: object) -> Optional[ImageInputs]:
421 + pixel_values = kwargs.pop("pixel_values", None)
422 + image_embeds = kwargs.pop("image_embeds", None)
423 + rows = kwargs.pop("rows", None)
424 + cols = kwargs.pop("cols", None)
425 + pixel_attention_mask = kwargs.pop("pixel_attention_mask", None)
426 +
427 + if pixel_values is None and image_embeds is None:
428 + return None
429 +
430 + if image_embeds is not None:
431 + if not isinstance(image_embeds, (torch.Tensor, list)):
432 + raise ValueError("Incorrect type of image embeddings. "
433 + f"Got type: {type(image_embeds)}")
434 +
435 + return Idefics3ImageEmbeddingInputs(
436 + type="image_embeds",
437 + data=flatten_bn(image_embeds, concat=True),
438 + )
439 +
440 + if pixel_values is not None:
441 + if not isinstance(pixel_values, (torch.Tensor, list)):
442 + raise ValueError("Incorrect type of pixel values. "
443 + f"Got type: {type(pixel_values)}")
444 +
445 + return Idefics3ImagePixelInputs(type="pixel_values",
446 + data=self._validate_pixel_values(
447 + flatten_bn(pixel_values,
448 + concat=True)),
449 + rows=rows,
450 + cols=cols,
451 + pixel_attention_mask=flatten_bn(
452 + pixel_attention_mask,
453 + concat=True))
454 +
455 + raise AssertionError("This line should be unreachable.")
456 +
457 + def _image_pixels_to_features(
458 + self,
459 + pixel_values: torch.Tensor,
460 + pixel_attention_mask: Optional[torch.BoolTensor] = None,
461 + ) -> torch.Tensor:
462 + # NOTE: we skip the step to select the vision feature layer since
463 + # this is already done inside the vision tower
464 + batch_size, num_images, num_channels, height, width = pixel_values.shape
465 + pixel_values = pixel_values.to(
466 + dtype=self.vision_model.embeddings.patch_embedding.weight.dtype
467 + ) # fp16 compatibility
468 + pixel_values = pixel_values.view(batch_size * num_images,
469 + *pixel_values.shape[2:])
470 +
471 + # Remove padding images - padding images are full 0.
472 + nb_values_per_image = pixel_values.shape[1:].numel()
473 + real_images_inds = (pixel_values == 0.0).sum(
474 + dim=(-1, -2, -3)) != nb_values_per_image
475 + pixel_values = pixel_values[real_images_inds].contiguous()
476 +
477 + # Handle the vision attention mask
478 + if pixel_attention_mask is None:
479 + pixel_attention_mask = torch.ones(
480 + size=(pixel_values.size(0), pixel_values.size(2),
481 + pixel_values.size(3)),
482 + dtype=torch.bool,
483 + device=pixel_values.device,
484 + )
485 + else:
486 + # Remove padding images from the mask
487 + pixel_attention_mask = pixel_attention_mask.view(
488 + batch_size * num_images, *pixel_attention_mask.shape[2:])
489 + pixel_attention_mask = pixel_attention_mask[
490 + real_images_inds].contiguous()
491 +
492 + patch_size = self.config.vision_config.patch_size
493 + patches_subgrid = pixel_attention_mask.unfold(dimension=1,
494 + size=patch_size,
495 + step=patch_size)
496 + patches_subgrid = patches_subgrid.unfold(dimension=2,
497 + size=patch_size,
498 + step=patch_size)
499 + patch_attention_mask = (patches_subgrid.sum(dim=(-1, -2)) > 0).bool()
500 +
501 + # Get sequence from the vision encoder
502 + image_hidden_states = self.vision_model(
503 + pixel_values=pixel_values,
504 + patch_attention_mask=patch_attention_mask,
505 + )
506 +
507 + return image_hidden_states
508 +
509 + def _process_image_pixels(
510 + self, inputs: Idefics3ImagePixelInputs) -> torch.Tensor:
511 + assert self.vision_model is not None
512 +
513 + pixel_values = inputs["data"]
514 + pixel_attention_mask = inputs["pixel_attention_mask"]
515 +
516 + return self._image_pixels_to_features(pixel_values,
517 + pixel_attention_mask)
518 +
519 + def _process_image_input(self, image_input: ImageInputs) -> torch.Tensor:
520 + if image_input["type"] == "image_embeds":
521 + return image_input["data"]
522 +
523 + assert self.vision_model is not None
524 + image_features = self._process_image_pixels(image_input)
525 + return self.connector(image_features)
526 +
527 + def forward(
528 + self,
529 + input_ids: torch.Tensor,
530 + positions: torch.Tensor,
531 + kv_caches: List[torch.Tensor],
532 + attn_metadata: AttentionMetadata,
533 + intermediate_tensors: Optional[IntermediateTensors] = None,
534 + **kwargs: object,
535 + ) -> Union[torch.Tensor, IntermediateTensors]:
536 + if intermediate_tensors is not None:
537 + input_ids = None
538 + inputs_embeds = None
539 + else:
540 + # always pass the input via `inputs_embeds`
541 + # to make sure the computation graph is consistent
542 + image_input = self._parse_and_validate_image_input(**kwargs)
543 +
544 + if image_input is not None:
545 + vision_embeddings = self._process_image_input(image_input)
546 + inputs_embeds = self.text_model.get_input_embeddings(input_ids)
547 +
548 + inputs_embeds = merge_multimodal_embeddings(
549 + input_ids, inputs_embeds, vision_embeddings,
550 + self.image_token_id)
551 + else:
552 + inputs_embeds = self.text_model.get_input_embeddings(input_ids)
553 + input_ids = None
554 +
555 + hidden_states = self.text_model(
556 + input_ids,
557 + positions,
558 + kv_caches,
559 + attn_metadata,
560 + intermediate_tensors,
561 + inputs_embeds=inputs_embeds,
562 + )
563 + return hidden_states
564 +
565 +
566 +@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_idefics3)
567 +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_idefics3_image_tokens)
568 +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_idefics3)
569 +@INPUT_REGISTRY.register_input_processor(input_processor_for_idefics3)
570 +class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal):
571 +
572 + def __init__(
573 + self,
574 + config: Idefics3Config,
575 + multimodal_config: MultiModalConfig,
576 + cache_config: Optional[CacheConfig] = None,
577 + quant_config: Optional[QuantizationConfig] = None,
578 + ) -> None:
579 + super().__init__()
580 +
581 + self.config = config
582 + self.multimodal_config = multimodal_config
583 +
584 + self.model = Idefics3Model(config, cache_config, quant_config)
585 + self.image_token_id = self.config.image_token_id
586 +
587 + self.lm_head = ParallelLMHead(
588 + config.text_config.vocab_size,
589 + config.text_config.hidden_size,
590 + quant_config=quant_config,
591 + )
592 + if self.config.text_config.tie_word_embeddings:
593 + self.lm_head.weight = self.model.text_model.wte.weight
594 + self.logits_processor = LogitsProcessor(config.text_config.vocab_size)
595 + self.sampler = Sampler()
596 +
597 + def forward(
598 + self,
599 + input_ids: torch.Tensor,
600 + positions: torch.Tensor,
601 + kv_caches: List[torch.Tensor],
602 + attn_metadata: AttentionMetadata,
603 + intermediate_tensors: Optional[IntermediateTensors] = None,
604 + **kwargs: object,
605 + ) -> Union[torch.Tensor, IntermediateTensors]:
606 + hidden_states = self.model(
607 + input_ids,
608 + positions,
609 + kv_caches,
610 + attn_metadata,
611 + intermediate_tensors,
612 + **kwargs,
613 + )
614 + return hidden_states
615 +
616 + def compute_logits(self, hidden_states: torch.Tensor,
617 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
618 + logits = self.logits_processor(self.lm_head, hidden_states,
619 + sampling_metadata)
620 + return logits
621 +
622 + def sample(
623 + self,
624 + logits: torch.Tensor,
625 + sampling_metadata: SamplingMetadata,
626 + ) -> Optional[SamplerOutput]:
627 + next_tokens = self.sampler(logits, sampling_metadata)
628 + return next_tokens
629 +
630 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
631 + loader = AutoWeightsLoader(self)
632 + loader.load_weights(weights)
vllm/model_executor/models/registry.py
+1
@@ -120,6 +120,7 @@ _MULTIMODAL_MODELS = {
120 "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
121 "H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
122 "InternVLChatModel": ("internvl", "InternVLChatModel"),
123 + "Idefics3ForConditionalGeneration":("idefics3","Idefics3ForConditionalGeneration"),
124 "LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"),
125 "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"), # noqa: E501
126 "LlavaNextVideoForConditionalGeneration": ("llava_next_video", "LlavaNextVideoForConditionalGeneration"), # noqa: E501