A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
[Bugfix] `embed_is_patch` for Idefics3 (#15696)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Mar 28, 2025 at 23:27 UTC
541d1df486ac863ab057d842791695763de6f58b
5 files changed
+320
-188
vllm/model_executor/models/commandr.py
-1
@@ -24,7 +24,6 @@
24
from typing import Iterable, Optional, Set, Tuple, Union
25
26
import torch
27
-import torch.utils.checkpoint
27
from torch import nn
28
from transformers import CohereConfig
29
vllm/model_executor/models/idefics3.py
+318
-183
@@ -17,16 +17,14 @@
17
18
import math
19
from collections.abc import Iterable, Mapping, Sequence
20
-from typing import Dict, List, Literal, Optional, Set, Tuple, TypedDict, Union
20
+from typing import Dict, Literal, Optional, Set, Tuple, TypedDict, Union
21
22
import torch
23
-import torch.utils.checkpoint
23
from torch import nn
24
from transformers import (BatchFeature, Idefics3Config, Idefics3ImageProcessor,
25
Idefics3Processor)
26
27
from vllm.config import VllmConfig
29
-from vllm.logger import init_logger
28
from vllm.model_executor.layers.linear import ReplicatedLinear
29
from vllm.model_executor.layers.logits_processor import LogitsProcessor
30
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -35,13 +33,16 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
33
from vllm.model_executor.models.module_mapping import MultiModelKeys
34
from vllm.model_executor.sampling_metadata import SamplingMetadata
35
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
38
-from vllm.multimodal.inputs import NestedTensors
39
-from vllm.multimodal.parse import ImageProcessorItems
36
+from vllm.multimodal.parse import ImageProcessorItems, ImageSize
37
+# yapf conflicts with isort for this block
38
+# yapf: disable
39
from vllm.multimodal.processing import (BaseMultiModalProcessor,
40
BaseProcessingInfo,
41
MultiModalDataItems,
42
MultiModalFieldConfig,
44
- PromptReplacement, PromptUpdate)
43
+ PromptReplacement, PromptUpdate,
44
+ encode_tokens)
45
+# yapf: enable
46
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
47
from vllm.sequence import IntermediateTensors
48
@@ -53,18 +54,28 @@ from .interfaces import MultiModalEmbeddings, SupportsLoRA, SupportsMultiModal
54
from .llama import LlamaModel
55
from .utils import (AutoWeightsLoader, flatten_bn, maybe_prefix,
56
merge_multimodal_embeddings)
56
-
57
-logger = init_logger(__name__)
57
+from .vision import scatter_patch_features, select_patch_features
58
59
60
class Idefics3ImagePixelInputs(TypedDict):
61
type: Literal["pixel_values"]
62
- data: torch.Tensor
62
+ pixel_values: torch.Tensor
63
"""
64
Shape: `(batch_size * num_images * num_patches,
65
num_channels, height, width)`
66
"""
67
- pixel_attention_mask: Optional[torch.BoolTensor]
67
+ pixel_attention_mask: torch.Tensor
68
+
69
+ num_patches: torch.Tensor
70
+ """Shape: `(batch_size * num_images)`"""
71
+
72
+ embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
73
+ """
74
+ A boolean mask indicating which image embeddings correspond
75
+ to patch tokens.
76
+
77
+ Shape: `(batch_size * num_images, num_embeds)`
78
+ """
79
80
81
class Idefics3ImageEmbeddingInputs(TypedDict):
@@ -75,6 +86,14 @@ class Idefics3ImageEmbeddingInputs(TypedDict):
86
`hidden_size` must match the hidden size of language model backbone.
87
"""
88
89
+ embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
90
+ """
91
+ A boolean mask indicating which image embeddings correspond
92
+ to patch tokens.
93
+
94
+ Shape: `(batch_size * num_images, num_embeds)`
95
+ """
96
+
97
98
ImageInputs = Union[Idefics3ImagePixelInputs, Idefics3ImageEmbeddingInputs]
99
@@ -100,32 +119,14 @@ class Idefics3ProcessingInfo(BaseProcessingInfo):
119
seq_len: int,
120
mm_counts: Mapping[str, int],
121
) -> Mapping[str, int]:
103
- hf_processor = self.get_hf_processor()
104
- image_processor: Idefics3ImageProcessor = hf_processor.image_processor
105
- grid_w, grid_h = self._get_image_feature_grid_size(
106
- image_width=image_processor.size['longest_edge'],
107
- image_height=image_processor.size['longest_edge'],
108
- )
109
- num_image_token = (grid_w * grid_h + 1) * hf_processor.image_seq_len
110
- # Calculate Non-image-token length
111
- # NOTE: <row_1_col_1> and <global-img> are special token for SmolVLM
112
- # but not for Idefic3, so we need to tokenize them to get actual length.
113
- tokenizer = self.get_tokenizer()
114
- tile_token_len = len(tokenizer.tokenize("<row_1_col_1>"))
115
- glob_token_len = len(tokenizer.tokenize(hf_processor.global_image_tag))
116
- # linebreak and <fake_token_around_image> always cost 1 token
117
- fake_token_len = lb_len = 1
118
- non_image_token = (grid_w * grid_h) * (
119
- tile_token_len + fake_token_len) + glob_token_len + (
120
- grid_h + 1) * lb_len + fake_token_len
121
- return {"image": num_image_token + non_image_token}
122
+ return {"image": self.get_max_image_tokens()}
123
124
def _resize_output_size(self,
125
*,
126
height: int,
127
width: int,
128
max_len: Optional[int] = None,
128
- min_len: Optional[int] = 1,
129
+ min_len: int = 1,
130
max_size: Optional[int] = None) -> tuple[int, int]:
131
# Set default value for max_len if not provided
132
max_len = max(height, width) if max_len is None else max_len
@@ -181,10 +182,13 @@ class Idefics3ProcessingInfo(BaseProcessingInfo):
182
*,
183
image_width: int,
184
image_height: int,
184
- size: Optional[dict[str, object]] = None,
185
+ processor: Optional[Idefics3Processor],
186
) -> tuple[int, int]:
186
- hf_processor = self.get_hf_processor(size=size)
187
- image_processor: Idefics3ImageProcessor = hf_processor.image_processor
187
+ if processor is None:
188
+ processor = self.get_hf_processor()
189
+
190
+ image_processor: Idefics3ImageProcessor = processor.image_processor
191
+
192
max_image_size = image_processor.max_image_size['longest_edge']
193
size = image_processor.size['longest_edge']
194
assert size % max_image_size == 0, (
@@ -204,6 +208,105 @@ class Idefics3ProcessingInfo(BaseProcessingInfo):
208
grid_h = grid_w = 0
209
return grid_w, grid_h
210
211
+ def get_num_patches(
212
+ self,
213
+ *,
214
+ image_width: int,
215
+ image_height: int,
216
+ processor: Optional[Idefics3Processor],
217
+ ) -> int:
218
+ grid_w, grid_h = self._get_image_feature_grid_size(
219
+ image_width=image_width,
220
+ image_height=image_height,
221
+ processor=processor,
222
+ )
223
+
224
+ return grid_w * grid_h + 1
225
+
226
+ def get_image_repl(
227
+ self,
228
+ *,
229
+ image_width: int,
230
+ image_height: int,
231
+ processor: Optional[Idefics3Processor],
232
+ ) -> str:
233
+ if processor is None:
234
+ processor = self.get_hf_processor()
235
+
236
+ image_token = processor.image_token.content
237
+ fake_image_token = processor.fake_image_token.content
238
+ global_img_token = processor.global_image_tag
239
+ image_seq_len = processor.image_seq_len
240
+ grid_placeholder = "<row_{n_h}_col_{n_w}>"
241
+
242
+ p_img = image_token * image_seq_len
243
+ global_img_placeholder = fake_image_token + global_img_token + p_img
244
+ tile_img_placeholder = fake_image_token + grid_placeholder + p_img
245
+
246
+ grid_w, grid_h = self._get_image_feature_grid_size(
247
+ image_width=image_width,
248
+ image_height=image_height,
249
+ processor=processor,
250
+ )
251
+ if grid_w == 0 and grid_h == 0:
252
+ return global_img_placeholder + fake_image_token
253
+
254
+ tiles_placeholder = list[str]()
255
+ for i in range(grid_h):
256
+ for j in range(grid_w):
257
+ placeholder_per_tile = tile_img_placeholder.format(n_h=i + 1,
258
+ n_w=j + 1)
259
+ tiles_placeholder.append(placeholder_per_tile)
260
+ # Add line break if it is the last tile in the row
261
+ if j == grid_w - 1:
262
+ tiles_placeholder.append("\n")
263
+
264
+ return "".join([
265
+ *tiles_placeholder,
266
+ "\n",
267
+ global_img_placeholder,
268
+ fake_image_token,
269
+ ])
270
+
271
+ def get_num_image_tokens(
272
+ self,
273
+ *,
274
+ image_width: int,
275
+ image_height: int,
276
+ processor: Optional[Idefics3Processor],
277
+ ) -> int:
278
+ tokenizer = self.get_tokenizer()
279
+ image_repl = self.get_image_repl(
280
+ image_width=image_width,
281
+ image_height=image_height,
282
+ processor=processor,
283
+ )
284
+
285
+ image_repl_tokens = encode_tokens(
286
+ tokenizer,
287
+ image_repl,
288
+ add_special_tokens=False,
289
+ )
290
+ return len(image_repl_tokens)
291
+
292
+ def get_image_size_with_most_features(self) -> ImageSize:
293
+ processor = self.get_hf_processor()
294
+ image_processor: Idefics3ImageProcessor = processor.image_processor
295
+
296
+ return ImageSize(
297
+ width=image_processor.size["longest_edge"],
298
+ height=image_processor.size["longest_edge"],
299
+ )
300
+
301
+ def get_max_image_tokens(self) -> int:
302
+ target_width, target_height = self.get_image_size_with_most_features()
303
+
304
+ return self.get_num_image_tokens(
305
+ image_width=target_width,
306
+ image_height=target_height,
307
+ processor=None,
308
+ )
309
+
310
311
class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo]
312
):
@@ -217,7 +320,7 @@ class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo]
320
hf_processor = self.info.get_hf_processor()
321
image_processor: Idefics3ImageProcessor = hf_processor.image_processor
322
longest_edge = image_processor.max_image_size['longest_edge']
220
- image_token: str = hf_processor.image_token.content
323
+ image_token = hf_processor.image_token.content
324
325
mm_data = {
326
"image":
@@ -241,26 +344,61 @@ class Idefics3MultiModalProcessor(
344
mm_data: Mapping[str, object],
345
mm_kwargs: Mapping[str, object],
346
) -> BatchFeature:
244
- if mm_data:
245
- processed_outputs = super()._call_hf_processor(
246
- prompt, mm_data, mm_kwargs)
247
- image_grids = [
248
- self.info._get_image_feature_grid_size(
249
- image_width=img.width,
250
- image_height=img.height,
251
- **mm_kwargs,
252
- ) for img in mm_data["images"]
253
- ]
254
- image_patches = list(map(lambda x: math.prod(x) + 1, image_grids))
255
- for key in ("pixel_values", "pixel_attention_mask"):
256
- data = processed_outputs.pop(key)
257
- data = data.flatten(0, 1).split(image_patches)
258
- processed_outputs[key] = data
259
- else:
260
- tokenizer = self.info.get_tokenizer()
261
- processed_outputs = tokenizer(prompt,
262
- add_special_tokens=True,
263
- return_tensors="pt")
347
+ # Text-only input not supported in composite processor
348
+ if not (images := mm_data.get("images", [])):
349
+ prompt_ids = self.info.get_tokenizer().encode(prompt)
350
+ prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
351
+ return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
352
+
353
+ processed_outputs = super()._call_hf_processor(
354
+ prompt,
355
+ mm_data,
356
+ mm_kwargs,
357
+ )
358
+
359
+ parsed_images = (self._get_data_parser().parse_mm_data({
360
+ "image": images
361
+ }).get_items("image", ImageProcessorItems))
362
+ image_sizes = [
363
+ parsed_images.get_image_size(i) for i in range(len(parsed_images))
364
+ ]
365
+ hf_processor = self.info.get_hf_processor(**mm_kwargs)
366
+
367
+ image_repl_features = [
368
+ self.info.get_image_repl(image_width=size.width,
369
+ image_height=size.height,
370
+ processor=hf_processor)
371
+ for size in image_sizes
372
+ ]
373
+
374
+ tokenizer = self.info.get_tokenizer()
375
+ image_repls_feature_tokens = [
376
+ tokenizer.encode(image_repl, add_special_tokens=False)
377
+ for image_repl in image_repl_features
378
+ ]
379
+
380
+ vocab = tokenizer.get_vocab()
381
+ image_token_id = vocab[hf_processor.image_token.content]
382
+
383
+ embed_is_patch = [
384
+ torch.tensor(image_repl_tokens) == image_token_id
385
+ for image_repl_tokens in image_repls_feature_tokens
386
+ ]
387
+ processed_outputs["embed_is_patch"] = embed_is_patch
388
+
389
+ num_patches = [
390
+ self.info.get_num_patches(
391
+ image_width=size.width,
392
+ image_height=size.height,
393
+ processor=hf_processor,
394
+ ) for size in image_sizes
395
+ ]
396
+ processed_outputs["num_patches"] = torch.tensor(num_patches)
397
+
398
+ # Remove the extra batch dimension
399
+ processed_outputs["pixel_values"].squeeze_(0)
400
+ processed_outputs["pixel_attention_mask"].squeeze_(0)
401
+
402
return processed_outputs
403
404
def _get_mm_fields_config(
@@ -268,10 +406,16 @@ class Idefics3MultiModalProcessor(
406
hf_inputs: BatchFeature,
407
hf_processor_mm_kwargs: Mapping[str, object],
408
) -> Mapping[str, MultiModalFieldConfig]:
409
+ num_patches = hf_inputs.get("num_patches", torch.empty(0))
410
+
411
return dict(
272
- pixel_values=MultiModalFieldConfig.batched("image"),
273
- pixel_attention_mask=MultiModalFieldConfig.batched("image"),
412
+ pixel_values=MultiModalFieldConfig.flat_from_sizes(
413
+ "image", num_patches),
414
+ pixel_attention_mask=MultiModalFieldConfig.flat_from_sizes(
415
+ "image", num_patches),
416
image_embeds=MultiModalFieldConfig.batched("image"),
417
+ num_patches=MultiModalFieldConfig.batched("image"),
418
+ embed_is_patch=MultiModalFieldConfig.batched("image"),
419
)
420
421
def _get_prompt_updates(
@@ -281,42 +425,18 @@ class Idefics3MultiModalProcessor(
425
out_mm_kwargs: MultiModalKwargs,
426
) -> Sequence[PromptUpdate]:
427
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
284
-
428
image_token = hf_processor.image_token.content
286
- fake_image_token = hf_processor.fake_image_token.content
287
- global_img_token = hf_processor.global_image_tag
288
- image_seq_len = hf_processor.image_seq_len
289
- grid_placeholder = "<row_{n_h}_col_{n_w}>"
290
-
291
- p_img = image_token * image_seq_len
292
- global_img_placeholder = fake_image_token + global_img_token + p_img
293
- tile_img_placeholder = fake_image_token + grid_placeholder + p_img
429
430
def get_replacement_idefics3(item_idx: int) -> str:
431
images = mm_items.get_items("image", ImageProcessorItems)
432
433
image_size = images.get_image_size(item_idx)
299
- grid_w, grid_h = self.info._get_image_feature_grid_size(
434
+
435
+ return self.info.get_image_repl(
436
image_width=image_size.width,
437
image_height=image_size.height,
302
- **hf_processor_mm_kwargs,
438
+ processor=hf_processor,
439
)
304
- if grid_w == 0 and grid_h == 0:
305
- image_placeholder = global_img_placeholder
306
- else:
307
- tiles_placeholder = list[str]()
308
- for i in range(grid_h):
309
- for j in range(grid_w):
310
- placeholder_per_tile = tile_img_placeholder.format(
311
- n_h=i + 1, n_w=j + 1)
312
- tiles_placeholder.append(placeholder_per_tile)
313
- # Add line break if it is the last tile in the row
314
- if j == grid_w - 1:
315
- tiles_placeholder.append("\n")
316
-
317
- image_placeholder = "".join(
318
- [*tiles_placeholder, "\n", global_img_placeholder])
319
- return image_placeholder + fake_image_token
440
441
return [
442
PromptReplacement(
@@ -424,73 +544,13 @@ class Idefics3Model(nn.Module):
544
config.vision_config.patch_size)**2) / (config.scale_factor**2))
545
self.image_token_id = self.config.image_token_id
546
427
- def _validate_pixel_values(
428
- self, data: Union[torch.Tensor, List[torch.Tensor]]
429
- ) -> Union[torch.Tensor, List[torch.Tensor]]:
430
-
431
- h = w = self.config.vision_config.image_size
432
- expected_dims = (3, h, w)
433
-
434
- def _validate_shape(d: torch.Tensor):
435
- actual_dims = tuple(d.shape[1:])
436
-
437
- if actual_dims != expected_dims:
438
- expected_expr = ("num_patches", *map(str, expected_dims))
439
- raise ValueError(
440
- "The expected shape of pixel values per image per batch "
441
- f"is {expected_expr}. You supplied {tuple(d.shape)}.")
442
-
443
- for d in data:
444
- _validate_shape(d)
445
-
446
- return data
447
-
448
- def _parse_and_validate_image_input(
449
- self, **kwargs: object) -> Optional[ImageInputs]:
450
- pixel_values = kwargs.pop("pixel_values", None)
451
- image_embeds = kwargs.pop("image_embeds", None)
452
- pixel_attention_mask = kwargs.pop("pixel_attention_mask", None)
453
-
454
- if pixel_values is None and image_embeds is None:
455
- return None
456
-
457
- if image_embeds is not None:
458
- if not isinstance(image_embeds, (torch.Tensor, list)):
459
- raise ValueError("Incorrect type of image embeddings. "
460
- f"Got type: {type(image_embeds)}")
461
-
462
- return Idefics3ImageEmbeddingInputs(
463
- type="image_embeds",
464
- data=flatten_bn(image_embeds, concat=True),
465
- )
466
-
467
- if pixel_values is not None:
468
- if not isinstance(pixel_values, (torch.Tensor, list)):
469
- raise ValueError("Incorrect type of pixel values. "
470
- f"Got type: {type(pixel_values)}")
471
-
472
- if isinstance(pixel_values, list):
473
- pixel_values = torch.cat(pixel_values, dim=1)
474
- pixel_attention_mask = torch.cat(pixel_attention_mask, dim=1)
475
- else:
476
- pixel_values = flatten_bn(pixel_values)
477
- pixel_attention_mask = flatten_bn(pixel_attention_mask)
478
-
479
- return Idefics3ImagePixelInputs(
480
- type="pixel_values",
481
- data=self._validate_pixel_values(pixel_values),
482
- pixel_attention_mask=pixel_attention_mask)
483
-
484
- raise AssertionError("This line should be unreachable.")
485
-
486
- def _image_pixels_to_features(
547
+ def image_pixels_to_features(
548
self,
549
pixel_values: torch.Tensor,
489
- pixel_attention_mask: Optional[torch.BoolTensor] = None,
490
- ) -> NestedTensors:
550
+ pixel_attention_mask: torch.Tensor,
551
+ ) -> torch.Tensor:
552
# NOTE: we skip the step to select the vision feature layer since
553
# this is already done inside the vision tower
493
- num_patches = [x.size(0) for x in pixel_values]
554
pixel_values = pixel_values.to(
555
dtype=self.vision_model.embeddings.patch_embedding.weight.dtype
556
) # fp16 compatibility
@@ -502,17 +562,9 @@ class Idefics3Model(nn.Module):
562
pixel_values = pixel_values[real_images_inds].contiguous()
563
564
# Handle the vision attention mask
505
- if pixel_attention_mask is None:
506
- pixel_attention_mask = torch.ones(
507
- size=(pixel_values.size(0), pixel_values.size(2),
508
- pixel_values.size(3)),
509
- dtype=torch.bool,
510
- device=pixel_values.device,
511
- )
512
- else:
513
- # Remove padding images from the mask
514
- pixel_attention_mask = pixel_attention_mask[
515
- real_images_inds].contiguous()
565
+ # Remove padding images from the mask
566
+ pixel_attention_mask = pixel_attention_mask[
567
+ real_images_inds].contiguous()
568
569
patch_size = self.config.vision_config.patch_size
570
patches_subgrid = pixel_attention_mask.unfold(dimension=1,
@@ -529,27 +581,7 @@ class Idefics3Model(nn.Module):
581
patch_attention_mask=patch_attention_mask,
582
)
583
532
- return image_hidden_states.split(num_patches)
533
-
534
- def _process_image_pixels(
535
- self, inputs: Idefics3ImagePixelInputs) -> NestedTensors:
536
- assert self.vision_model is not None
537
-
538
- pixel_values = inputs["data"]
539
- pixel_attention_mask = inputs["pixel_attention_mask"]
540
-
541
- return self._image_pixels_to_features(pixel_values,
542
- pixel_attention_mask)
543
-
544
- def _process_image_input(self, image_input: ImageInputs) -> torch.Tensor:
545
- if image_input["type"] == "image_embeds":
546
- return image_input["data"]
547
-
548
- assert self.vision_model is not None
549
- image_features = self._process_image_pixels(image_input)
550
- num_patches = [x.size(0) for x in image_features]
551
- image_features = torch.cat(image_features)
552
- return self.connector(image_features).split(num_patches)
584
+ return image_hidden_states
585
586
def get_input_embeddings(
587
self,
@@ -616,13 +648,113 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
648
self.logits_processor = LogitsProcessor(config.text_config.vocab_size)
649
self.sampler = get_sampler()
650
651
+ def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
652
+ h = w = self.config.vision_config.image_size
653
+ expected_dims = (3, h, w)
654
+
655
+ def _validate_shape(d: torch.Tensor):
656
+ actual_dims = tuple(d.shape)
657
+
658
+ if actual_dims != expected_dims:
659
+ expected_expr = str(expected_dims)
660
+ raise ValueError(
661
+ "The expected shape of pixel values per image per batch "
662
+ f" per patch is {expected_expr}. "
663
+ f"You supplied {tuple(d.shape)}.")
664
+
665
+ for d in data:
666
+ _validate_shape(d)
667
+
668
+ return data
669
+
670
+ def _parse_and_validate_image_input(
671
+ self, **kwargs: object) -> Optional[ImageInputs]:
672
+ pixel_values = kwargs.pop("pixel_values", None)
673
+ image_embeds = kwargs.pop("image_embeds", None)
674
+
675
+ if pixel_values is None and image_embeds is None:
676
+ return None
677
+
678
+ embed_is_patch = kwargs.pop("embed_is_patch")
679
+ if not isinstance(embed_is_patch, (torch.Tensor, list)):
680
+ raise ValueError("Incorrect type of embed_is_patch. "
681
+ f"Got type: {type(embed_is_patch)}")
682
+
683
+ embed_is_patch = flatten_bn(embed_is_patch)
684
+
685
+ if image_embeds is not None:
686
+ if not isinstance(image_embeds, (torch.Tensor, list)):
687
+ raise ValueError("Incorrect type of image embeddings. "
688
+ f"Got type: {type(image_embeds)}")
689
+
690
+ return Idefics3ImageEmbeddingInputs(
691
+ type="image_embeds",
692
+ data=flatten_bn(image_embeds, concat=True),
693
+ embed_is_patch=embed_is_patch,
694
+ )
695
+
696
+ if pixel_values is not None:
697
+ if not isinstance(pixel_values, (torch.Tensor, list)):
698
+ raise ValueError("Incorrect type of pixel values. "
699
+ f"Got type: {type(pixel_values)}")
700
+
701
+ pixel_attention_mask = kwargs.pop("pixel_attention_mask")
702
+ if not isinstance(pixel_attention_mask, (torch.Tensor, list)):
703
+ raise ValueError("Incorrect type of pixel_attention_mask. "
704
+ f"Got type: {type(pixel_attention_mask)}")
705
+
706
+ num_patches = kwargs.pop("num_patches")
707
+ if not isinstance(num_patches, (torch.Tensor, list)):
708
+ raise ValueError("Incorrect type of num_patches. "
709
+ f"Got type: {type(num_patches)}")
710
+
711
+ pixel_values = flatten_bn(pixel_values, concat=True)
712
+ pixel_attention_mask = flatten_bn(pixel_attention_mask,
713
+ concat=True)
714
+ num_patches = flatten_bn(num_patches, concat=True)
715
+
716
+ return Idefics3ImagePixelInputs(
717
+ type="pixel_values",
718
+ pixel_values=self._validate_pixel_values(pixel_values),
719
+ pixel_attention_mask=pixel_attention_mask,
720
+ num_patches=num_patches,
721
+ embed_is_patch=embed_is_patch,
722
+ )
723
+
724
+ raise AssertionError("This line should be unreachable.")
725
+
726
+ def _process_image_pixels(
727
+ self, inputs: Idefics3ImagePixelInputs) -> torch.Tensor:
728
+ pixel_values = inputs["pixel_values"]
729
+ pixel_attention_mask = inputs["pixel_attention_mask"]
730
+
731
+ return self.model.image_pixels_to_features(
732
+ pixel_values,
733
+ pixel_attention_mask=pixel_attention_mask,
734
+ )
735
+
736
+ def _process_image_input(self, image_input: ImageInputs) -> torch.Tensor:
737
+ if image_input["type"] == "image_embeds":
738
+ return image_input["data"]
739
+
740
+ image_features = self._process_image_pixels(image_input)
741
+ image_features = self.model.connector(image_features)
742
+
743
+ num_patches = image_input["num_patches"]
744
+ return image_features.split(num_patches.tolist())
745
+
746
def get_multimodal_embeddings(
747
self, **kwargs: object) -> Optional[MultiModalEmbeddings]:
621
- image_input = self.model._parse_and_validate_image_input(**kwargs)
748
+ image_input = self._parse_and_validate_image_input(**kwargs)
749
if image_input is None:
750
return None
624
- vision_embeddings = self.model._process_image_input(image_input)
625
- return vision_embeddings
751
+
752
+ image_features = self._process_image_input(image_input)
753
+
754
+ return scatter_patch_features(
755
+ image_features,
756
+ image_input["embed_is_patch"],
757
+ )
758
759
def get_input_embeddings(
760
self,
@@ -632,8 +764,11 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
764
inputs_embeds = self.model.get_input_embeddings(input_ids)
765
if multimodal_embeddings is not None:
766
inputs_embeds = merge_multimodal_embeddings(
635
- input_ids, inputs_embeds, multimodal_embeddings,
636
- self.config.image_token_id)
767
+ input_ids,
768
+ inputs_embeds,
769
+ select_patch_features(multimodal_embeddings),
770
+ self.config.image_token_id,
771
+ )
772
return inputs_embeds
773
774
def forward(
vllm/model_executor/models/mllama.py
-1
@@ -21,7 +21,6 @@ from typing import List, Literal, Optional, Set, Tuple, TypedDict, Union
21
import numpy as np
22
import torch
23
import torch.nn.functional as F
24
-import torch.utils.checkpoint
24
import transformers.models.mllama.configuration_mllama as config_mllama
25
from PIL.Image import Image
26
from torch import nn
vllm/model_executor/models/qwen2_audio.py
+1
-1
@@ -160,7 +160,7 @@ class Qwen2AudioMultiModalProcessor(
160
mm_kwargs: Mapping[str, Any],
161
) -> BatchFeature:
162
# Text-only input not supported in composite processor
163
- if not mm_data or not mm_data.get("audios", []):
163
+ if not mm_data.get("audios", []):
164
prompt_ids = self.info.get_tokenizer().encode(prompt)
165
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
166
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
vllm/model_executor/models/ultravox.py
+1
-2
@@ -8,7 +8,6 @@ from functools import cached_property
8
from typing import Any, Literal, Optional, Set, Tuple, TypedDict, Union
9
10
import torch
11
-import torch.utils.checkpoint
11
from torch import nn
12
from torch.nn import functional as F
13
from transformers import BatchFeature, ProcessorMixin
@@ -160,7 +159,7 @@ class UltravoxMultiModalProcessor(
159
mm_kwargs: Mapping[str, object],
160
) -> BatchFeature:
161
# Text-only input not supported in composite processor
163
- if not mm_data or not mm_data.get("audios", []):
162
+ if not mm_data.get("audios", []):
163
prompt_ids = self.info.get_tokenizer().encode(
164
prompt, add_special_tokens=False)
165
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)