A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Bugfix] Fix Idefics3 fails during multi-image inference (#11080)
Signed-off-by: B-201 <Joy25810@foxmail.com>
B-201 committed
Dec 11, 2024 at 17:27 UTC
2e32f5d28db3cd79f6a421f640e083be1f9468b7
1 file changed
+13
-8
vllm/model_executor/models/idefics3.py
+13
-8
@@ -60,7 +60,8 @@ class Idefics3ImagePixelInputs(TypedDict):
60
type: Literal["pixel_values"]
61
data: torch.Tensor
62
"""
63
- Shape: `(batch_size * num_images, num_channels, height, width)`
63
+ Shape: `(batch_size * num_images * num_patches,
64
+ num_channels, height, width)`
65
"""
66
pixel_attention_mask: Optional[torch.BoolTensor]
67
@@ -520,13 +521,17 @@ class Idefics3Model(nn.Module):
521
raise ValueError("Incorrect type of pixel values. "
522
f"Got type: {type(pixel_values)}")
523
523
- return Idefics3ImagePixelInputs(type="pixel_values",
524
- data=self._validate_pixel_values(
525
- flatten_bn(pixel_values,
526
- concat=True)),
527
- pixel_attention_mask=flatten_bn(
528
- pixel_attention_mask,
529
- concat=True))
524
+ if isinstance(pixel_values, list):
525
+ pixel_values = torch.cat(pixel_values, dim=1)
526
+ pixel_attention_mask = torch.cat(pixel_attention_mask, dim=1)
527
+ else:
528
+ pixel_values = flatten_bn(pixel_values)
529
+ pixel_attention_mask = flatten_bn(pixel_attention_mask)
530
+
531
+ return Idefics3ImagePixelInputs(
532
+ type="pixel_values",
533
+ data=self._validate_pixel_values(pixel_values),
534
+ pixel_attention_mask=pixel_attention_mask)
535
536
raise AssertionError("This line should be unreachable.")
537