@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2e32f5d28d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Bugfix] Fix Idefics3 fails during multi-image inference (#11080)

Signed-off-by: B-201 <Joy25810@foxmail.com>

B-201 committed Dec 11, 2024 at 17:27 UTC 2e32f5d28db3cd79f6a421f640e083be1f9468b7
1 file changed +13 -8
vllm/model_executor/models/idefics3.py
+13 -8
@@ -60,7 +60,8 @@ class Idefics3ImagePixelInputs(TypedDict):
60 type: Literal["pixel_values"]
61 data: torch.Tensor
62 """
63 - Shape: `(batch_size * num_images, num_channels, height, width)`
63 + Shape: `(batch_size * num_images * num_patches,
64 + num_channels, height, width)`
65 """
66 pixel_attention_mask: Optional[torch.BoolTensor]
67
@@ -520,13 +521,17 @@ class Idefics3Model(nn.Module):
521 raise ValueError("Incorrect type of pixel values. "
522 f"Got type: {type(pixel_values)}")
523
523 - return Idefics3ImagePixelInputs(type="pixel_values",
524 - data=self._validate_pixel_values(
525 - flatten_bn(pixel_values,
526 - concat=True)),
527 - pixel_attention_mask=flatten_bn(
528 - pixel_attention_mask,
529 - concat=True))
524 + if isinstance(pixel_values, list):
525 + pixel_values = torch.cat(pixel_values, dim=1)
526 + pixel_attention_mask = torch.cat(pixel_attention_mask, dim=1)
527 + else:
528 + pixel_values = flatten_bn(pixel_values)
529 + pixel_attention_mask = flatten_bn(pixel_attention_mask)
530 +
531 + return Idefics3ImagePixelInputs(
532 + type="pixel_values",
533 + data=self._validate_pixel_values(pixel_values),
534 + pixel_attention_mask=pixel_attention_mask)
535
536 raise AssertionError("This line should be unreachable.")
537