A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Model][Bugfix] Fix batching with multi-image in PixtralHF (#9518)
Michael Goin committed
Oct 21, 2024 at 14:20 UTC
5241aa1494a7410f7e89eb341700821e30d04199
2 files changed
+54
-17
vllm/model_executor/models/llava.py
+48
-12
@@ -287,6 +287,34 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
287
288
return data
289
290
+ def _validate_image_sizes(self, images: List[torch.Tensor],
291
+ sizes: List[torch.Tensor]) -> List[torch.Tensor]:
292
+ if not isinstance(sizes, list):
293
+ sizes = [sizes]
294
+
295
+ total_images = sum(size.numel() // 2 for size in sizes)
296
+ if total_images != len(images):
297
+ raise ValueError("Mismatch in number of images. "
298
+ f"Expected {total_images}, got {len(images)}")
299
+ img_idx = 0
300
+ for size in sizes:
301
+ # Flatten the size tensor to a list of (height, width) pairs
302
+ size = size.view(-1, 2).tolist()
303
+ for expected_h, expected_w in size:
304
+ if img_idx >= len(images):
305
+ raise ValueError("Ran out of images before sizes. "
306
+ f"{img_idx} >= {len(images)}")
307
+ img = images[img_idx]
308
+ if img.shape[-2:] != (expected_h, expected_w):
309
+ raise ValueError(
310
+ "Image size mismatch. Expected "
311
+ f"{(expected_h, expected_w)}, got {img.shape[-2:]}")
312
+ if img.shape[-3] != 3:
313
+ raise ValueError("Image channel mismatch. Expected 3, "
314
+ f"got {img.shape[-3]}")
315
+ img_idx += 1
316
+ return images
317
+
318
def _parse_and_validate_image_input(
319
self, **kwargs: object) -> Optional[LlavaImageInputs]:
320
pixel_values = kwargs.pop("pixel_values", None)
@@ -305,20 +333,28 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
333
# so we need to produce a list of tensors
334
if image_sizes is not None:
335
images = pixel_values
308
- if isinstance(images, torch.Tensor):
309
- # if passed as batch take all images
310
- NN, N, B, C, W, H = images.shape
311
- images = images.reshape(NN * N * B, C, W, H)
312
- images = [images[i] for i in range(images.size(0))]
313
- elif isinstance(images, list):
314
- # if passed as list flatten lists of tensors
315
- while isinstance(images, list) and len(images) == 1:
316
- images = images[0]
317
-
318
- # TODO: Add validation based on image_sizes
336
+
337
+ def flatten_to_3d_tensors(item):
338
+ if isinstance(item, torch.Tensor):
339
+ if item.dim() >= 3:
340
+ return [t for t in item.view(-1, *item.shape[-3:])]
341
+ else:
342
+ raise ValueError(
343
+ f"Unexpected tensor dimension: {item.dim()}")
344
+ elif isinstance(item, list):
345
+ return [
346
+ t for subitem in item
347
+ for t in flatten_to_3d_tensors(subitem)
348
+ ]
349
+ else:
350
+ raise ValueError(f"Unexpected type: {type(item)}")
351
+
352
+ # Restructure the batched images into a list of lists of images
353
+ images = flatten_to_3d_tensors(pixel_values)
354
+
355
return LlavaImagePixelInputs(
356
type="pixel_values",
321
- data=images,
357
+ data=self._validate_image_sizes(images, image_sizes),
358
)
359
360
return LlavaImagePixelInputs(
vllm/model_executor/models/pixtral.py
+6
-5
@@ -907,17 +907,18 @@ class PixtralHFVisionModel(nn.Module):
907
) -> torch.Tensor:
908
"""
909
Args:
910
- pixel_values: tensor of token features for
911
- all tokens of all images of shape (N_toks, D)
910
+ pixel_values: Each image to be processed will be a separate tensor
911
+ in pixel_values. This means it will be a list of tensors
912
+ because multiple requests batched can have multiple images,
913
+ each with their own shape potentially
914
+
915
Returns:
916
image_features: tensor of token features for
917
all tokens of all images of shape (N_toks, D)
918
"""
919
# pass images through initial convolution independently
920
patch_embeds_list = [
918
- self.patch_conv(
919
- img.reshape(-1, img.shape[-3], img.shape[-2],
920
- img.shape[-1]).to(self.dtype))
921
+ self.patch_conv(img.unsqueeze(0).to(self.dtype))
922
for img in pixel_values
923
]
924