@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5241aa1494
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Model][Bugfix] Fix batching with multi-image in PixtralHF (#9518)

Michael Goin committed Oct 21, 2024 at 14:20 UTC 5241aa1494a7410f7e89eb341700821e30d04199
2 files changed +54 -17
vllm/model_executor/models/llava.py
+48 -12
@@ -287,6 +287,34 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
287
288 return data
289
290 + def _validate_image_sizes(self, images: List[torch.Tensor],
291 + sizes: List[torch.Tensor]) -> List[torch.Tensor]:
292 + if not isinstance(sizes, list):
293 + sizes = [sizes]
294 +
295 + total_images = sum(size.numel() // 2 for size in sizes)
296 + if total_images != len(images):
297 + raise ValueError("Mismatch in number of images. "
298 + f"Expected {total_images}, got {len(images)}")
299 + img_idx = 0
300 + for size in sizes:
301 + # Flatten the size tensor to a list of (height, width) pairs
302 + size = size.view(-1, 2).tolist()
303 + for expected_h, expected_w in size:
304 + if img_idx >= len(images):
305 + raise ValueError("Ran out of images before sizes. "
306 + f"{img_idx} >= {len(images)}")
307 + img = images[img_idx]
308 + if img.shape[-2:] != (expected_h, expected_w):
309 + raise ValueError(
310 + "Image size mismatch. Expected "
311 + f"{(expected_h, expected_w)}, got {img.shape[-2:]}")
312 + if img.shape[-3] != 3:
313 + raise ValueError("Image channel mismatch. Expected 3, "
314 + f"got {img.shape[-3]}")
315 + img_idx += 1
316 + return images
317 +
318 def _parse_and_validate_image_input(
319 self, **kwargs: object) -> Optional[LlavaImageInputs]:
320 pixel_values = kwargs.pop("pixel_values", None)
@@ -305,20 +333,28 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
333 # so we need to produce a list of tensors
334 if image_sizes is not None:
335 images = pixel_values
308 - if isinstance(images, torch.Tensor):
309 - # if passed as batch take all images
310 - NN, N, B, C, W, H = images.shape
311 - images = images.reshape(NN * N * B, C, W, H)
312 - images = [images[i] for i in range(images.size(0))]
313 - elif isinstance(images, list):
314 - # if passed as list flatten lists of tensors
315 - while isinstance(images, list) and len(images) == 1:
316 - images = images[0]
317 -
318 - # TODO: Add validation based on image_sizes
336 +
337 + def flatten_to_3d_tensors(item):
338 + if isinstance(item, torch.Tensor):
339 + if item.dim() >= 3:
340 + return [t for t in item.view(-1, *item.shape[-3:])]
341 + else:
342 + raise ValueError(
343 + f"Unexpected tensor dimension: {item.dim()}")
344 + elif isinstance(item, list):
345 + return [
346 + t for subitem in item
347 + for t in flatten_to_3d_tensors(subitem)
348 + ]
349 + else:
350 + raise ValueError(f"Unexpected type: {type(item)}")
351 +
352 + # Restructure the batched images into a list of lists of images
353 + images = flatten_to_3d_tensors(pixel_values)
354 +
355 return LlavaImagePixelInputs(
356 type="pixel_values",
321 - data=images,
357 + data=self._validate_image_sizes(images, image_sizes),
358 )
359
360 return LlavaImagePixelInputs(
vllm/model_executor/models/pixtral.py
+6 -5
@@ -907,17 +907,18 @@ class PixtralHFVisionModel(nn.Module):
907 ) -> torch.Tensor:
908 """
909 Args:
910 - pixel_values: tensor of token features for
911 - all tokens of all images of shape (N_toks, D)
910 + pixel_values: Each image to be processed will be a separate tensor
911 + in pixel_values. This means it will be a list of tensors
912 + because multiple requests batched can have multiple images,
913 + each with their own shape potentially
914 +
915 Returns:
916 image_features: tensor of token features for
917 all tokens of all images of shape (N_toks, D)
918 """
919 # pass images through initial convolution independently
920 patch_embeds_list = [
918 - self.patch_conv(
919 - img.reshape(-1, img.shape[-3], img.shape[-2],
920 - img.shape[-1]).to(self.dtype))
921 + self.patch_conv(img.unsqueeze(0).to(self.dtype))
922 for img in pixel_values
923 ]
924