@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ef9baee3c5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Bugfix][VLM] Fix incompatibility between #7902 and #7230 (#7948)

Cyrus Leung committed Aug 28, 2024 at 23:11 UTC ef9baee3c52f719df64a646db72b6c4ede8a29a0
10 files changed +120 -92
vllm/model_executor/models/blip2.py
+2 -2
@@ -40,13 +40,13 @@ BLIP2_IMAGE_TOKEN_ID = 50265
40 class Blip2ImagePixelInputs(TypedDict):
41 type: Literal["pixel_values"]
42 data: torch.Tensor
43 - """Shape: (batch_size, num_channels, height, width)"""
43 + """Shape: `(batch_size * num_images, num_channels, height, width)`"""
44
45
46 class Blip2ImageEmbeddingInputs(TypedDict):
47 type: Literal["image_embeds"]
48 data: torch.Tensor
49 - """Shape: `(batch_size, image_feature_size, hidden_size)`
49 + """Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
50
51 `hidden_size` must match the hidden size of language model backbone.
52 """
vllm/model_executor/models/chameleon.py
+1 -1
@@ -53,7 +53,7 @@ CHAMELEON_SEP_TOKEN_ID = 8710
53 class ChameleonImagePixelInputs(TypedDict):
54 type: Literal["pixel_values"]
55 data: torch.Tensor
56 - """Shape: `(batch_size, num_channels, height, width)`"""
56 + """Shape: `(batch_size * num_images, num_channels, height, width)`"""
57
58
59 def get_max_chameleon_image_tokens(ctx: InputContext):
vllm/model_executor/models/internvl.py
+15 -31
@@ -29,7 +29,7 @@ from vllm.sequence import IntermediateTensors, SamplerOutput
29 from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
30 get_clip_num_patches)
31 from .interfaces import SupportsMultiModal
32 -from .utils import (filter_weights, init_vllm_registered_model,
32 +from .utils import (filter_weights, flatten_bn, init_vllm_registered_model,
33 merge_multimodal_embeddings)
34
35 IMG_START = '<img>'
@@ -42,19 +42,17 @@ IMAGENET_STD = (0.229, 0.224, 0.225)
42
43 class InternVLImagePixelInputs(TypedDict):
44 type: Literal["pixel_values"]
45 - data: Union[torch.Tensor, List[torch.Tensor]]
45 + data: torch.Tensor
46 """
47 - Shape: `(batch_size, 1 + num_patches, num_channels, height, width)`
48 -
49 - Note that `num_patches` may be different for each batch, in which case
50 - the data is passed as a list instead of a batched tensor.
47 + Shape:
48 + `(batch_size * num_images * (1 + num_patches), num_channels, height, width)`
49 """
50
51
52 class InternVLImageEmbeddingInputs(TypedDict):
53 type: Literal["image_embeds"]
56 - data: Union[torch.Tensor, List[torch.Tensor]]
57 - """Shape: `(batch_size, image_feature_size, hidden_size)`
54 + data: torch.Tensor
55 + """Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
56
57 `hidden_size` must match the hidden size of language model backbone.
58 """
@@ -357,7 +355,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal):
355 x = x.permute(0, 2, 1, 3).contiguous()
356 return x
357
360 - def extract_feature(self, pixel_values):
358 + def extract_feature(self, pixel_values: torch.Tensor) -> torch.Tensor:
359 vit_embeds = self.vision_model(pixel_values=pixel_values)
360 vit_embeds = vit_embeds[:, 1:, :]
361
@@ -370,17 +368,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal):
368 vit_embeds = self.mlp1(vit_embeds)
369 return vit_embeds
370
373 - def _validate_image_sizes(self, data: torch.Tensor) -> torch.Tensor:
374 - if list(data.shape[1:]) != [2]:
375 - raise ValueError(
376 - f"The expected image sizes shape is batch dimension plus "
377 - f"{[2]}. You supplied {data.shape}.")
378 -
379 - return data
380 -
381 - def _validate_pixel_values(
382 - self, data: Union[torch.Tensor, List[torch.Tensor]]
383 - ) -> Union[torch.Tensor, List[torch.Tensor]]:
371 + def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
372
373 h = w = self.config.vision_config.image_size
374 expected_dims = (3, h, w)
@@ -389,10 +377,11 @@ class InternVLChatModel(nn.Module, SupportsMultiModal):
377 actual_dims = tuple(d.shape)
378
379 if actual_dims != expected_dims:
392 - expected_expr = ("num_patches", *map(str, expected_dims))
380 + expected_expr = str(expected_dims)
381 raise ValueError(
394 - "The expected shape of pixel values in each batch element "
395 - f"is {expected_expr}. You supplied {tuple(d.shape)}.")
382 + "The expected shape of pixel values per image per batch "
383 + f" per patch is {expected_expr}. "
384 + f"You supplied {tuple(d.shape)}.")
385
386 for d in data:
387 _validate_shape(d)
@@ -413,12 +402,9 @@ class InternVLChatModel(nn.Module, SupportsMultiModal):
402 raise ValueError("Incorrect type of image embeddings. "
403 f"Got type: {type(image_embeds)}")
404
416 - # Flatten the B and N dimensions
417 - image_embeds = image_embeds.flatten(0, 2)
418 -
405 return InternVLImageEmbeddingInputs(
406 type="image_embeds",
421 - data=image_embeds,
407 + data=flatten_bn(image_embeds),
408 )
409
410 self.img_context_token_id = image_token_id[0]
@@ -428,12 +414,10 @@ class InternVLChatModel(nn.Module, SupportsMultiModal):
414 raise ValueError("Incorrect type of pixel values. "
415 f"Got type: {type(pixel_values)}")
416
431 - # Flatten the B and N dimensions
432 - pixel_values = pixel_values.flatten(0, 2)
433 -
417 return InternVLImagePixelInputs(
418 type="pixel_values",
436 - data=self._validate_pixel_values(pixel_values),
419 + data=self._validate_pixel_values(
420 + flatten_bn(pixel_values, concat=True).flatten(0, 1)),
421 )
422
423 raise AssertionError("This line should be unreachable.")
vllm/model_executor/models/llava.py
+2 -2
@@ -30,13 +30,13 @@ from .utils import (filter_weights, init_vllm_registered_model,
30 class LlavaImagePixelInputs(TypedDict):
31 type: Literal["pixel_values"]
32 data: torch.Tensor
33 - """Shape: `(batch_size, num_channels, height, width)`"""
33 + """Shape: `(batch_size * num_images, num_channels, height, width)`"""
34
35
36 class LlavaImageEmbeddingInputs(TypedDict):
37 type: Literal["image_embeds"]
38 data: torch.Tensor
39 - """Shape: `(batch_size, image_feature_size, hidden_size)`
39 + """Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
40
41 `hidden_size` must match the hidden size of language model backbone.
42 """
vllm/model_executor/models/llava_next.py
+26 -26
@@ -29,7 +29,7 @@ from .llava import LlavaMultiModalProjector
29 from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
30 dummy_seq_data_for_siglip, get_siglip_image_feature_size,
31 get_siglip_patch_grid_length, input_processor_for_siglip)
32 -from .utils import (filter_weights, init_vllm_registered_model,
32 +from .utils import (filter_weights, flatten_bn, init_vllm_registered_model,
33 merge_multimodal_embeddings)
34
35 logger = init_logger(__name__)
@@ -47,15 +47,16 @@ class LlavaNextImagePixelInputs(TypedDict):
47 type: Literal["pixel_values"]
48 data: Union[torch.Tensor, List[torch.Tensor]]
49 """
50 - Shape: `(batch_size, 1 + num_patches, num_channels, height, width)`
50 + Shape:
51 + `(batch_size * num_images, 1 + num_patches, num_channels, height, width)`
52
52 - Note that `num_patches` may be different for each batch, in which case
53 - the data is passed as a list instead of a batched tensor.
53 + Note that `num_patches` may be different per batch and image,
54 + in which case the data is passed as a list instead of a batched tensor.
55 """
56
57 image_sizes: NotRequired[torch.Tensor]
58 """
58 - Shape: `(batch_size, 2)`
59 + Shape: `(batch_size * num_images, 2)`
60
61 This should be in `(height, width)` format.
62 """
@@ -64,7 +65,7 @@ class LlavaNextImagePixelInputs(TypedDict):
65 class LlavaNextImageEmbeddingInputs(TypedDict):
66 type: Literal["image_embeds"]
67 data: torch.Tensor
67 - """Shape: `(batch_size, image_feature_size, hidden_size)`
68 + """Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
69
70 `hidden_size` must match the hidden size of language model backbone.
71 """
@@ -315,10 +316,19 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal):
316 torch.empty(config.text_config.hidden_size))
317
318 def _validate_image_sizes(self, data: torch.Tensor) -> torch.Tensor:
318 - if list(data.shape[1:]) != [2]:
319 - raise ValueError(
320 - f"The expected image sizes shape is batch dimension plus "
321 - f"{[2]}. You supplied {data.shape}.")
319 + expected_dims = (2, )
320 +
321 + def _validate_shape(d: torch.Tensor):
322 + actual_dims = tuple(d.shape)
323 +
324 + if actual_dims != expected_dims:
325 + expected_expr = str(expected_dims)
326 + raise ValueError(
327 + f"The expected shape of image sizes per image per batch "
328 + f"is {expected_expr}. You supplied {tuple(d.shape)}.")
329 +
330 + for d in data:
331 + _validate_shape(d)
332
333 return data
334
@@ -335,7 +345,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal):
345 if actual_dims != expected_dims:
346 expected_expr = ("num_patches", *map(str, expected_dims))
347 raise ValueError(
338 - "The expected shape of pixel values in each batch element "
348 + "The expected shape of pixel values per image per batch "
349 f"is {expected_expr}. You supplied {tuple(d.shape)}.")
350
351 for d in data:
@@ -357,22 +367,15 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal):
367 raise ValueError("Incorrect type of pixel values. "
368 f"Got type: {type(pixel_values)}")
369
360 - if not isinstance(image_sizes, torch.Tensor):
370 + if not isinstance(image_sizes, (torch.Tensor, list)):
371 raise ValueError("Incorrect type of image sizes. "
372 f"Got type: {type(image_sizes)}")
373
364 - # Remove the N dimension until multiple images are supported.
365 - if isinstance(pixel_values, torch.Tensor):
366 - pixel_values = pixel_values.squeeze(1)
367 - else:
368 - pixel_values = [t.squeeze(0) for t in pixel_values]
369 -
370 - image_sizes = image_sizes.squeeze(1)
371 -
374 return LlavaNextImagePixelInputs(
375 type="pixel_values",
374 - data=self._validate_pixel_values(pixel_values),
375 - image_sizes=self._validate_image_sizes(image_sizes),
376 + data=self._validate_pixel_values(flatten_bn(pixel_values)),
377 + image_sizes=self._validate_image_sizes(
378 + flatten_bn(image_sizes, concat=True)),
379 )
380
381 if image_embeds is not None:
@@ -380,12 +383,9 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal):
383 raise ValueError("Incorrect type of image embeds. "
384 f"Got type: {type(image_embeds)}")
385
383 - # Remove the N dimension until multiple images are supported.
384 - image_embeds = image_embeds.squeeze(1)
385 -
386 return LlavaNextImageEmbeddingInputs(
387 type="image_embeds",
388 - data=image_embeds,
388 + data=flatten_bn(image_embeds),
389 )
390
391 raise AssertionError("This line should be unreachable.")
vllm/model_executor/models/paligemma.py
+2 -2
@@ -34,13 +34,13 @@ _KEYS_TO_MODIFY_MAPPING = {
34 class PaliGemmaImagePixelInputs(TypedDict):
35 type: Literal["pixel_values"]
36 data: torch.Tensor
37 - """Shape: (batch_size, num_channels, height, width)"""
37 + """Shape: `(batch_size * num_images, num_channels, height, width)`"""
38
39
40 class PaliGemmaImageEmbeddingInputs(TypedDict):
41 type: Literal["image_embeds"]
42 data: torch.Tensor
43 - """Shape: `(batch_size, image_feature_size, hidden_size)`
43 + """Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
44
45 `hidden_size` must match the hidden size of language model backbone.
46 """
vllm/model_executor/models/phi3v.py
+27 -23
@@ -44,7 +44,7 @@ from vllm.utils import is_list_of
44
45 from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
46 from .interfaces import SupportsMultiModal
47 -from .utils import merge_multimodal_embeddings
47 +from .utils import flatten_bn, merge_multimodal_embeddings
48
49 logger = init_logger(__name__)
50
@@ -75,15 +75,16 @@ class Phi3VImagePixelInputs(TypedDict):
75 type: Literal["pixel_values"]
76 data: Union[torch.Tensor, List[torch.Tensor]]
77 """
78 - Shape: `(batch_size, 1 + num_patches, num_channels, height, width)`
78 + Shape:
79 + `(batch_size * num_images, 1 + num_patches, num_channels, height, width)`
80
80 - Note that `num_patches` may be different for each batch, in which case
81 - the data is passed as a list instead of a batched tensor.
81 + Note that `num_patches` may be different per batch and image,
82 + in which case the data is passed as a list instead of a batched tensor.
83 """
84
85 image_sizes: torch.Tensor
86 """
86 - Shape: `(batch_size, 2)`
87 + Shape: `(batch_size * num_images, 2)`
88
89 This should be in `(height, width)` format.
90 """
@@ -92,7 +93,7 @@ class Phi3VImagePixelInputs(TypedDict):
93 class Phi3VImageEmbeddingInputs(TypedDict):
94 type: Literal["image_embeds"]
95 data: Union[torch.Tensor, List[torch.Tensor]]
95 - """Shape: `(batch_size, image_feature_size, hidden_size)`
96 + """Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
97
98 `hidden_size` must match the hidden size of language model backbone.
99 """
@@ -511,10 +512,19 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal):
512 self.sampler = Sampler()
513
514 def _validate_image_sizes(self, data: torch.Tensor) -> torch.Tensor:
514 - if list(data.shape[1:]) != [2]:
515 - raise ValueError(
516 - f"The expected shape of image sizes is batch dimension plus "
517 - f"{[2]}. You supplied {tuple(data.shape)}.")
515 + expected_dims = (2, )
516 +
517 + def _validate_shape(d: torch.Tensor):
518 + actual_dims = tuple(d.shape)
519 +
520 + if actual_dims != expected_dims:
521 + expected_expr = str(expected_dims)
522 + raise ValueError(
523 + f"The expected shape of image sizes per image per batch "
524 + f"is {expected_expr}. You supplied {tuple(d.shape)}.")
525 +
526 + for d in data:
527 + _validate_shape(d)
528
529 return data
530
@@ -531,7 +541,7 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal):
541 if actual_dims != expected_dims:
542 expected_expr = ("num_patches", *map(str, expected_dims))
543 raise ValueError(
534 - "The expected shape of pixel values in each batch element "
544 + "The expected shape of pixel values per image per batch "
545 f"is {expected_expr}. You supplied {tuple(d.shape)}.")
546
547 for d in data:
@@ -556,30 +566,24 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal):
566 raise ValueError("Incorrect type of pixel values. "
567 f"Got type: {type(pixel_values)}")
568
559 - if not isinstance(image_sizes, torch.Tensor):
569 + if not isinstance(image_sizes, (torch.Tensor, list)):
570 raise ValueError("Incorrect type of image sizes. "
571 f"Got type: {type(image_sizes)}")
572
563 - # Merge the B and N dimensions.
564 - if isinstance(pixel_values, torch.Tensor):
565 - pixel_values = pixel_values.flatten(0, 1)
566 - else:
567 - pixel_values = torch.cat(pixel_values)
568 -
569 - image_sizes = image_sizes.flatten(0, 1)
570 -
573 return Phi3VImagePixelInputs(
574 type="pixel_values",
573 - data=self._validate_pixel_values(pixel_values),
574 - image_sizes=self._validate_image_sizes(image_sizes))
575 + data=self._validate_pixel_values(flatten_bn(pixel_values)),
576 + image_sizes=self._validate_image_sizes(
577 + flatten_bn(image_sizes, concat=True)))
578
579 if image_embeds is not None:
580 if not isinstance(image_embeds, torch.Tensor):
581 raise ValueError("Incorrect type of image embeddings. "
582 f"Got type: {type(image_embeds)}")
583 +
584 return Phi3VImageEmbeddingInputs(
585 type="image_embeds",
582 - data=image_embeds,
586 + data=flatten_bn(image_embeds),
587 )
588
589 raise AssertionError("This line should be unreachable.")
vllm/model_executor/models/ultravox.py
+1 -1
@@ -49,7 +49,7 @@ logger = init_logger(__name__)
49 class UltravoxAudioFeatureInputs(TypedDict):
50 type: Literal["audio_features"]
51 data: Union[torch.Tensor, List[torch.Tensor]]
52 - """Shape: `(batch_size, 80, M)"""
52 + """Shape: `(batch_size * num_audios, 80, M)"""
53
54
55 class UltravoxAudioEmbeddingInputs(TypedDict):
vllm/model_executor/models/utils.py
+42 -2
@@ -1,4 +1,5 @@
1 -from typing import Dict, Iterable, List, Optional, Protocol, Tuple
1 +from typing import (Dict, Iterable, List, Literal, Optional, Protocol, Tuple,
2 + Union, overload)
3
4 import numpy as np
5 import torch
@@ -55,6 +56,44 @@ def init_vllm_registered_model(
56 )
57
58
59 +@overload
60 +def flatten_bn(x: torch.Tensor) -> torch.Tensor:
61 + ...
62 +
63 +
64 +@overload
65 +def flatten_bn(x: List[torch.Tensor]) -> List[torch.Tensor]:
66 + ...
67 +
68 +
69 +@overload
70 +def flatten_bn(
71 + x: Union[List[torch.Tensor], torch.Tensor],
72 + *,
73 + concat: Literal[True],
74 +) -> torch.Tensor:
75 + ...
76 +
77 +
78 +def flatten_bn(
79 + x: Union[List[torch.Tensor], torch.Tensor],
80 + *,
81 + concat: bool = False,
82 +) -> Union[List[torch.Tensor], torch.Tensor]:
83 + """
84 + Flatten the ``B`` and ``N`` dimensions of batched multimodal inputs.
85 +
86 + The input tensor should have shape ``(B, N, ...)```.
87 + """
88 + if isinstance(x, torch.Tensor):
89 + return x.flatten(0, 1)
90 +
91 + if concat:
92 + return torch.cat(x)
93 +
94 + return [x_n for x_b in x for x_n in x_b]
95 +
96 +
97 def _flatten_embeddings(embeddings: NestedTensors) -> torch.Tensor:
98 """
99 Recursively concatenates NestedTensors along any heterogeneously sized
@@ -93,7 +132,8 @@ def merge_multimodal_embeddings(input_ids: torch.Tensor,
132 This updates ``inputs_embeds`` in place.
133 """
134 mask = (input_ids == placeholder_token_id)
96 - num_expected_tokens = mask.sum()
135 + num_expected_tokens = mask.sum().item()
136 + assert isinstance(num_expected_tokens, int)
137
138 flattened = _flatten_embeddings(multimodal_embeddings)
139 *dims, embed_dim = flattened.shape
vllm/multimodal/base.py
+2 -2
@@ -18,7 +18,7 @@ from vllm.utils import JSONTree, is_list_of, json_map_leaves
18
19 logger = init_logger(__name__)
20
21 -NestedTensors = Union[List["NestedTensors"], torch.Tensor]
21 +NestedTensors = Union[List["NestedTensors"], List[torch.Tensor], torch.Tensor]
22 """
23 Uses a list instead of a tensor if the dimensions of each element do not match.
24 """
@@ -61,7 +61,7 @@ class MultiModalInputs(_MultiModalInputsBase):
61 tensors_ = cast(List[torch.Tensor], stacked)
62 if any(t.shape != tensors_[0].shape for t in tensors_):
63 # The tensors have incompatible shapes and can't be stacked.
64 - return stacked
64 + return tensors_
65
66 return torch.stack(tensors_)
67