A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Bugfix][VLM] Fix incompatibility between #7902 and #7230 (#7948)
Cyrus Leung committed
Aug 28, 2024 at 23:11 UTC
ef9baee3c52f719df64a646db72b6c4ede8a29a0
10 files changed
+120
-92
vllm/model_executor/models/blip2.py
+2
-2
@@ -40,13 +40,13 @@ BLIP2_IMAGE_TOKEN_ID = 50265
40
class Blip2ImagePixelInputs(TypedDict):
41
type: Literal["pixel_values"]
42
data: torch.Tensor
43
- """Shape: (batch_size, num_channels, height, width)"""
43
+ """Shape: `(batch_size * num_images, num_channels, height, width)`"""
44
45
46
class Blip2ImageEmbeddingInputs(TypedDict):
47
type: Literal["image_embeds"]
48
data: torch.Tensor
49
- """Shape: `(batch_size, image_feature_size, hidden_size)`
49
+ """Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
50
51
`hidden_size` must match the hidden size of language model backbone.
52
"""
vllm/model_executor/models/chameleon.py
+1
-1
@@ -53,7 +53,7 @@ CHAMELEON_SEP_TOKEN_ID = 8710
53
class ChameleonImagePixelInputs(TypedDict):
54
type: Literal["pixel_values"]
55
data: torch.Tensor
56
- """Shape: `(batch_size, num_channels, height, width)`"""
56
+ """Shape: `(batch_size * num_images, num_channels, height, width)`"""
57
58
59
def get_max_chameleon_image_tokens(ctx: InputContext):
vllm/model_executor/models/internvl.py
+15
-31
@@ -29,7 +29,7 @@ from vllm.sequence import IntermediateTensors, SamplerOutput
29
from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
30
get_clip_num_patches)
31
from .interfaces import SupportsMultiModal
32
-from .utils import (filter_weights, init_vllm_registered_model,
32
+from .utils import (filter_weights, flatten_bn, init_vllm_registered_model,
33
merge_multimodal_embeddings)
34
35
IMG_START = '<img>'
@@ -42,19 +42,17 @@ IMAGENET_STD = (0.229, 0.224, 0.225)
42
43
class InternVLImagePixelInputs(TypedDict):
44
type: Literal["pixel_values"]
45
- data: Union[torch.Tensor, List[torch.Tensor]]
45
+ data: torch.Tensor
46
"""
47
- Shape: `(batch_size, 1 + num_patches, num_channels, height, width)`
48
-
49
- Note that `num_patches` may be different for each batch, in which case
50
- the data is passed as a list instead of a batched tensor.
47
+ Shape:
48
+ `(batch_size * num_images * (1 + num_patches), num_channels, height, width)`
49
"""
50
51
52
class InternVLImageEmbeddingInputs(TypedDict):
53
type: Literal["image_embeds"]
56
- data: Union[torch.Tensor, List[torch.Tensor]]
57
- """Shape: `(batch_size, image_feature_size, hidden_size)`
54
+ data: torch.Tensor
55
+ """Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
56
57
`hidden_size` must match the hidden size of language model backbone.
58
"""
@@ -357,7 +355,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal):
355
x = x.permute(0, 2, 1, 3).contiguous()
356
return x
357
360
- def extract_feature(self, pixel_values):
358
+ def extract_feature(self, pixel_values: torch.Tensor) -> torch.Tensor:
359
vit_embeds = self.vision_model(pixel_values=pixel_values)
360
vit_embeds = vit_embeds[:, 1:, :]
361
@@ -370,17 +368,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal):
368
vit_embeds = self.mlp1(vit_embeds)
369
return vit_embeds
370
373
- def _validate_image_sizes(self, data: torch.Tensor) -> torch.Tensor:
374
- if list(data.shape[1:]) != [2]:
375
- raise ValueError(
376
- f"The expected image sizes shape is batch dimension plus "
377
- f"{[2]}. You supplied {data.shape}.")
378
-
379
- return data
380
-
381
- def _validate_pixel_values(
382
- self, data: Union[torch.Tensor, List[torch.Tensor]]
383
- ) -> Union[torch.Tensor, List[torch.Tensor]]:
371
+ def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
372
373
h = w = self.config.vision_config.image_size
374
expected_dims = (3, h, w)
@@ -389,10 +377,11 @@ class InternVLChatModel(nn.Module, SupportsMultiModal):
377
actual_dims = tuple(d.shape)
378
379
if actual_dims != expected_dims:
392
- expected_expr = ("num_patches", *map(str, expected_dims))
380
+ expected_expr = str(expected_dims)
381
raise ValueError(
394
- "The expected shape of pixel values in each batch element "
395
- f"is {expected_expr}. You supplied {tuple(d.shape)}.")
382
+ "The expected shape of pixel values per image per batch "
383
+ f" per patch is {expected_expr}. "
384
+ f"You supplied {tuple(d.shape)}.")
385
386
for d in data:
387
_validate_shape(d)
@@ -413,12 +402,9 @@ class InternVLChatModel(nn.Module, SupportsMultiModal):
402
raise ValueError("Incorrect type of image embeddings. "
403
f"Got type: {type(image_embeds)}")
404
416
- # Flatten the B and N dimensions
417
- image_embeds = image_embeds.flatten(0, 2)
418
-
405
return InternVLImageEmbeddingInputs(
406
type="image_embeds",
421
- data=image_embeds,
407
+ data=flatten_bn(image_embeds),
408
)
409
410
self.img_context_token_id = image_token_id[0]
@@ -428,12 +414,10 @@ class InternVLChatModel(nn.Module, SupportsMultiModal):
414
raise ValueError("Incorrect type of pixel values. "
415
f"Got type: {type(pixel_values)}")
416
431
- # Flatten the B and N dimensions
432
- pixel_values = pixel_values.flatten(0, 2)
433
-
417
return InternVLImagePixelInputs(
418
type="pixel_values",
436
- data=self._validate_pixel_values(pixel_values),
419
+ data=self._validate_pixel_values(
420
+ flatten_bn(pixel_values, concat=True).flatten(0, 1)),
421
)
422
423
raise AssertionError("This line should be unreachable.")
vllm/model_executor/models/llava.py
+2
-2
@@ -30,13 +30,13 @@ from .utils import (filter_weights, init_vllm_registered_model,
30
class LlavaImagePixelInputs(TypedDict):
31
type: Literal["pixel_values"]
32
data: torch.Tensor
33
- """Shape: `(batch_size, num_channels, height, width)`"""
33
+ """Shape: `(batch_size * num_images, num_channels, height, width)`"""
34
35
36
class LlavaImageEmbeddingInputs(TypedDict):
37
type: Literal["image_embeds"]
38
data: torch.Tensor
39
- """Shape: `(batch_size, image_feature_size, hidden_size)`
39
+ """Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
40
41
`hidden_size` must match the hidden size of language model backbone.
42
"""
vllm/model_executor/models/llava_next.py
+26
-26
@@ -29,7 +29,7 @@ from .llava import LlavaMultiModalProjector
29
from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
30
dummy_seq_data_for_siglip, get_siglip_image_feature_size,
31
get_siglip_patch_grid_length, input_processor_for_siglip)
32
-from .utils import (filter_weights, init_vllm_registered_model,
32
+from .utils import (filter_weights, flatten_bn, init_vllm_registered_model,
33
merge_multimodal_embeddings)
34
35
logger = init_logger(__name__)
@@ -47,15 +47,16 @@ class LlavaNextImagePixelInputs(TypedDict):
47
type: Literal["pixel_values"]
48
data: Union[torch.Tensor, List[torch.Tensor]]
49
"""
50
- Shape: `(batch_size, 1 + num_patches, num_channels, height, width)`
50
+ Shape:
51
+ `(batch_size * num_images, 1 + num_patches, num_channels, height, width)`
52
52
- Note that `num_patches` may be different for each batch, in which case
53
- the data is passed as a list instead of a batched tensor.
53
+ Note that `num_patches` may be different per batch and image,
54
+ in which case the data is passed as a list instead of a batched tensor.
55
"""
56
57
image_sizes: NotRequired[torch.Tensor]
58
"""
58
- Shape: `(batch_size, 2)`
59
+ Shape: `(batch_size * num_images, 2)`
60
61
This should be in `(height, width)` format.
62
"""
@@ -64,7 +65,7 @@ class LlavaNextImagePixelInputs(TypedDict):
65
class LlavaNextImageEmbeddingInputs(TypedDict):
66
type: Literal["image_embeds"]
67
data: torch.Tensor
67
- """Shape: `(batch_size, image_feature_size, hidden_size)`
68
+ """Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
69
70
`hidden_size` must match the hidden size of language model backbone.
71
"""
@@ -315,10 +316,19 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal):
316
torch.empty(config.text_config.hidden_size))
317
318
def _validate_image_sizes(self, data: torch.Tensor) -> torch.Tensor:
318
- if list(data.shape[1:]) != [2]:
319
- raise ValueError(
320
- f"The expected image sizes shape is batch dimension plus "
321
- f"{[2]}. You supplied {data.shape}.")
319
+ expected_dims = (2, )
320
+
321
+ def _validate_shape(d: torch.Tensor):
322
+ actual_dims = tuple(d.shape)
323
+
324
+ if actual_dims != expected_dims:
325
+ expected_expr = str(expected_dims)
326
+ raise ValueError(
327
+ f"The expected shape of image sizes per image per batch "
328
+ f"is {expected_expr}. You supplied {tuple(d.shape)}.")
329
+
330
+ for d in data:
331
+ _validate_shape(d)
332
333
return data
334
@@ -335,7 +345,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal):
345
if actual_dims != expected_dims:
346
expected_expr = ("num_patches", *map(str, expected_dims))
347
raise ValueError(
338
- "The expected shape of pixel values in each batch element "
348
+ "The expected shape of pixel values per image per batch "
349
f"is {expected_expr}. You supplied {tuple(d.shape)}.")
350
351
for d in data:
@@ -357,22 +367,15 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal):
367
raise ValueError("Incorrect type of pixel values. "
368
f"Got type: {type(pixel_values)}")
369
360
- if not isinstance(image_sizes, torch.Tensor):
370
+ if not isinstance(image_sizes, (torch.Tensor, list)):
371
raise ValueError("Incorrect type of image sizes. "
372
f"Got type: {type(image_sizes)}")
373
364
- # Remove the N dimension until multiple images are supported.
365
- if isinstance(pixel_values, torch.Tensor):
366
- pixel_values = pixel_values.squeeze(1)
367
- else:
368
- pixel_values = [t.squeeze(0) for t in pixel_values]
369
-
370
- image_sizes = image_sizes.squeeze(1)
371
-
374
return LlavaNextImagePixelInputs(
375
type="pixel_values",
374
- data=self._validate_pixel_values(pixel_values),
375
- image_sizes=self._validate_image_sizes(image_sizes),
376
+ data=self._validate_pixel_values(flatten_bn(pixel_values)),
377
+ image_sizes=self._validate_image_sizes(
378
+ flatten_bn(image_sizes, concat=True)),
379
)
380
381
if image_embeds is not None:
@@ -380,12 +383,9 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal):
383
raise ValueError("Incorrect type of image embeds. "
384
f"Got type: {type(image_embeds)}")
385
383
- # Remove the N dimension until multiple images are supported.
384
- image_embeds = image_embeds.squeeze(1)
385
-
386
return LlavaNextImageEmbeddingInputs(
387
type="image_embeds",
388
- data=image_embeds,
388
+ data=flatten_bn(image_embeds),
389
)
390
391
raise AssertionError("This line should be unreachable.")
vllm/model_executor/models/paligemma.py
+2
-2
@@ -34,13 +34,13 @@ _KEYS_TO_MODIFY_MAPPING = {
34
class PaliGemmaImagePixelInputs(TypedDict):
35
type: Literal["pixel_values"]
36
data: torch.Tensor
37
- """Shape: (batch_size, num_channels, height, width)"""
37
+ """Shape: `(batch_size * num_images, num_channels, height, width)`"""
38
39
40
class PaliGemmaImageEmbeddingInputs(TypedDict):
41
type: Literal["image_embeds"]
42
data: torch.Tensor
43
- """Shape: `(batch_size, image_feature_size, hidden_size)`
43
+ """Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
44
45
`hidden_size` must match the hidden size of language model backbone.
46
"""
vllm/model_executor/models/phi3v.py
+27
-23
@@ -44,7 +44,7 @@ from vllm.utils import is_list_of
44
45
from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
46
from .interfaces import SupportsMultiModal
47
-from .utils import merge_multimodal_embeddings
47
+from .utils import flatten_bn, merge_multimodal_embeddings
48
49
logger = init_logger(__name__)
50
@@ -75,15 +75,16 @@ class Phi3VImagePixelInputs(TypedDict):
75
type: Literal["pixel_values"]
76
data: Union[torch.Tensor, List[torch.Tensor]]
77
"""
78
- Shape: `(batch_size, 1 + num_patches, num_channels, height, width)`
78
+ Shape:
79
+ `(batch_size * num_images, 1 + num_patches, num_channels, height, width)`
80
80
- Note that `num_patches` may be different for each batch, in which case
81
- the data is passed as a list instead of a batched tensor.
81
+ Note that `num_patches` may be different per batch and image,
82
+ in which case the data is passed as a list instead of a batched tensor.
83
"""
84
85
image_sizes: torch.Tensor
86
"""
86
- Shape: `(batch_size, 2)`
87
+ Shape: `(batch_size * num_images, 2)`
88
89
This should be in `(height, width)` format.
90
"""
@@ -92,7 +93,7 @@ class Phi3VImagePixelInputs(TypedDict):
93
class Phi3VImageEmbeddingInputs(TypedDict):
94
type: Literal["image_embeds"]
95
data: Union[torch.Tensor, List[torch.Tensor]]
95
- """Shape: `(batch_size, image_feature_size, hidden_size)`
96
+ """Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
97
98
`hidden_size` must match the hidden size of language model backbone.
99
"""
@@ -511,10 +512,19 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal):
512
self.sampler = Sampler()
513
514
def _validate_image_sizes(self, data: torch.Tensor) -> torch.Tensor:
514
- if list(data.shape[1:]) != [2]:
515
- raise ValueError(
516
- f"The expected shape of image sizes is batch dimension plus "
517
- f"{[2]}. You supplied {tuple(data.shape)}.")
515
+ expected_dims = (2, )
516
+
517
+ def _validate_shape(d: torch.Tensor):
518
+ actual_dims = tuple(d.shape)
519
+
520
+ if actual_dims != expected_dims:
521
+ expected_expr = str(expected_dims)
522
+ raise ValueError(
523
+ f"The expected shape of image sizes per image per batch "
524
+ f"is {expected_expr}. You supplied {tuple(d.shape)}.")
525
+
526
+ for d in data:
527
+ _validate_shape(d)
528
529
return data
530
@@ -531,7 +541,7 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal):
541
if actual_dims != expected_dims:
542
expected_expr = ("num_patches", *map(str, expected_dims))
543
raise ValueError(
534
- "The expected shape of pixel values in each batch element "
544
+ "The expected shape of pixel values per image per batch "
545
f"is {expected_expr}. You supplied {tuple(d.shape)}.")
546
547
for d in data:
@@ -556,30 +566,24 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal):
566
raise ValueError("Incorrect type of pixel values. "
567
f"Got type: {type(pixel_values)}")
568
559
- if not isinstance(image_sizes, torch.Tensor):
569
+ if not isinstance(image_sizes, (torch.Tensor, list)):
570
raise ValueError("Incorrect type of image sizes. "
571
f"Got type: {type(image_sizes)}")
572
563
- # Merge the B and N dimensions.
564
- if isinstance(pixel_values, torch.Tensor):
565
- pixel_values = pixel_values.flatten(0, 1)
566
- else:
567
- pixel_values = torch.cat(pixel_values)
568
-
569
- image_sizes = image_sizes.flatten(0, 1)
570
-
573
return Phi3VImagePixelInputs(
574
type="pixel_values",
573
- data=self._validate_pixel_values(pixel_values),
574
- image_sizes=self._validate_image_sizes(image_sizes))
575
+ data=self._validate_pixel_values(flatten_bn(pixel_values)),
576
+ image_sizes=self._validate_image_sizes(
577
+ flatten_bn(image_sizes, concat=True)))
578
579
if image_embeds is not None:
580
if not isinstance(image_embeds, torch.Tensor):
581
raise ValueError("Incorrect type of image embeddings. "
582
f"Got type: {type(image_embeds)}")
583
+
584
return Phi3VImageEmbeddingInputs(
585
type="image_embeds",
582
- data=image_embeds,
586
+ data=flatten_bn(image_embeds),
587
)
588
589
raise AssertionError("This line should be unreachable.")
vllm/model_executor/models/ultravox.py
+1
-1
@@ -49,7 +49,7 @@ logger = init_logger(__name__)
49
class UltravoxAudioFeatureInputs(TypedDict):
50
type: Literal["audio_features"]
51
data: Union[torch.Tensor, List[torch.Tensor]]
52
- """Shape: `(batch_size, 80, M)"""
52
+ """Shape: `(batch_size * num_audios, 80, M)"""
53
54
55
class UltravoxAudioEmbeddingInputs(TypedDict):
vllm/model_executor/models/utils.py
+42
-2
@@ -1,4 +1,5 @@
1
-from typing import Dict, Iterable, List, Optional, Protocol, Tuple
1
+from typing import (Dict, Iterable, List, Literal, Optional, Protocol, Tuple,
2
+ Union, overload)
3
4
import numpy as np
5
import torch
@@ -55,6 +56,44 @@ def init_vllm_registered_model(
56
)
57
58
59
+@overload
60
+def flatten_bn(x: torch.Tensor) -> torch.Tensor:
61
+ ...
62
+
63
+
64
+@overload
65
+def flatten_bn(x: List[torch.Tensor]) -> List[torch.Tensor]:
66
+ ...
67
+
68
+
69
+@overload
70
+def flatten_bn(
71
+ x: Union[List[torch.Tensor], torch.Tensor],
72
+ *,
73
+ concat: Literal[True],
74
+) -> torch.Tensor:
75
+ ...
76
+
77
+
78
+def flatten_bn(
79
+ x: Union[List[torch.Tensor], torch.Tensor],
80
+ *,
81
+ concat: bool = False,
82
+) -> Union[List[torch.Tensor], torch.Tensor]:
83
+ """
84
+ Flatten the ``B`` and ``N`` dimensions of batched multimodal inputs.
85
+
86
+ The input tensor should have shape ``(B, N, ...)```.
87
+ """
88
+ if isinstance(x, torch.Tensor):
89
+ return x.flatten(0, 1)
90
+
91
+ if concat:
92
+ return torch.cat(x)
93
+
94
+ return [x_n for x_b in x for x_n in x_b]
95
+
96
+
97
def _flatten_embeddings(embeddings: NestedTensors) -> torch.Tensor:
98
"""
99
Recursively concatenates NestedTensors along any heterogeneously sized
@@ -93,7 +132,8 @@ def merge_multimodal_embeddings(input_ids: torch.Tensor,
132
This updates ``inputs_embeds`` in place.
133
"""
134
mask = (input_ids == placeholder_token_id)
96
- num_expected_tokens = mask.sum()
135
+ num_expected_tokens = mask.sum().item()
136
+ assert isinstance(num_expected_tokens, int)
137
138
flattened = _flatten_embeddings(multimodal_embeddings)
139
*dims, embed_dim = flattened.shape
vllm/multimodal/base.py
+2
-2
@@ -18,7 +18,7 @@ from vllm.utils import JSONTree, is_list_of, json_map_leaves
18
19
logger = init_logger(__name__)
20
21
-NestedTensors = Union[List["NestedTensors"], torch.Tensor]
21
+NestedTensors = Union[List["NestedTensors"], List[torch.Tensor], torch.Tensor]
22
"""
23
Uses a list instead of a tensor if the dimensions of each element do not match.
24
"""
@@ -61,7 +61,7 @@ class MultiModalInputs(_MultiModalInputsBase):
61
tensors_ = cast(List[torch.Tensor], stacked)
62
if any(t.shape != tensors_[0].shape for t in tensors_):
63
# The tensors have incompatible shapes and can't be stacked.
64
- return stacked
64
+ return tensors_
65
66
return torch.stack(tensors_)
67