A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Core][VLM] Stack multimodal tensors to represent multiple images within each prompt (#7902)
Peter Salas committed
Aug 27, 2024 at 18:53 UTC
fab5f53e2dbf8e076304d7f8a205370673fbcd02
15 files changed
+214
-60
docs/source/dev/multimodal/multimodal_index.rst
-2
@@ -45,8 +45,6 @@ Base Classes
45
46
.. autodata:: vllm.multimodal.NestedTensors
47
48
-.. autodata:: vllm.multimodal.BatchedTensors
49
-
48
.. autodata:: vllm.multimodal.BatchedTensorInputs
49
50
.. autoclass:: vllm.multimodal.MultiModalDataBuiltins
tests/multimodal/test_base.py
new
+83
@@ -0,0 +1,83 @@
1
+import torch
2
+
3
+from vllm.multimodal.base import MultiModalInputs, NestedTensors
4
+
5
+
6
+def assert_nested_tensors_equal(expected: NestedTensors,
7
+ actual: NestedTensors):
8
+ assert type(expected) == type(actual)
9
+ if isinstance(expected, torch.Tensor):
10
+ assert torch.equal(expected, actual)
11
+ else:
12
+ for expected_item, actual_item in zip(expected, actual):
13
+ assert_nested_tensors_equal(expected_item, actual_item)
14
+
15
+
16
+def assert_multimodal_inputs_equal(expected: MultiModalInputs,
17
+ actual: MultiModalInputs):
18
+ assert set(expected.keys()) == set(actual.keys())
19
+ for key in expected:
20
+ assert_nested_tensors_equal(expected[key], actual[key])
21
+
22
+
23
+def test_multimodal_input_batch_single_tensor():
24
+ t = torch.rand([1, 2])
25
+ result = MultiModalInputs.batch([{"image": t}])
26
+ assert_multimodal_inputs_equal(result, {"image": t.unsqueeze(0)})
27
+
28
+
29
+def test_multimodal_input_batch_multiple_tensors():
30
+ a = torch.rand([1, 1, 2])
31
+ b = torch.rand([1, 1, 2])
32
+ c = torch.rand([1, 1, 2])
33
+ result = MultiModalInputs.batch([{"image": a}, {"image": b}, {"image": c}])
34
+ assert_multimodal_inputs_equal(result, {"image": torch.stack([a, b, c])})
35
+
36
+
37
+def test_multimodal_input_batch_multiple_heterogeneous_tensors():
38
+ a = torch.rand([1, 2, 2])
39
+ b = torch.rand([1, 3, 2])
40
+ c = torch.rand([1, 4, 2])
41
+ result = MultiModalInputs.batch([{"image": a}, {"image": b}, {"image": c}])
42
+ assert_multimodal_inputs_equal(result, {"image": [a, b, c]})
43
+
44
+
45
+def test_multimodal_input_batch_nested_tensors():
46
+ a = torch.rand([2, 3])
47
+ b = torch.rand([2, 3])
48
+ c = torch.rand([2, 3])
49
+ result = MultiModalInputs.batch([{
50
+ "image": [a]
51
+ }, {
52
+ "image": [b]
53
+ }, {
54
+ "image": [c]
55
+ }])
56
+ assert_multimodal_inputs_equal(result, {
57
+ "image":
58
+ torch.stack([a.unsqueeze(0),
59
+ b.unsqueeze(0),
60
+ c.unsqueeze(0)])
61
+ })
62
+
63
+
64
+def test_multimodal_input_batch_heterogeneous_lists():
65
+ a = torch.rand([1, 2, 3])
66
+ b = torch.rand([1, 2, 3])
67
+ c = torch.rand([1, 2, 3])
68
+ result = MultiModalInputs.batch([{"image": [a, b]}, {"image": [c]}])
69
+ assert_multimodal_inputs_equal(
70
+ result,
71
+ {"image": [torch.stack([a, b]), c.unsqueeze(0)]})
72
+
73
+
74
+def test_multimodal_input_batch_multiple_batchable_lists():
75
+ a = torch.rand([1, 2, 3])
76
+ b = torch.rand([1, 2, 3])
77
+ c = torch.rand([1, 2, 3])
78
+ d = torch.rand([1, 2, 3])
79
+ result = MultiModalInputs.batch([{"image": [a, b]}, {"image": [c, d]}])
80
+ assert_multimodal_inputs_equal(
81
+ result,
82
+ {"image": torch.stack([torch.stack([a, b]),
83
+ torch.stack([c, d])])})
vllm/model_executor/models/blip2.py
+7
@@ -555,6 +555,9 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal):
555
raise ValueError("Incorrect type of pixel values. "
556
f"Got type: {type(pixel_values)}")
557
558
+ # Remove the N dimension until multiple images are supported.
559
+ pixel_values = pixel_values.squeeze(1)
560
+
561
return Blip2ImagePixelInputs(
562
type="pixel_values",
563
data=self._validate_pixel_values(pixel_values),
@@ -564,6 +567,10 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal):
567
if not isinstance(image_embeds, torch.Tensor):
568
raise ValueError("Incorrect type of image embeddings. "
569
f"Got type: {type(image_embeds)}")
570
+
571
+ # Remove the N dimension until multiple images are supported.
572
+ image_embeds = image_embeds.squeeze(1)
573
+
574
return Blip2ImageEmbeddingInputs(
575
type="image_embeds",
576
data=image_embeds,
vllm/model_executor/models/chameleon.py
+3
@@ -946,6 +946,9 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal):
946
raise ValueError("Incorrect type of pixel values. "
947
f"Got type: {type(pixel_values)}")
948
949
+ # Remove the N dimension until multiple images are supported.
950
+ pixel_values = pixel_values.squeeze(1)
951
+
952
return ChameleonImagePixelInputs(
953
type="pixel_values",
954
data=self._validate_pixel_values(pixel_values),
vllm/model_executor/models/fuyu.py
+3
@@ -249,6 +249,9 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal):
249
image_patches = kwargs.pop("image_patches", None)
250
251
if isinstance(image_patches, torch.Tensor):
252
+ # Remove the N dimension until multiple images are supported.
253
+ image_patches = image_patches.squeeze(1)
254
+
255
expected_feature_size = self.image_feature_size
256
if image_patches.size(-1) != expected_feature_size:
257
raise ValueError(
vllm/model_executor/models/internvl.py
+9
@@ -244,6 +244,8 @@ def input_mapper_for_internvl(ctx: InputContext, data: object):
244
min_num,
245
max_num,
246
use_thumbnail=use_thumbnail)
247
+ # Add an N dimension for number of images per prompt (currently 1).
248
+ data = data.unsqueeze(0)
249
model_config = ctx.model_config
250
tokenizer = cached_get_tokenizer(model_config.tokenizer,
251
trust_remote_code=True)
@@ -410,6 +412,10 @@ class InternVLChatModel(nn.Module, SupportsMultiModal):
412
if not isinstance(image_embeds, torch.Tensor):
413
raise ValueError("Incorrect type of image embeddings. "
414
f"Got type: {type(image_embeds)}")
415
+
416
+ # Flatten the B and N dimensions
417
+ image_embeds = image_embeds.flatten(0, 2)
418
+
419
return InternVLImageEmbeddingInputs(
420
type="image_embeds",
421
data=image_embeds,
@@ -422,6 +428,9 @@ class InternVLChatModel(nn.Module, SupportsMultiModal):
428
raise ValueError("Incorrect type of pixel values. "
429
f"Got type: {type(pixel_values)}")
430
431
+ # Flatten the B and N dimensions
432
+ pixel_values = pixel_values.flatten(0, 2)
433
+
434
return InternVLImagePixelInputs(
435
type="pixel_values",
436
data=self._validate_pixel_values(pixel_values),
vllm/model_executor/models/llava.py
+8
@@ -232,6 +232,10 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal):
232
if not isinstance(pixel_values, torch.Tensor):
233
raise ValueError("Incorrect type of pixel values. "
234
f"Got type: {type(pixel_values)}")
235
+
236
+ # Remove the N dimension until multiple images are supported.
237
+ pixel_values = pixel_values.squeeze(1)
238
+
239
return LlavaImagePixelInputs(
240
type="pixel_values",
241
data=self._validate_pixel_values(pixel_values),
@@ -241,6 +245,10 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal):
245
if not isinstance(image_embeds, torch.Tensor):
246
raise ValueError("Incorrect type of image embeddings. "
247
f"Got type: {type(image_embeds)}")
248
+
249
+ # Remove the N dimension until multiple images are supported.
250
+ image_embeds = image_embeds.squeeze(1)
251
+
252
return LlavaImageEmbeddingInputs(
253
type="image_embeds",
254
data=image_embeds,
vllm/model_executor/models/llava_next.py
+11
@@ -361,6 +361,14 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal):
361
raise ValueError("Incorrect type of image sizes. "
362
f"Got type: {type(image_sizes)}")
363
364
+ # Remove the N dimension until multiple images are supported.
365
+ if isinstance(pixel_values, torch.Tensor):
366
+ pixel_values = pixel_values.squeeze(1)
367
+ else:
368
+ pixel_values = [t.squeeze(0) for t in pixel_values]
369
+
370
+ image_sizes = image_sizes.squeeze(1)
371
+
372
return LlavaNextImagePixelInputs(
373
type="pixel_values",
374
data=self._validate_pixel_values(pixel_values),
@@ -372,6 +380,9 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal):
380
raise ValueError("Incorrect type of image embeds. "
381
f"Got type: {type(image_embeds)}")
382
383
+ # Remove the N dimension until multiple images are supported.
384
+ image_embeds = image_embeds.squeeze(1)
385
+
386
return LlavaNextImageEmbeddingInputs(
387
type="image_embeds",
388
data=image_embeds,
vllm/model_executor/models/minicpmv.py
+8
-3
@@ -594,9 +594,14 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal):
594
595
pixel_values_flat: List[torch.Tensor] = []
596
tgt_sizes_flat: List[torch.Tensor] = []
597
- for b in range(len(pixel_values)):
598
- pixel_values_flat += pixel_values[b]
599
- tgt_sizes_flat += tgt_sizes[b]
597
+ for pixel_b, tgt_b in zip(pixel_values, tgt_sizes):
598
+ if len(pixel_b) != len(tgt_b):
599
+ raise ValueError("Inconsistent N lengths, found: "
600
+ f"{len(pixel_b)} vs {len(tgt_b)}")
601
+
602
+ for pixel_n, tgt_n in zip(pixel_b, tgt_b):
603
+ pixel_values_flat += pixel_n
604
+ tgt_sizes_flat += tgt_n
605
606
# NOTE: Input IDs does not contain image tokens during memory profiling,
607
# so we allow it to be empty
vllm/model_executor/models/paligemma.py
+8
@@ -185,6 +185,10 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal):
185
if not isinstance(pixel_values, torch.Tensor):
186
raise ValueError("Incorrect type of pixel values. "
187
f"Got type: {type(pixel_values)}")
188
+
189
+ # Remove the N dimension until multiple images are supported.
190
+ pixel_values = pixel_values.squeeze(1)
191
+
192
return PaliGemmaImagePixelInputs(
193
type="pixel_values",
194
data=self._validate_pixel_values(pixel_values),
@@ -194,6 +198,10 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal):
198
if not isinstance(image_embeds, torch.Tensor):
199
raise ValueError("Incorrect type of image embeddings. "
200
f"Got type: {type(image_embeds)}")
201
+
202
+ # Remove the N dimension until multiple images are supported.
203
+ image_embeds = image_embeds.squeeze(1)
204
+
205
return PaliGemmaImageEmbeddingInputs(
206
type="image_embeds",
207
data=image_embeds,
vllm/model_executor/models/phi3v.py
+8
@@ -560,6 +560,14 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal):
560
raise ValueError("Incorrect type of image sizes. "
561
f"Got type: {type(image_sizes)}")
562
563
+ # Merge the B and N dimensions.
564
+ if isinstance(pixel_values, torch.Tensor):
565
+ pixel_values = pixel_values.flatten(0, 1)
566
+ else:
567
+ pixel_values = torch.cat(pixel_values)
568
+
569
+ image_sizes = image_sizes.flatten(0, 1)
570
+
571
return Phi3VImagePixelInputs(
572
type="pixel_values",
573
data=self._validate_pixel_values(pixel_values),
vllm/model_executor/models/ultravox.py
+9
@@ -333,6 +333,12 @@ class UltravoxModel(nn.Module, SupportsMultiModal):
333
raise ValueError("Incorrect type of audio features. "
334
f"Got type: {type(audio_features)}")
335
336
+ # Remove the N dimension until multiple audios are supported.
337
+ if isinstance(audio_features, torch.Tensor):
338
+ audio_features = audio_features.squeeze(1)
339
+ else:
340
+ audio_features = [t.squeeze(0) for t in audio_features]
341
+
342
return UltravoxAudioFeatureInputs(type="audio_features",
343
data=audio_features)
344
@@ -341,6 +347,9 @@ class UltravoxModel(nn.Module, SupportsMultiModal):
347
raise ValueError("Incorrect type of audio embeds. "
348
f"Got type: {type(audio_embeds)}")
349
350
+ # Remove the N dimension until multiple audios are supported.
351
+ audio_embeds = audio_embeds.squeeze(1)
352
+
353
return UltravoxAudioEmbeddingInputs(type="audio_embeds",
354
data=audio_embeds)
355
vllm/model_executor/models/utils.py
+37
-23
@@ -1,5 +1,6 @@
1
from typing import Dict, Iterable, List, Optional, Protocol, Tuple
2
3
+import numpy as np
4
import torch
5
import torch.nn as nn
6
from torch.func import functional_call
@@ -10,7 +11,7 @@ from vllm.config import (CacheConfig, LoRAConfig, MultiModalConfig,
11
from vllm.model_executor.layers.quantization import QuantizationConfig
12
from vllm.model_executor.model_loader.loader import build_model
13
from vllm.model_executor.models import ModelRegistry
13
-from vllm.multimodal import BatchedTensors
14
+from vllm.multimodal.base import NestedTensors
15
from vllm.utils import is_pin_memory_available
16
17
@@ -54,9 +55,34 @@ def init_vllm_registered_model(
55
)
56
57
58
+def _flatten_embeddings(embeddings: NestedTensors) -> torch.Tensor:
59
+ """
60
+ Recursively concatenates NestedTensors along any heterogeneously sized
61
+ dimensions.
62
+ """
63
+
64
+ if isinstance(embeddings, torch.Tensor):
65
+ return embeddings
66
+
67
+ return torch.cat(tuple(_flatten_embeddings(t) for t in embeddings))
68
+
69
+
70
+def _embedding_count_expression(embeddings: NestedTensors) -> str:
71
+ """
72
+ Constructs a debugging representation of the number of embeddings in the
73
+ NestedTensors.
74
+ """
75
+
76
+ if isinstance(embeddings, torch.Tensor):
77
+ return " x ".join([str(dim) for dim in embeddings.shape[:-1]])
78
+
79
+ return " + ".join(
80
+ _embedding_count_expression(inner) for inner in embeddings)
81
+
82
+
83
def merge_multimodal_embeddings(input_ids: torch.Tensor,
84
inputs_embeds: torch.Tensor,
59
- multimodal_embeddings: BatchedTensors,
85
+ multimodal_embeddings: NestedTensors,
86
placeholder_token_id: int) -> torch.Tensor:
87
"""
88
Merge ``multimodal_embeddings`` into ``inputs_embeds`` by overwriting the
@@ -69,28 +95,16 @@ def merge_multimodal_embeddings(input_ids: torch.Tensor,
95
mask = (input_ids == placeholder_token_id)
96
num_expected_tokens = mask.sum()
97
72
- if isinstance(multimodal_embeddings, torch.Tensor):
73
- batch_size, batch_tokens, *_, embed_dim = multimodal_embeddings.shape
74
- total_tokens = batch_size * batch_tokens
75
- if num_expected_tokens != total_tokens:
76
- expr = f"{batch_size} x {batch_tokens}"
77
- raise ValueError(
78
- f"Attempted to assign {expr} = {total_tokens} "
79
- f"multimodal tokens to {num_expected_tokens} placeholders")
80
-
81
- inputs_embeds[mask] = multimodal_embeddings.view(
82
- total_tokens, embed_dim)
83
- else:
84
- size_per_batch = [t.shape[0] for t in multimodal_embeddings]
85
- total_tokens = sum(size_per_batch)
86
- if num_expected_tokens != total_tokens:
87
- expr = ' + '.join(map(str, size_per_batch))
88
- raise ValueError(
89
- f"Attempted to assign {expr} = {total_tokens} "
90
- f"multimodal tokens to {num_expected_tokens} placeholders")
91
-
92
- inputs_embeds[mask] = torch.cat(multimodal_embeddings)
98
+ flattened = _flatten_embeddings(multimodal_embeddings)
99
+ *dims, embed_dim = flattened.shape
100
+ num_multimodal_embeddings = np.prod(dims)
101
+ if num_multimodal_embeddings != num_expected_tokens:
102
+ expr = _embedding_count_expression(multimodal_embeddings)
103
+ raise ValueError(
104
+ f"Attempted to assign {expr} = {num_multimodal_embeddings} "
105
+ f"multimodal tokens to {num_expected_tokens} placeholders")
106
107
+ inputs_embeds[mask] = flattened.view(num_expected_tokens, embed_dim)
108
return inputs_embeds
109
110
vllm/multimodal/__init__.py
+1
-2
@@ -1,4 +1,4 @@
1
-from .base import (BatchedTensorInputs, BatchedTensors, MultiModalDataBuiltins,
1
+from .base import (BatchedTensorInputs, MultiModalDataBuiltins,
2
MultiModalDataDict, MultiModalInputs, MultiModalPlugin,
3
NestedTensors)
4
from .registry import MultiModalRegistry
@@ -14,7 +14,6 @@ See also:
14
15
__all__ = [
16
"BatchedTensorInputs",
17
- "BatchedTensors",
17
"MultiModalDataBuiltins",
18
"MultiModalDataDict",
19
"MultiModalInputs",
vllm/multimodal/base.py
+19
-30
@@ -1,9 +1,8 @@
1
import sys
2
from abc import ABC, abstractmethod
3
from collections import UserDict, defaultdict
4
-from typing import Callable, Dict, List, Mapping, Optional
5
-from typing import Sequence as GenericSequence
6
-from typing import Tuple, Type, TypedDict, TypeVar, Union, cast, final
4
+from typing import (Callable, Dict, List, Mapping, Optional, Tuple, Type,
5
+ TypedDict, TypeVar, Union, cast, final)
6
7
import numpy as np
8
import torch
@@ -15,23 +14,16 @@ from typing_extensions import TypeAlias
14
from vllm.config import ModelConfig
15
from vllm.inputs import InputContext
16
from vllm.logger import init_logger
18
-from vllm.utils import JSONTree, json_map_leaves
17
+from vllm.utils import json_map_leaves
18
19
logger = init_logger(__name__)
20
22
-NestedTensors = Union[GenericSequence[torch.Tensor], torch.Tensor]
21
+NestedTensors = Union[List["NestedTensors"], torch.Tensor]
22
"""
24
-Use a list instead of a tensor if the dimensions of each element do not match.
25
-Currently only supports up to singly nested list of tensors.
23
+Uses a list instead of a tensor if the dimensions of each element do not match.
24
"""
25
28
-BatchedTensors: TypeAlias = JSONTree[torch.Tensor]
29
-"""
30
-A nested JSON structure of tensors which have been batched via
31
-:meth:`MultiModalInputs.batch`.
32
-"""
33
-
34
-BatchedTensorInputs: TypeAlias = Dict[str, JSONTree[torch.Tensor]]
26
+BatchedTensorInputs: TypeAlias = Dict[str, NestedTensors]
27
"""
28
A dictionary containing nested tensors which have been batched via
29
:meth:`MultiModalInputs.batch`.
@@ -54,26 +46,23 @@ class MultiModalInputs(_MultiModalInputsBase):
46
"""
47
48
@staticmethod
57
- def _try_concat(tensors: List[NestedTensors]) -> BatchedTensors:
49
+ def _try_stack(nested_tensors: NestedTensors) -> NestedTensors:
50
"""
59
- If each input tensor in the batch has the same shape, return a single
60
- batched tensor; otherwise, return a list of :class:`NestedTensors` with
61
- one element per item in the batch.
51
+ Recursively stacks lists of tensors when they all have the same shape.
52
"""
63
- # may be list rather than tensors
64
- if isinstance(tensors[0], list):
65
- return [[t for t in tensor[0]]
66
- for tensor in cast(List[List[torch.Tensor]], tensors)]
67
-
68
- tensors_ = cast(List[torch.Tensor], tensors)
53
+ if isinstance(nested_tensors, torch.Tensor):
54
+ return nested_tensors
55
70
- unbatched_shape = tensors_[0].shape[1:]
56
+ stacked = [MultiModalInputs._try_stack(t) for t in nested_tensors]
57
+ if any(isinstance(t, list) for t in stacked):
58
+ return stacked
59
72
- for tensor in tensors_:
73
- if tensor.shape[1:] != unbatched_shape:
74
- return [tensor.squeeze(0) for tensor in tensors_]
60
+ tensors_ = cast(List[torch.Tensor], stacked)
61
+ if any(t.shape != tensors_[0].shape for t in tensors_):
62
+ # The tensors have incompatible shapes and can't be stacked.
63
+ return tensors_
64
76
- return torch.cat(tensors_, dim=0)
65
+ return torch.stack(tensors_)
66
67
@staticmethod
68
def batch(inputs_list: List["MultiModalInputs"]) -> BatchedTensorInputs:
@@ -102,7 +91,7 @@ class MultiModalInputs(_MultiModalInputsBase):
91
item_lists[k].append(v)
92
93
return {
105
- k: MultiModalInputs._try_concat(item_list)
94
+ k: MultiModalInputs._try_stack(item_list)
95
for k, item_list in item_lists.items()
96
}
97