@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fab5f53e2d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Core][VLM] Stack multimodal tensors to represent multiple images within each prompt (#7902)

Peter Salas committed Aug 27, 2024 at 18:53 UTC fab5f53e2dbf8e076304d7f8a205370673fbcd02
15 files changed +214 -60
docs/source/dev/multimodal/multimodal_index.rst
-2
@@ -45,8 +45,6 @@ Base Classes
45
46 .. autodata:: vllm.multimodal.NestedTensors
47
48 -.. autodata:: vllm.multimodal.BatchedTensors
49 -
48 .. autodata:: vllm.multimodal.BatchedTensorInputs
49
50 .. autoclass:: vllm.multimodal.MultiModalDataBuiltins
tests/multimodal/test_base.py new
+83
@@ -0,0 +1,83 @@
1 +import torch
2 +
3 +from vllm.multimodal.base import MultiModalInputs, NestedTensors
4 +
5 +
6 +def assert_nested_tensors_equal(expected: NestedTensors,
7 + actual: NestedTensors):
8 + assert type(expected) == type(actual)
9 + if isinstance(expected, torch.Tensor):
10 + assert torch.equal(expected, actual)
11 + else:
12 + for expected_item, actual_item in zip(expected, actual):
13 + assert_nested_tensors_equal(expected_item, actual_item)
14 +
15 +
16 +def assert_multimodal_inputs_equal(expected: MultiModalInputs,
17 + actual: MultiModalInputs):
18 + assert set(expected.keys()) == set(actual.keys())
19 + for key in expected:
20 + assert_nested_tensors_equal(expected[key], actual[key])
21 +
22 +
23 +def test_multimodal_input_batch_single_tensor():
24 + t = torch.rand([1, 2])
25 + result = MultiModalInputs.batch([{"image": t}])
26 + assert_multimodal_inputs_equal(result, {"image": t.unsqueeze(0)})
27 +
28 +
29 +def test_multimodal_input_batch_multiple_tensors():
30 + a = torch.rand([1, 1, 2])
31 + b = torch.rand([1, 1, 2])
32 + c = torch.rand([1, 1, 2])
33 + result = MultiModalInputs.batch([{"image": a}, {"image": b}, {"image": c}])
34 + assert_multimodal_inputs_equal(result, {"image": torch.stack([a, b, c])})
35 +
36 +
37 +def test_multimodal_input_batch_multiple_heterogeneous_tensors():
38 + a = torch.rand([1, 2, 2])
39 + b = torch.rand([1, 3, 2])
40 + c = torch.rand([1, 4, 2])
41 + result = MultiModalInputs.batch([{"image": a}, {"image": b}, {"image": c}])
42 + assert_multimodal_inputs_equal(result, {"image": [a, b, c]})
43 +
44 +
45 +def test_multimodal_input_batch_nested_tensors():
46 + a = torch.rand([2, 3])
47 + b = torch.rand([2, 3])
48 + c = torch.rand([2, 3])
49 + result = MultiModalInputs.batch([{
50 + "image": [a]
51 + }, {
52 + "image": [b]
53 + }, {
54 + "image": [c]
55 + }])
56 + assert_multimodal_inputs_equal(result, {
57 + "image":
58 + torch.stack([a.unsqueeze(0),
59 + b.unsqueeze(0),
60 + c.unsqueeze(0)])
61 + })
62 +
63 +
64 +def test_multimodal_input_batch_heterogeneous_lists():
65 + a = torch.rand([1, 2, 3])
66 + b = torch.rand([1, 2, 3])
67 + c = torch.rand([1, 2, 3])
68 + result = MultiModalInputs.batch([{"image": [a, b]}, {"image": [c]}])
69 + assert_multimodal_inputs_equal(
70 + result,
71 + {"image": [torch.stack([a, b]), c.unsqueeze(0)]})
72 +
73 +
74 +def test_multimodal_input_batch_multiple_batchable_lists():
75 + a = torch.rand([1, 2, 3])
76 + b = torch.rand([1, 2, 3])
77 + c = torch.rand([1, 2, 3])
78 + d = torch.rand([1, 2, 3])
79 + result = MultiModalInputs.batch([{"image": [a, b]}, {"image": [c, d]}])
80 + assert_multimodal_inputs_equal(
81 + result,
82 + {"image": torch.stack([torch.stack([a, b]),
83 + torch.stack([c, d])])})
vllm/model_executor/models/blip2.py
+7
@@ -555,6 +555,9 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal):
555 raise ValueError("Incorrect type of pixel values. "
556 f"Got type: {type(pixel_values)}")
557
558 + # Remove the N dimension until multiple images are supported.
559 + pixel_values = pixel_values.squeeze(1)
560 +
561 return Blip2ImagePixelInputs(
562 type="pixel_values",
563 data=self._validate_pixel_values(pixel_values),
@@ -564,6 +567,10 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal):
567 if not isinstance(image_embeds, torch.Tensor):
568 raise ValueError("Incorrect type of image embeddings. "
569 f"Got type: {type(image_embeds)}")
570 +
571 + # Remove the N dimension until multiple images are supported.
572 + image_embeds = image_embeds.squeeze(1)
573 +
574 return Blip2ImageEmbeddingInputs(
575 type="image_embeds",
576 data=image_embeds,
vllm/model_executor/models/chameleon.py
+3
@@ -946,6 +946,9 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal):
946 raise ValueError("Incorrect type of pixel values. "
947 f"Got type: {type(pixel_values)}")
948
949 + # Remove the N dimension until multiple images are supported.
950 + pixel_values = pixel_values.squeeze(1)
951 +
952 return ChameleonImagePixelInputs(
953 type="pixel_values",
954 data=self._validate_pixel_values(pixel_values),
vllm/model_executor/models/fuyu.py
+3
@@ -249,6 +249,9 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal):
249 image_patches = kwargs.pop("image_patches", None)
250
251 if isinstance(image_patches, torch.Tensor):
252 + # Remove the N dimension until multiple images are supported.
253 + image_patches = image_patches.squeeze(1)
254 +
255 expected_feature_size = self.image_feature_size
256 if image_patches.size(-1) != expected_feature_size:
257 raise ValueError(
vllm/model_executor/models/internvl.py
+9
@@ -244,6 +244,8 @@ def input_mapper_for_internvl(ctx: InputContext, data: object):
244 min_num,
245 max_num,
246 use_thumbnail=use_thumbnail)
247 + # Add an N dimension for number of images per prompt (currently 1).
248 + data = data.unsqueeze(0)
249 model_config = ctx.model_config
250 tokenizer = cached_get_tokenizer(model_config.tokenizer,
251 trust_remote_code=True)
@@ -410,6 +412,10 @@ class InternVLChatModel(nn.Module, SupportsMultiModal):
412 if not isinstance(image_embeds, torch.Tensor):
413 raise ValueError("Incorrect type of image embeddings. "
414 f"Got type: {type(image_embeds)}")
415 +
416 + # Flatten the B and N dimensions
417 + image_embeds = image_embeds.flatten(0, 2)
418 +
419 return InternVLImageEmbeddingInputs(
420 type="image_embeds",
421 data=image_embeds,
@@ -422,6 +428,9 @@ class InternVLChatModel(nn.Module, SupportsMultiModal):
428 raise ValueError("Incorrect type of pixel values. "
429 f"Got type: {type(pixel_values)}")
430
431 + # Flatten the B and N dimensions
432 + pixel_values = pixel_values.flatten(0, 2)
433 +
434 return InternVLImagePixelInputs(
435 type="pixel_values",
436 data=self._validate_pixel_values(pixel_values),
vllm/model_executor/models/llava.py
+8
@@ -232,6 +232,10 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal):
232 if not isinstance(pixel_values, torch.Tensor):
233 raise ValueError("Incorrect type of pixel values. "
234 f"Got type: {type(pixel_values)}")
235 +
236 + # Remove the N dimension until multiple images are supported.
237 + pixel_values = pixel_values.squeeze(1)
238 +
239 return LlavaImagePixelInputs(
240 type="pixel_values",
241 data=self._validate_pixel_values(pixel_values),
@@ -241,6 +245,10 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal):
245 if not isinstance(image_embeds, torch.Tensor):
246 raise ValueError("Incorrect type of image embeddings. "
247 f"Got type: {type(image_embeds)}")
248 +
249 + # Remove the N dimension until multiple images are supported.
250 + image_embeds = image_embeds.squeeze(1)
251 +
252 return LlavaImageEmbeddingInputs(
253 type="image_embeds",
254 data=image_embeds,
vllm/model_executor/models/llava_next.py
+11
@@ -361,6 +361,14 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal):
361 raise ValueError("Incorrect type of image sizes. "
362 f"Got type: {type(image_sizes)}")
363
364 + # Remove the N dimension until multiple images are supported.
365 + if isinstance(pixel_values, torch.Tensor):
366 + pixel_values = pixel_values.squeeze(1)
367 + else:
368 + pixel_values = [t.squeeze(0) for t in pixel_values]
369 +
370 + image_sizes = image_sizes.squeeze(1)
371 +
372 return LlavaNextImagePixelInputs(
373 type="pixel_values",
374 data=self._validate_pixel_values(pixel_values),
@@ -372,6 +380,9 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal):
380 raise ValueError("Incorrect type of image embeds. "
381 f"Got type: {type(image_embeds)}")
382
383 + # Remove the N dimension until multiple images are supported.
384 + image_embeds = image_embeds.squeeze(1)
385 +
386 return LlavaNextImageEmbeddingInputs(
387 type="image_embeds",
388 data=image_embeds,
vllm/model_executor/models/minicpmv.py
+8 -3
@@ -594,9 +594,14 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal):
594
595 pixel_values_flat: List[torch.Tensor] = []
596 tgt_sizes_flat: List[torch.Tensor] = []
597 - for b in range(len(pixel_values)):
598 - pixel_values_flat += pixel_values[b]
599 - tgt_sizes_flat += tgt_sizes[b]
597 + for pixel_b, tgt_b in zip(pixel_values, tgt_sizes):
598 + if len(pixel_b) != len(tgt_b):
599 + raise ValueError("Inconsistent N lengths, found: "
600 + f"{len(pixel_b)} vs {len(tgt_b)}")
601 +
602 + for pixel_n, tgt_n in zip(pixel_b, tgt_b):
603 + pixel_values_flat += pixel_n
604 + tgt_sizes_flat += tgt_n
605
606 # NOTE: Input IDs does not contain image tokens during memory profiling,
607 # so we allow it to be empty
vllm/model_executor/models/paligemma.py
+8
@@ -185,6 +185,10 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal):
185 if not isinstance(pixel_values, torch.Tensor):
186 raise ValueError("Incorrect type of pixel values. "
187 f"Got type: {type(pixel_values)}")
188 +
189 + # Remove the N dimension until multiple images are supported.
190 + pixel_values = pixel_values.squeeze(1)
191 +
192 return PaliGemmaImagePixelInputs(
193 type="pixel_values",
194 data=self._validate_pixel_values(pixel_values),
@@ -194,6 +198,10 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal):
198 if not isinstance(image_embeds, torch.Tensor):
199 raise ValueError("Incorrect type of image embeddings. "
200 f"Got type: {type(image_embeds)}")
201 +
202 + # Remove the N dimension until multiple images are supported.
203 + image_embeds = image_embeds.squeeze(1)
204 +
205 return PaliGemmaImageEmbeddingInputs(
206 type="image_embeds",
207 data=image_embeds,
vllm/model_executor/models/phi3v.py
+8
@@ -560,6 +560,14 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal):
560 raise ValueError("Incorrect type of image sizes. "
561 f"Got type: {type(image_sizes)}")
562
563 + # Merge the B and N dimensions.
564 + if isinstance(pixel_values, torch.Tensor):
565 + pixel_values = pixel_values.flatten(0, 1)
566 + else:
567 + pixel_values = torch.cat(pixel_values)
568 +
569 + image_sizes = image_sizes.flatten(0, 1)
570 +
571 return Phi3VImagePixelInputs(
572 type="pixel_values",
573 data=self._validate_pixel_values(pixel_values),
vllm/model_executor/models/ultravox.py
+9
@@ -333,6 +333,12 @@ class UltravoxModel(nn.Module, SupportsMultiModal):
333 raise ValueError("Incorrect type of audio features. "
334 f"Got type: {type(audio_features)}")
335
336 + # Remove the N dimension until multiple audios are supported.
337 + if isinstance(audio_features, torch.Tensor):
338 + audio_features = audio_features.squeeze(1)
339 + else:
340 + audio_features = [t.squeeze(0) for t in audio_features]
341 +
342 return UltravoxAudioFeatureInputs(type="audio_features",
343 data=audio_features)
344
@@ -341,6 +347,9 @@ class UltravoxModel(nn.Module, SupportsMultiModal):
347 raise ValueError("Incorrect type of audio embeds. "
348 f"Got type: {type(audio_embeds)}")
349
350 + # Remove the N dimension until multiple audios are supported.
351 + audio_embeds = audio_embeds.squeeze(1)
352 +
353 return UltravoxAudioEmbeddingInputs(type="audio_embeds",
354 data=audio_embeds)
355
vllm/model_executor/models/utils.py
+37 -23
@@ -1,5 +1,6 @@
1 from typing import Dict, Iterable, List, Optional, Protocol, Tuple
2
3 +import numpy as np
4 import torch
5 import torch.nn as nn
6 from torch.func import functional_call
@@ -10,7 +11,7 @@ from vllm.config import (CacheConfig, LoRAConfig, MultiModalConfig,
11 from vllm.model_executor.layers.quantization import QuantizationConfig
12 from vllm.model_executor.model_loader.loader import build_model
13 from vllm.model_executor.models import ModelRegistry
13 -from vllm.multimodal import BatchedTensors
14 +from vllm.multimodal.base import NestedTensors
15 from vllm.utils import is_pin_memory_available
16
17
@@ -54,9 +55,34 @@ def init_vllm_registered_model(
55 )
56
57
58 +def _flatten_embeddings(embeddings: NestedTensors) -> torch.Tensor:
59 + """
60 + Recursively concatenates NestedTensors along any heterogeneously sized
61 + dimensions.
62 + """
63 +
64 + if isinstance(embeddings, torch.Tensor):
65 + return embeddings
66 +
67 + return torch.cat(tuple(_flatten_embeddings(t) for t in embeddings))
68 +
69 +
70 +def _embedding_count_expression(embeddings: NestedTensors) -> str:
71 + """
72 + Constructs a debugging representation of the number of embeddings in the
73 + NestedTensors.
74 + """
75 +
76 + if isinstance(embeddings, torch.Tensor):
77 + return " x ".join([str(dim) for dim in embeddings.shape[:-1]])
78 +
79 + return " + ".join(
80 + _embedding_count_expression(inner) for inner in embeddings)
81 +
82 +
83 def merge_multimodal_embeddings(input_ids: torch.Tensor,
84 inputs_embeds: torch.Tensor,
59 - multimodal_embeddings: BatchedTensors,
85 + multimodal_embeddings: NestedTensors,
86 placeholder_token_id: int) -> torch.Tensor:
87 """
88 Merge ``multimodal_embeddings`` into ``inputs_embeds`` by overwriting the
@@ -69,28 +95,16 @@ def merge_multimodal_embeddings(input_ids: torch.Tensor,
95 mask = (input_ids == placeholder_token_id)
96 num_expected_tokens = mask.sum()
97
72 - if isinstance(multimodal_embeddings, torch.Tensor):
73 - batch_size, batch_tokens, *_, embed_dim = multimodal_embeddings.shape
74 - total_tokens = batch_size * batch_tokens
75 - if num_expected_tokens != total_tokens:
76 - expr = f"{batch_size} x {batch_tokens}"
77 - raise ValueError(
78 - f"Attempted to assign {expr} = {total_tokens} "
79 - f"multimodal tokens to {num_expected_tokens} placeholders")
80 -
81 - inputs_embeds[mask] = multimodal_embeddings.view(
82 - total_tokens, embed_dim)
83 - else:
84 - size_per_batch = [t.shape[0] for t in multimodal_embeddings]
85 - total_tokens = sum(size_per_batch)
86 - if num_expected_tokens != total_tokens:
87 - expr = ' + '.join(map(str, size_per_batch))
88 - raise ValueError(
89 - f"Attempted to assign {expr} = {total_tokens} "
90 - f"multimodal tokens to {num_expected_tokens} placeholders")
91 -
92 - inputs_embeds[mask] = torch.cat(multimodal_embeddings)
98 + flattened = _flatten_embeddings(multimodal_embeddings)
99 + *dims, embed_dim = flattened.shape
100 + num_multimodal_embeddings = np.prod(dims)
101 + if num_multimodal_embeddings != num_expected_tokens:
102 + expr = _embedding_count_expression(multimodal_embeddings)
103 + raise ValueError(
104 + f"Attempted to assign {expr} = {num_multimodal_embeddings} "
105 + f"multimodal tokens to {num_expected_tokens} placeholders")
106
107 + inputs_embeds[mask] = flattened.view(num_expected_tokens, embed_dim)
108 return inputs_embeds
109
110
vllm/multimodal/__init__.py
+1 -2
@@ -1,4 +1,4 @@
1 -from .base import (BatchedTensorInputs, BatchedTensors, MultiModalDataBuiltins,
1 +from .base import (BatchedTensorInputs, MultiModalDataBuiltins,
2 MultiModalDataDict, MultiModalInputs, MultiModalPlugin,
3 NestedTensors)
4 from .registry import MultiModalRegistry
@@ -14,7 +14,6 @@ See also:
14
15 __all__ = [
16 "BatchedTensorInputs",
17 - "BatchedTensors",
17 "MultiModalDataBuiltins",
18 "MultiModalDataDict",
19 "MultiModalInputs",
vllm/multimodal/base.py
+19 -30
@@ -1,9 +1,8 @@
1 import sys
2 from abc import ABC, abstractmethod
3 from collections import UserDict, defaultdict
4 -from typing import Callable, Dict, List, Mapping, Optional
5 -from typing import Sequence as GenericSequence
6 -from typing import Tuple, Type, TypedDict, TypeVar, Union, cast, final
4 +from typing import (Callable, Dict, List, Mapping, Optional, Tuple, Type,
5 + TypedDict, TypeVar, Union, cast, final)
6
7 import numpy as np
8 import torch
@@ -15,23 +14,16 @@ from typing_extensions import TypeAlias
14 from vllm.config import ModelConfig
15 from vllm.inputs import InputContext
16 from vllm.logger import init_logger
18 -from vllm.utils import JSONTree, json_map_leaves
17 +from vllm.utils import json_map_leaves
18
19 logger = init_logger(__name__)
20
22 -NestedTensors = Union[GenericSequence[torch.Tensor], torch.Tensor]
21 +NestedTensors = Union[List["NestedTensors"], torch.Tensor]
22 """
24 -Use a list instead of a tensor if the dimensions of each element do not match.
25 -Currently only supports up to singly nested list of tensors.
23 +Uses a list instead of a tensor if the dimensions of each element do not match.
24 """
25
28 -BatchedTensors: TypeAlias = JSONTree[torch.Tensor]
29 -"""
30 -A nested JSON structure of tensors which have been batched via
31 -:meth:`MultiModalInputs.batch`.
32 -"""
33 -
34 -BatchedTensorInputs: TypeAlias = Dict[str, JSONTree[torch.Tensor]]
26 +BatchedTensorInputs: TypeAlias = Dict[str, NestedTensors]
27 """
28 A dictionary containing nested tensors which have been batched via
29 :meth:`MultiModalInputs.batch`.
@@ -54,26 +46,23 @@ class MultiModalInputs(_MultiModalInputsBase):
46 """
47
48 @staticmethod
57 - def _try_concat(tensors: List[NestedTensors]) -> BatchedTensors:
49 + def _try_stack(nested_tensors: NestedTensors) -> NestedTensors:
50 """
59 - If each input tensor in the batch has the same shape, return a single
60 - batched tensor; otherwise, return a list of :class:`NestedTensors` with
61 - one element per item in the batch.
51 + Recursively stacks lists of tensors when they all have the same shape.
52 """
63 - # may be list rather than tensors
64 - if isinstance(tensors[0], list):
65 - return [[t for t in tensor[0]]
66 - for tensor in cast(List[List[torch.Tensor]], tensors)]
67 -
68 - tensors_ = cast(List[torch.Tensor], tensors)
53 + if isinstance(nested_tensors, torch.Tensor):
54 + return nested_tensors
55
70 - unbatched_shape = tensors_[0].shape[1:]
56 + stacked = [MultiModalInputs._try_stack(t) for t in nested_tensors]
57 + if any(isinstance(t, list) for t in stacked):
58 + return stacked
59
72 - for tensor in tensors_:
73 - if tensor.shape[1:] != unbatched_shape:
74 - return [tensor.squeeze(0) for tensor in tensors_]
60 + tensors_ = cast(List[torch.Tensor], stacked)
61 + if any(t.shape != tensors_[0].shape for t in tensors_):
62 + # The tensors have incompatible shapes and can't be stacked.
63 + return tensors_
64
76 - return torch.cat(tensors_, dim=0)
65 + return torch.stack(tensors_)
66
67 @staticmethod
68 def batch(inputs_list: List["MultiModalInputs"]) -> BatchedTensorInputs:
@@ -102,7 +91,7 @@ class MultiModalInputs(_MultiModalInputsBase):
91 item_lists[k].append(v)
92
93 return {
105 - k: MultiModalInputs._try_concat(item_list)
94 + k: MultiModalInputs._try_stack(item_list)
95 for k, item_list in item_lists.items()
96 }
97