A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[VLM][Core] Fix exceptions on ragged NestedTensors (#7974)
Peter Salas committed
Aug 28, 2024 at 20:24 UTC
74d5543ec589daaa4ac042d65d52dccf26ee3f2c
3 files changed
+21
-11
tests/multimodal/test_base.py
+12
@@ -81,3 +81,15 @@ def test_multimodal_input_batch_multiple_batchable_lists():
81
result,
82
{"image": torch.stack([torch.stack([a, b]),
83
torch.stack([c, d])])})
84
+
85
+
86
+def test_multimodal_input_batch_mixed_stacking_depths():
87
+ a = torch.rand([1, 2, 3])
88
+ b = torch.rand([1, 3, 3])
89
+ c = torch.rand([1, 4, 3])
90
+
91
+ result = MultiModalInputs.batch([{"image": [a, b]}, {"image": [c]}])
92
+ assert_multimodal_inputs_equal(result, {"image": [[a, b], c.unsqueeze(0)]})
93
+
94
+ result = MultiModalInputs.batch([{"image": [a]}, {"image": [b, c]}])
95
+ assert_multimodal_inputs_equal(result, {"image": [a.unsqueeze(0), [b, c]]})
vllm/model_executor/models/utils.py
+7
-9
@@ -1,7 +1,6 @@
1
from typing import (Dict, Iterable, List, Literal, Optional, Protocol, Tuple,
2
Union, overload)
3
4
-import numpy as np
4
import torch
5
import torch.nn as nn
6
from torch.func import functional_call
@@ -96,12 +95,13 @@ def flatten_bn(
95
96
def _flatten_embeddings(embeddings: NestedTensors) -> torch.Tensor:
97
"""
99
- Recursively concatenates NestedTensors along any heterogeneously sized
100
- dimensions.
98
+ Recursively flattens and concatenates NestedTensors on all but the last
99
+ dimension.
100
"""
101
102
if isinstance(embeddings, torch.Tensor):
104
- return embeddings
103
+ # Flatten all but the last dimension.
104
+ return embeddings.flatten(0, -2)
105
106
return torch.cat(tuple(_flatten_embeddings(t) for t in embeddings))
107
@@ -136,15 +136,13 @@ def merge_multimodal_embeddings(input_ids: torch.Tensor,
136
assert isinstance(num_expected_tokens, int)
137
138
flattened = _flatten_embeddings(multimodal_embeddings)
139
- *dims, embed_dim = flattened.shape
140
- num_multimodal_embeddings = np.prod(dims)
141
- if num_multimodal_embeddings != num_expected_tokens:
139
+ if flattened.shape[0] != num_expected_tokens:
140
expr = _embedding_count_expression(multimodal_embeddings)
141
raise ValueError(
144
- f"Attempted to assign {expr} = {num_multimodal_embeddings} "
142
+ f"Attempted to assign {expr} = {flattened.shape[0]} "
143
f"multimodal tokens to {num_expected_tokens} placeholders")
144
147
- inputs_embeds[mask] = flattened.view(num_expected_tokens, embed_dim)
145
+ inputs_embeds[mask] = flattened
146
return inputs_embeds
147
148
vllm/multimodal/base.py
+2
-2
@@ -54,8 +54,8 @@ class MultiModalInputs(_MultiModalInputsBase):
54
return nested_tensors
55
56
stacked = [MultiModalInputs._try_stack(t) for t in nested_tensors]
57
- if is_list_of(stacked, list):
58
- # Do not stack nested lists
57
+ if not is_list_of(stacked, torch.Tensor, check="all"):
58
+ # Only tensors (not lists) can be stacked.
59
return stacked
60
61
tensors_ = cast(List[torch.Tensor], stacked)