@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 74d5543ec5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[VLM][Core] Fix exceptions on ragged NestedTensors (#7974)

Peter Salas committed Aug 28, 2024 at 20:24 UTC 74d5543ec589daaa4ac042d65d52dccf26ee3f2c
3 files changed +21 -11
tests/multimodal/test_base.py
+12
@@ -81,3 +81,15 @@ def test_multimodal_input_batch_multiple_batchable_lists():
81 result,
82 {"image": torch.stack([torch.stack([a, b]),
83 torch.stack([c, d])])})
84 +
85 +
86 +def test_multimodal_input_batch_mixed_stacking_depths():
87 + a = torch.rand([1, 2, 3])
88 + b = torch.rand([1, 3, 3])
89 + c = torch.rand([1, 4, 3])
90 +
91 + result = MultiModalInputs.batch([{"image": [a, b]}, {"image": [c]}])
92 + assert_multimodal_inputs_equal(result, {"image": [[a, b], c.unsqueeze(0)]})
93 +
94 + result = MultiModalInputs.batch([{"image": [a]}, {"image": [b, c]}])
95 + assert_multimodal_inputs_equal(result, {"image": [a.unsqueeze(0), [b, c]]})
vllm/model_executor/models/utils.py
+7 -9
@@ -1,7 +1,6 @@
1 from typing import (Dict, Iterable, List, Literal, Optional, Protocol, Tuple,
2 Union, overload)
3
4 -import numpy as np
4 import torch
5 import torch.nn as nn
6 from torch.func import functional_call
@@ -96,12 +95,13 @@ def flatten_bn(
95
96 def _flatten_embeddings(embeddings: NestedTensors) -> torch.Tensor:
97 """
99 - Recursively concatenates NestedTensors along any heterogeneously sized
100 - dimensions.
98 + Recursively flattens and concatenates NestedTensors on all but the last
99 + dimension.
100 """
101
102 if isinstance(embeddings, torch.Tensor):
104 - return embeddings
103 + # Flatten all but the last dimension.
104 + return embeddings.flatten(0, -2)
105
106 return torch.cat(tuple(_flatten_embeddings(t) for t in embeddings))
107
@@ -136,15 +136,13 @@ def merge_multimodal_embeddings(input_ids: torch.Tensor,
136 assert isinstance(num_expected_tokens, int)
137
138 flattened = _flatten_embeddings(multimodal_embeddings)
139 - *dims, embed_dim = flattened.shape
140 - num_multimodal_embeddings = np.prod(dims)
141 - if num_multimodal_embeddings != num_expected_tokens:
139 + if flattened.shape[0] != num_expected_tokens:
140 expr = _embedding_count_expression(multimodal_embeddings)
141 raise ValueError(
144 - f"Attempted to assign {expr} = {num_multimodal_embeddings} "
142 + f"Attempted to assign {expr} = {flattened.shape[0]} "
143 f"multimodal tokens to {num_expected_tokens} placeholders")
144
147 - inputs_embeds[mask] = flattened.view(num_expected_tokens, embed_dim)
145 + inputs_embeds[mask] = flattened
146 return inputs_embeds
147
148
vllm/multimodal/base.py
+2 -2
@@ -54,8 +54,8 @@ class MultiModalInputs(_MultiModalInputsBase):
54 return nested_tensors
55
56 stacked = [MultiModalInputs._try_stack(t) for t in nested_tensors]
57 - if is_list_of(stacked, list):
58 - # Do not stack nested lists
57 + if not is_list_of(stacked, torch.Tensor, check="all"):
58 + # Only tensors (not lists) can be stacked.
59 return stacked
60
61 tensors_ = cast(List[torch.Tensor], stacked)