A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Bugfix] Fixes Phi3v & Ultravox Multimodal EmbeddingInputs (#8979)
hhzhang16 committed
Oct 4, 2024 at 22:05 UTC
53b3a330273967a3c4124cbfef2cacac92f553ba
2 files changed
+43
-25
vllm/model_executor/models/phi3v.py
+14
-6
@@ -467,9 +467,10 @@ def input_processor_for_phi3v(ctx: InputContext,
467
input_height=h,
468
num_crops=num_crops))
469
elif isinstance(image_data, torch.Tensor):
470
- num_images, image_feature_size, hidden_size = image_data.shape
470
+ image_feature_size = [image_data.shape[0]]
471
+ image_data = [image_data]
472
elif is_list_of(image_data, torch.Tensor):
472
- image_feature_size = [item.shape[1] for item in image_data]
473
+ image_feature_size = [item.shape[0] for item in image_data]
474
else:
475
raise TypeError(f"Invalid image type: {type(image_data)}")
476
@@ -611,9 +612,6 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
612
image_sizes = kwargs.pop("image_sizes", None)
613
image_embeds = kwargs.pop("image_embeds", None)
614
614
- if pixel_values is None:
615
- return None
616
-
615
if pixel_values is None and image_embeds is None:
616
return None
617
@@ -650,7 +648,17 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
648
) -> torch.Tensor:
649
650
if image_input["type"] == "image_embeds":
653
- return image_input["data"]
651
+ image_data = image_input["data"]
652
+ if is_list_of(image_data, torch.Tensor):
653
+ # it's already a list of tensors
654
+ return image_data
655
+ if len(image_data.shape) == 3:
656
+ # 3D tensor
657
+ return list(torch.unbind(image_data, dim=0))
658
+ raise ValueError(
659
+ "We expect batched 2D tensors;"
660
+ "this can be either a list of 2D tensors or a single 3D tensor."
661
+ )
662
663
assert self.vision_embed_tokens is not None
664
image_embeds = self.vision_embed_tokens(image_input["data"],
vllm/model_executor/models/ultravox.py
+29
-19
@@ -38,6 +38,7 @@ from vllm.multimodal.utils import (cached_get_tokenizer,
38
from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
39
SequenceData)
40
from vllm.transformers_utils.configs.ultravox import UltravoxConfig
41
+from vllm.utils import is_list_of
42
43
from .interfaces import SupportsMultiModal, SupportsPP
44
@@ -119,6 +120,10 @@ def input_mapper_for_ultravox(ctx: InputContext, data: object):
120
if not isinstance(data, list):
121
data = [data]
122
123
+ # If the audio inputs are embeddings, no need for preprocessing
124
+ if is_list_of(data, torch.Tensor, check="all"):
125
+ return MultiModalInputs({"audio_embeds": data})
126
+
127
audio_features = []
128
for audio_input in data:
129
if not isinstance(audio_input, tuple):
@@ -165,25 +170,30 @@ def input_processor_for_ultravox(ctx: InputContext, llm_inputs: LLMInputs):
170
audios = [audios]
171
172
audio_token_counts = []
168
- for audio_data, sample_rate in audios:
169
- audio_length = audio_data.shape[0]
170
- if sample_rate != feature_extractor.sampling_rate:
171
- # Account for resampling.
172
- adjustment = feature_extractor.sampling_rate / sample_rate
173
- audio_length = math.ceil(adjustment * audio_length)
174
-
175
- feature_extractor_output_length = math.ceil(
176
- (audio_length - (feature_extractor.hop_length - 1)) /
177
- feature_extractor.hop_length)
178
-
179
- uv_config = ctx.get_hf_config(UltravoxConfig)
180
- audio_num_tokens = min(
181
- max(
182
- 1,
183
- math.ceil(feature_extractor_output_length /
184
- (uv_config.stack_factor * 2))),
185
- get_ultravox_max_audio_tokens(ctx))
186
- audio_token_counts.append(audio_num_tokens)
173
+ for audio in audios:
174
+ if isinstance(audio, torch.Tensor):
175
+ audio_num_tokens = audio.shape[1]
176
+ audio_token_counts.append(audio_num_tokens)
177
+ else:
178
+ audio_data, sample_rate = audio
179
+ audio_length = audio_data.shape[0]
180
+ if sample_rate != feature_extractor.sampling_rate:
181
+ # Account for resampling.
182
+ adjustment = feature_extractor.sampling_rate / sample_rate
183
+ audio_length = math.ceil(adjustment * audio_length)
184
+
185
+ feature_extractor_output_length = math.ceil(
186
+ (audio_length - (feature_extractor.hop_length - 1)) /
187
+ feature_extractor.hop_length)
188
+
189
+ uv_config = ctx.get_hf_config(UltravoxConfig)
190
+ audio_num_tokens = min(
191
+ max(
192
+ 1,
193
+ math.ceil(feature_extractor_output_length /
194
+ (uv_config.stack_factor * 2))),
195
+ get_ultravox_max_audio_tokens(ctx))
196
+ audio_token_counts.append(audio_num_tokens)
197
198
tokenizer = cached_get_tokenizer(ctx.model_config.tokenizer)
199