@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 53b3a33027
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Bugfix] Fixes Phi3v & Ultravox Multimodal EmbeddingInputs (#8979)

hhzhang16 committed Oct 4, 2024 at 22:05 UTC 53b3a330273967a3c4124cbfef2cacac92f553ba
2 files changed +43 -25
vllm/model_executor/models/phi3v.py
+14 -6
@@ -467,9 +467,10 @@ def input_processor_for_phi3v(ctx: InputContext,
467 input_height=h,
468 num_crops=num_crops))
469 elif isinstance(image_data, torch.Tensor):
470 - num_images, image_feature_size, hidden_size = image_data.shape
470 + image_feature_size = [image_data.shape[0]]
471 + image_data = [image_data]
472 elif is_list_of(image_data, torch.Tensor):
472 - image_feature_size = [item.shape[1] for item in image_data]
473 + image_feature_size = [item.shape[0] for item in image_data]
474 else:
475 raise TypeError(f"Invalid image type: {type(image_data)}")
476
@@ -611,9 +612,6 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
612 image_sizes = kwargs.pop("image_sizes", None)
613 image_embeds = kwargs.pop("image_embeds", None)
614
614 - if pixel_values is None:
615 - return None
616 -
615 if pixel_values is None and image_embeds is None:
616 return None
617
@@ -650,7 +648,17 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
648 ) -> torch.Tensor:
649
650 if image_input["type"] == "image_embeds":
653 - return image_input["data"]
651 + image_data = image_input["data"]
652 + if is_list_of(image_data, torch.Tensor):
653 + # it's already a list of tensors
654 + return image_data
655 + if len(image_data.shape) == 3:
656 + # 3D tensor
657 + return list(torch.unbind(image_data, dim=0))
658 + raise ValueError(
659 + "We expect batched 2D tensors;"
660 + "this can be either a list of 2D tensors or a single 3D tensor."
661 + )
662
663 assert self.vision_embed_tokens is not None
664 image_embeds = self.vision_embed_tokens(image_input["data"],
vllm/model_executor/models/ultravox.py
+29 -19
@@ -38,6 +38,7 @@ from vllm.multimodal.utils import (cached_get_tokenizer,
38 from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
39 SequenceData)
40 from vllm.transformers_utils.configs.ultravox import UltravoxConfig
41 +from vllm.utils import is_list_of
42
43 from .interfaces import SupportsMultiModal, SupportsPP
44
@@ -119,6 +120,10 @@ def input_mapper_for_ultravox(ctx: InputContext, data: object):
120 if not isinstance(data, list):
121 data = [data]
122
123 + # If the audio inputs are embeddings, no need for preprocessing
124 + if is_list_of(data, torch.Tensor, check="all"):
125 + return MultiModalInputs({"audio_embeds": data})
126 +
127 audio_features = []
128 for audio_input in data:
129 if not isinstance(audio_input, tuple):
@@ -165,25 +170,30 @@ def input_processor_for_ultravox(ctx: InputContext, llm_inputs: LLMInputs):
170 audios = [audios]
171
172 audio_token_counts = []
168 - for audio_data, sample_rate in audios:
169 - audio_length = audio_data.shape[0]
170 - if sample_rate != feature_extractor.sampling_rate:
171 - # Account for resampling.
172 - adjustment = feature_extractor.sampling_rate / sample_rate
173 - audio_length = math.ceil(adjustment * audio_length)
174 -
175 - feature_extractor_output_length = math.ceil(
176 - (audio_length - (feature_extractor.hop_length - 1)) /
177 - feature_extractor.hop_length)
178 -
179 - uv_config = ctx.get_hf_config(UltravoxConfig)
180 - audio_num_tokens = min(
181 - max(
182 - 1,
183 - math.ceil(feature_extractor_output_length /
184 - (uv_config.stack_factor * 2))),
185 - get_ultravox_max_audio_tokens(ctx))
186 - audio_token_counts.append(audio_num_tokens)
173 + for audio in audios:
174 + if isinstance(audio, torch.Tensor):
175 + audio_num_tokens = audio.shape[1]
176 + audio_token_counts.append(audio_num_tokens)
177 + else:
178 + audio_data, sample_rate = audio
179 + audio_length = audio_data.shape[0]
180 + if sample_rate != feature_extractor.sampling_rate:
181 + # Account for resampling.
182 + adjustment = feature_extractor.sampling_rate / sample_rate
183 + audio_length = math.ceil(adjustment * audio_length)
184 +
185 + feature_extractor_output_length = math.ceil(
186 + (audio_length - (feature_extractor.hop_length - 1)) /
187 + feature_extractor.hop_length)
188 +
189 + uv_config = ctx.get_hf_config(UltravoxConfig)
190 + audio_num_tokens = min(
191 + max(
192 + 1,
193 + math.ceil(feature_extractor_output_length /
194 + (uv_config.stack_factor * 2))),
195 + get_ultravox_max_audio_tokens(ctx))
196 + audio_token_counts.append(audio_num_tokens)
197
198 tokenizer = cached_get_tokenizer(ctx.model_config.tokenizer)
199