@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2de197bdd4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[V1] Support audio language models on V1 (#11733)

Signed-off-by: Roger Wang <ywang@roblox.com>

Roger Wang committed Jan 7, 2025 at 03:47 UTC 2de197bdd4b82a004ff99806d054dce1d93b3ced
3 files changed +29 -12
docs/source/models/supported_models.md
+2 -2
@@ -710,7 +710,7 @@ See [this page](#generative-models) for more information on how to use generativ
710 - `Qwen/Qwen2-Audio-7B-Instruct`
711 -
712 - ✅︎
713 - -
713 + - ✅︎
714 * - `Qwen2VLForConditionalGeneration`
715 - Qwen2-VL
716 - T + I<sup>E+</sup> + V<sup>E+</sup>
@@ -724,7 +724,7 @@ See [this page](#generative-models) for more information on how to use generativ
724 - `fixie-ai/ultravox-v0_3`
725 -
726 - ✅︎
727 - -
727 + - ✅︎
728 ```
729
730 <sup>E</sup> Pre-computed embeddings can be inputted for this modality.
vllm/model_executor/models/qwen2_audio.py
+6 -3
@@ -335,13 +335,16 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
335 selected_audio_feature = audio_outputs.last_hidden_state
336 audio_features = self.multi_modal_projector(selected_audio_feature)
337 num_audios, max_audio_tokens, embed_dim = audio_features.shape
338 + audio_output_lengths = audio_output_lengths.unsqueeze(1)
339 audio_features_mask = torch.arange(max_audio_tokens).expand(
339 - num_audios, max_audio_tokens
340 - ).to(audio_output_lengths.device) < audio_output_lengths.unsqueeze(1)
340 + num_audios, max_audio_tokens).to(
341 + audio_output_lengths.device) < audio_output_lengths
342 masked_audio_features = audio_features[audio_features_mask].view(
343 -1, embed_dim)
344
344 - return masked_audio_features
345 + # Split to tuple of embeddings for individual audio input.
346 + return torch.split(masked_audio_features,
347 + audio_output_lengths.flatten().tolist())
348
349 def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
350 audio_input = self._parse_and_validate_audio_input(**kwargs)
vllm/model_executor/models/ultravox.py
+21 -7
@@ -1,6 +1,5 @@
1 # Adapted from https://github.com/fixie-ai/ultravox/blob/ecd58c4041030bae2ad15aa6bcf04ab43199ea02/ultravox/model/ultravox_model.py
2 """PyTorch Ultravox model."""
3 -
3 import math
4 from functools import cached_property
5 from typing import (Any, Iterable, List, Literal, Mapping, Optional, Set,
@@ -14,6 +13,7 @@ from transformers import BatchFeature, ProcessorMixin
13 from transformers.models.whisper import WhisperFeatureExtractor
14 from transformers.models.whisper.modeling_whisper import WhisperEncoder
15
16 +from vllm import envs
17 from vllm.attention import AttentionMetadata
18 from vllm.config import VllmConfig
19 from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
@@ -35,8 +35,11 @@ from vllm.transformers_utils.configs.ultravox import UltravoxConfig
35 from .interfaces import SupportsMultiModal, SupportsPP
36 from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
37 init_vllm_registered_model, maybe_prefix,
38 + merge_multimodal_embeddings,
39 merge_multimodal_embeddings_from_map)
40
41 +_AUDIO_PLACEHOLDER_OVERRIDE = "<|reserved_special_token_0|>"
42 +_AUDIO_PLACEHOLDER_TOKEN = 128002
43 _AUDIO_TOKENS_PER_SECOND = 6.25
44
45
@@ -64,7 +67,14 @@ class UltravoxProcessingMixin(ProcessingMixin):
67 # Ignored in initialization
68 sampling_rate: Optional[int] = None,
69 ) -> ProcessorMixin:
67 - return self.ctx.get_hf_processor()
70 + hf_processor = self.ctx.get_hf_processor()
71 +
72 + # NOTE: Ultravox processing definition uses '<|eot_id|>' as the
73 + # placeholder that will cause confusion with the actual end of turn
74 + # token, thus we override placeholder with a reserved special
75 + # token.
76 + hf_processor.audio_token_replacement = _AUDIO_PLACEHOLDER_OVERRIDE
77 + return hf_processor
78
79 def _get_feature_extractor(
80 self,
@@ -465,11 +475,15 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
475 inputs_embeds = self.language_model.get_input_embeddings(input_ids)
476 if multimodal_embeddings is not None:
477
468 - # TODO(ywang96): use merge_multimodal_embeddings after
469 - # v0 is deprecated
470 - merge_multimodal_embeddings_from_map(
471 - inputs_embeds, multimodal_embeddings,
472 - attn_metadata.multi_modal_placeholder_index_maps["audio"])
478 + # TODO(ywang96): remove this block after v0 is deprecated.
479 + if not envs.VLLM_USE_V1:
480 + merge_multimodal_embeddings_from_map(
481 + inputs_embeds, multimodal_embeddings,
482 + attn_metadata.multi_modal_placeholder_index_maps["audio"])
483 + else:
484 + inputs_embeds = merge_multimodal_embeddings(
485 + input_ids, inputs_embeds, multimodal_embeddings,
486 + _AUDIO_PLACEHOLDER_TOKEN)
487 return inputs_embeds
488
489 def forward(self,