A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[V1] Support audio language models on V1 (#11733)
Signed-off-by: Roger Wang <ywang@roblox.com>
Roger Wang committed
Jan 7, 2025 at 03:47 UTC
2de197bdd4b82a004ff99806d054dce1d93b3ced
3 files changed
+29
-12
docs/source/models/supported_models.md
+2
-2
@@ -710,7 +710,7 @@ See [this page](#generative-models) for more information on how to use generativ
710
- `Qwen/Qwen2-Audio-7B-Instruct`
711
-
712
- ✅︎
713
- -
713
+ - ✅︎
714
* - `Qwen2VLForConditionalGeneration`
715
- Qwen2-VL
716
- T + I<sup>E+</sup> + V<sup>E+</sup>
@@ -724,7 +724,7 @@ See [this page](#generative-models) for more information on how to use generativ
724
- `fixie-ai/ultravox-v0_3`
725
-
726
- ✅︎
727
- -
727
+ - ✅︎
728
```
729
730
<sup>E</sup> Pre-computed embeddings can be inputted for this modality.
vllm/model_executor/models/qwen2_audio.py
+6
-3
@@ -335,13 +335,16 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
335
selected_audio_feature = audio_outputs.last_hidden_state
336
audio_features = self.multi_modal_projector(selected_audio_feature)
337
num_audios, max_audio_tokens, embed_dim = audio_features.shape
338
+ audio_output_lengths = audio_output_lengths.unsqueeze(1)
339
audio_features_mask = torch.arange(max_audio_tokens).expand(
339
- num_audios, max_audio_tokens
340
- ).to(audio_output_lengths.device) < audio_output_lengths.unsqueeze(1)
340
+ num_audios, max_audio_tokens).to(
341
+ audio_output_lengths.device) < audio_output_lengths
342
masked_audio_features = audio_features[audio_features_mask].view(
343
-1, embed_dim)
344
344
- return masked_audio_features
345
+ # Split to tuple of embeddings for individual audio input.
346
+ return torch.split(masked_audio_features,
347
+ audio_output_lengths.flatten().tolist())
348
349
def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
350
audio_input = self._parse_and_validate_audio_input(**kwargs)
vllm/model_executor/models/ultravox.py
+21
-7
@@ -1,6 +1,5 @@
1
# Adapted from https://github.com/fixie-ai/ultravox/blob/ecd58c4041030bae2ad15aa6bcf04ab43199ea02/ultravox/model/ultravox_model.py
2
"""PyTorch Ultravox model."""
3
-
3
import math
4
from functools import cached_property
5
from typing import (Any, Iterable, List, Literal, Mapping, Optional, Set,
@@ -14,6 +13,7 @@ from transformers import BatchFeature, ProcessorMixin
13
from transformers.models.whisper import WhisperFeatureExtractor
14
from transformers.models.whisper.modeling_whisper import WhisperEncoder
15
16
+from vllm import envs
17
from vllm.attention import AttentionMetadata
18
from vllm.config import VllmConfig
19
from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
@@ -35,8 +35,11 @@ from vllm.transformers_utils.configs.ultravox import UltravoxConfig
35
from .interfaces import SupportsMultiModal, SupportsPP
36
from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
37
init_vllm_registered_model, maybe_prefix,
38
+ merge_multimodal_embeddings,
39
merge_multimodal_embeddings_from_map)
40
41
+_AUDIO_PLACEHOLDER_OVERRIDE = "<|reserved_special_token_0|>"
42
+_AUDIO_PLACEHOLDER_TOKEN = 128002
43
_AUDIO_TOKENS_PER_SECOND = 6.25
44
45
@@ -64,7 +67,14 @@ class UltravoxProcessingMixin(ProcessingMixin):
67
# Ignored in initialization
68
sampling_rate: Optional[int] = None,
69
) -> ProcessorMixin:
67
- return self.ctx.get_hf_processor()
70
+ hf_processor = self.ctx.get_hf_processor()
71
+
72
+ # NOTE: Ultravox processing definition uses '<|eot_id|>' as the
73
+ # placeholder that will cause confusion with the actual end of turn
74
+ # token, thus we override placeholder with a reserved special
75
+ # token.
76
+ hf_processor.audio_token_replacement = _AUDIO_PLACEHOLDER_OVERRIDE
77
+ return hf_processor
78
79
def _get_feature_extractor(
80
self,
@@ -465,11 +475,15 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
475
inputs_embeds = self.language_model.get_input_embeddings(input_ids)
476
if multimodal_embeddings is not None:
477
468
- # TODO(ywang96): use merge_multimodal_embeddings after
469
- # v0 is deprecated
470
- merge_multimodal_embeddings_from_map(
471
- inputs_embeds, multimodal_embeddings,
472
- attn_metadata.multi_modal_placeholder_index_maps["audio"])
478
+ # TODO(ywang96): remove this block after v0 is deprecated.
479
+ if not envs.VLLM_USE_V1:
480
+ merge_multimodal_embeddings_from_map(
481
+ inputs_embeds, multimodal_embeddings,
482
+ attn_metadata.multi_modal_placeholder_index_maps["audio"])
483
+ else:
484
+ inputs_embeds = merge_multimodal_embeddings(
485
+ input_ids, inputs_embeds, multimodal_embeddings,
486
+ _AUDIO_PLACEHOLDER_TOKEN)
487
return inputs_embeds
488
489
def forward(self,