A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[model] support qwen2audio embedding input (#23625)
Signed-off-by: Yuekai Zhang <zhangyuekai@foxmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Yuekai Zhang committed
Aug 26, 2025 at 23:48 UTC
9d4183dd2e751e94442d7f02966d33cc085de708
2 files changed
+93
-29
vllm/model_executor/models/qwen2_5_omni_thinker.py
+7
-6
@@ -47,7 +47,7 @@ from vllm.model_executor.models.qwen2_5_vl import (
47
Qwen2_5_VLProcessingInfo, Qwen2_5_VLVideoEmbeddingInputs,
48
Qwen2_5_VLVideoInputs, Qwen2_5_VLVideoPixelInputs)
49
from vllm.model_executor.models.qwen2_audio import (
50
- Qwen2AudioInputs, Qwen2AudioProcessingInfo,
50
+ Qwen2AudioFeatureInputs, Qwen2AudioProcessingInfo,
51
_get_feat_extract_output_lengths)
52
from vllm.model_executor.models.qwen2_vl import Qwen2VLMultiModalDataParser
53
from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -534,7 +534,7 @@ class Qwen2_5OmniConditionalGenerationMixin:
534
return torch.concat(mm_input, dim=dim)
535
536
def _parse_and_validate_audio_input(
537
- self, **kwargs: object) -> Optional[Qwen2AudioInputs]:
537
+ self, **kwargs: object) -> Optional[Qwen2AudioFeatureInputs]:
538
input_audio_features = kwargs.pop('input_audio_features', None)
539
audio_feature_lengths = kwargs.pop('audio_feature_lengths', None)
540
feature_attention_mask = kwargs.pop('feature_attention_mask', None)
@@ -548,9 +548,10 @@ class Qwen2_5OmniConditionalGenerationMixin:
548
if not isinstance(input_audio_features, (torch.Tensor, list)):
549
raise ValueError("Incorrect type of audio input features. "
550
f"Got type: {type(input_audio_features)}")
551
- return Qwen2AudioInputs(input_features=input_audio_features,
552
- audio_feature_lengths=audio_feature_lengths,
553
- feature_attention_mask=feature_attention_mask)
551
+ return Qwen2AudioFeatureInputs(
552
+ input_features=input_audio_features,
553
+ audio_feature_lengths=audio_feature_lengths,
554
+ feature_attention_mask=feature_attention_mask)
555
556
def _parse_and_validate_image_input(
557
self,
@@ -630,7 +631,7 @@ class Qwen2_5OmniConditionalGenerationMixin:
631
632
def _process_audio_input(
633
self,
633
- audio_input: Qwen2AudioInputs,
634
+ audio_input: Qwen2AudioFeatureInputs,
635
audio_hashes: list[str] = None,
636
cached_audio_features: torch.Tensor = None,
637
) -> torch.Tensor:
vllm/model_executor/models/qwen2_audio.py
+86
-23
@@ -23,7 +23,7 @@
23
# limitations under the License.
24
"""Inference-only Qwen2-Audio model compatible with HuggingFace weights."""
25
from collections.abc import Iterable, Mapping, Sequence
26
-from typing import Any, Optional, TypedDict, Union
26
+from typing import Any, Literal, Optional, TypedDict, Union
27
28
import torch
29
import torch.nn as nn
@@ -36,9 +36,11 @@ from transformers.models.whisper import WhisperFeatureExtractor
36
from vllm.config import VllmConfig
37
from vllm.model_executor.sampling_metadata import SamplingMetadata
38
from vllm.multimodal import MULTIMODAL_REGISTRY
39
-from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
39
+from vllm.multimodal.inputs import (AudioItem, ModalityData,
40
+ MultiModalDataDict, MultiModalFieldConfig,
41
MultiModalKwargsItems)
41
-from vllm.multimodal.parse import (AudioProcessorItems, MultiModalDataItems,
42
+from vllm.multimodal.parse import (AudioProcessorItems, DictEmbeddingItems,
43
+ ModalityDataItems, MultiModalDataItems,
44
MultiModalDataParser)
45
from vllm.multimodal.processing import (BaseMultiModalProcessor,
46
BaseProcessingInfo, PromptReplacement,
@@ -52,7 +54,8 @@ from .utils import (AutoWeightsLoader, init_vllm_registered_model,
54
55
56
# # === Audio Inputs === #
55
-class Qwen2AudioInputs(TypedDict):
57
+class Qwen2AudioFeatureInputs(TypedDict):
58
+ type: Literal["audio_features"]
59
input_features: torch.Tensor
60
"""Shape: `(num_audios, num_mel_bins, 3000)`"""
61
@@ -60,6 +63,16 @@ class Qwen2AudioInputs(TypedDict):
63
"""Shape: `(num_audios, 3000)`"""
64
65
66
+class Qwen2AudioEmbeddingInputs(TypedDict):
67
+ type: Literal["audio_embeds"]
68
+ audio_embeds: list[torch.Tensor]
69
+ """Shape: `(num_audio_features, hidden_size)`
70
+ `hidden_size` must match the hidden size of language model backbone.
71
+ """
72
+
73
+
74
+Qwen2AudioInputs = Union[Qwen2AudioFeatureInputs, Qwen2AudioEmbeddingInputs]
75
+
76
# === Audio Encoder === #
77
78
@@ -128,12 +141,38 @@ class Qwen2AudioDummyInputsBuilder(
141
}
142
143
144
+def _qwen2audio_field_config(hf_inputs: Mapping[str, torch.Tensor]):
145
+ return dict(
146
+ audio_embeds=MultiModalFieldConfig.batched("audio"),
147
+ input_features=MultiModalFieldConfig.batched("audio"),
148
+ feature_attention_mask=MultiModalFieldConfig.batched("audio"),
149
+ )
150
+
151
+
152
+class Qwen2AudioMultiModalDataParser(MultiModalDataParser):
153
+
154
+ def _parse_audio_data(
155
+ self,
156
+ data: Union[dict[str, torch.Tensor], ModalityData[AudioItem]],
157
+ ) -> Optional[ModalityDataItems[Any, Any]]:
158
+ if isinstance(data, dict):
159
+ return DictEmbeddingItems(
160
+ data,
161
+ modality="audio",
162
+ required_fields={"audio_embeds"},
163
+ fields_factory=_qwen2audio_field_config,
164
+ )
165
+
166
+ return super()._parse_audio_data(data)
167
+
168
+
169
class Qwen2AudioMultiModalProcessor(
170
BaseMultiModalProcessor[Qwen2AudioProcessingInfo]):
171
172
def _get_data_parser(self) -> MultiModalDataParser:
173
feature_extractor = self.info.get_feature_extractor()
136
- return MultiModalDataParser(target_sr=feature_extractor.sampling_rate)
174
+ return Qwen2AudioMultiModalDataParser(
175
+ target_sr=feature_extractor.sampling_rate)
176
177
def _call_hf_processor(
178
self,
@@ -173,10 +212,7 @@ class Qwen2AudioMultiModalProcessor(
212
hf_inputs: BatchFeature,
213
hf_processor_mm_kwargs: Mapping[str, object],
214
) -> Mapping[str, MultiModalFieldConfig]:
176
- return dict(
177
- input_features=MultiModalFieldConfig.batched("audio"),
178
- feature_attention_mask=MultiModalFieldConfig.batched("audio"),
179
- )
215
+ return _qwen2audio_field_config(hf_inputs)
216
217
def _get_prompt_updates(
218
self,
@@ -184,6 +220,7 @@ class Qwen2AudioMultiModalProcessor(
220
hf_processor_mm_kwargs: Mapping[str, object],
221
out_mm_kwargs: MultiModalKwargsItems,
222
) -> Sequence[PromptUpdate]:
223
+
224
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
225
tokenizer = self.info.get_tokenizer()
226
vocab = tokenizer.get_vocab()
@@ -211,7 +248,15 @@ class Qwen2AudioMultiModalProcessor(
248
audio_output_lengths = audio_output_lens.tolist()
249
250
def get_replacement_qwen2_audio(item_idx: int):
214
- num_features = audio_output_lengths[item_idx]
251
+
252
+ if audio_output_lengths:
253
+ num_features = audio_output_lengths[item_idx]
254
+ else:
255
+ audio_embeds = out_mm_data["audio_embeds"][item_idx]
256
+ assert len(audio_embeds.shape
257
+ ) == 2, "audio_embeds must be a 2D tensor"
258
+ num_features = audio_embeds.shape[0]
259
+
260
if num_features == 0:
261
audios = mm_items.get_items("audio", AudioProcessorItems)
262
audio_len = audios.get_audio_length(item_idx)
@@ -286,21 +331,39 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
331
def _parse_and_validate_audio_input(
332
self, **kwargs: object) -> Optional[Qwen2AudioInputs]:
333
input_features = kwargs.pop('input_features', None)
334
+ audio_embeds = kwargs.pop('audio_embeds', None)
335
feature_attention_mask = kwargs.pop('feature_attention_mask', None)
290
- if input_features is None:
336
+
337
+ if input_features is None and audio_embeds is None:
338
return None
292
- input_features = self._validate_and_reshape_mm_tensor(
293
- input_features, 'input_features')
294
- feature_attention_mask = self._validate_and_reshape_mm_tensor(
295
- feature_attention_mask, 'feature_attention_mask')
296
- if not isinstance(input_features, (torch.Tensor, list)):
297
- raise ValueError("Incorrect type of audio input features. "
298
- f"Got type: {type(input_features)}")
299
- return Qwen2AudioInputs(input_features=input_features,
300
- feature_attention_mask=feature_attention_mask)
301
-
302
- def _process_audio_input(self,
303
- audio_input: Qwen2AudioInputs) -> torch.Tensor:
339
+
340
+ if audio_embeds is not None:
341
+ if not isinstance(audio_embeds, (torch.Tensor, list)):
342
+ raise ValueError("Incorrect type of audio embeds. "
343
+ f"Got type: {type(audio_embeds)}")
344
+ audio_embeds = self._validate_and_reshape_mm_tensor(
345
+ audio_embeds, "audio_embeds")
346
+ return Qwen2AudioEmbeddingInputs(type="audio_embeds",
347
+ audio_embeds=audio_embeds)
348
+
349
+ if input_features is not None:
350
+ input_features = self._validate_and_reshape_mm_tensor(
351
+ input_features, 'input_features')
352
+ feature_attention_mask = self._validate_and_reshape_mm_tensor(
353
+ feature_attention_mask, 'feature_attention_mask')
354
+ return Qwen2AudioFeatureInputs(
355
+ type="audio_features",
356
+ input_features=input_features,
357
+ feature_attention_mask=feature_attention_mask)
358
+
359
+ raise AssertionError("This line should be unreachable.")
360
+
361
+ def _process_audio_input(
362
+ self, audio_input: Qwen2AudioInputs
363
+ ) -> Union[torch.Tensor, tuple[torch.Tensor, ...]]:
364
+ if audio_input["type"] == "audio_embeds":
365
+ audio_embeds = audio_input["audio_embeds"]
366
+ return tuple(audio_embeds)
367
368
input_features = audio_input["input_features"]
369
feature_attention_mask = audio_input["feature_attention_mask"]