@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fc6c274626
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Model] Add Qwen2-Audio model support (#9248)

Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>

Yunfei Chu committed Oct 24, 2024 at 01:54 UTC fc6c27462614924dca90898ef762d6c56c0874ba
7 files changed +515 -17
docs/source/models/supported_models.rst
+6
@@ -459,6 +459,12 @@ Text Generation
459 - :code:`Qwen/Qwen-VL`, :code:`Qwen/Qwen-VL-Chat`, etc.
460 -
461 - ✅︎
462 + * - :code:`Qwen2AudioForConditionalGeneration`
463 + - Qwen2-Audio
464 + - T + A\ :sup:`+`
465 + - :code:`Qwen/Qwen2-Audio-7B-Instruct`
466 + -
467 + - ✅︎
468 * - :code:`Qwen2VLForConditionalGeneration`
469 - Qwen2-VL
470 - T + I\ :sup:`E+` + V\ :sup:`+`
examples/offline_inference_audio_language.py
+38 -16
@@ -12,14 +12,15 @@ from vllm.assets.audio import AudioAsset
12 from vllm.utils import FlexibleArgumentParser
13
14 audio_assets = [AudioAsset("mary_had_lamb"), AudioAsset("winning_call")]
15 -question_per_audio_count = [
16 - "What is recited in the audio?",
17 - "What sport and what nursery rhyme are referenced?"
18 -]
15 +question_per_audio_count = {
16 + 0: "What is 1+1?",
17 + 1: "What is recited in the audio?",
18 + 2: "What sport and what nursery rhyme are referenced?"
19 +}
20
21
22 # Ultravox 0.3
22 -def run_ultravox(question, audio_count):
23 +def run_ultravox(question: str, audio_count: int):
24 model_name = "fixie-ai/ultravox-v0_3"
25
26 tokenizer = AutoTokenizer.from_pretrained(model_name)
@@ -42,9 +43,29 @@ def run_ultravox(question, audio_count):
43 return llm, prompt, stop_token_ids
44
45
45 -model_example_map = {
46 - "ultravox": run_ultravox,
47 -}
46 +# Qwen2-Audio
47 +def run_qwen2_audio(question: str, audio_count: int):
48 + model_name = "Qwen/Qwen2-Audio-7B-Instruct"
49 +
50 + llm = LLM(model=model_name,
51 + max_model_len=4096,
52 + max_num_seqs=5,
53 + limit_mm_per_prompt={"audio": audio_count})
54 +
55 + audio_in_prompt = "".join([
56 + f"Audio {idx+1}: "
57 + f"<|audio_bos|><|AUDIO|><|audio_eos|>\n" for idx in range(audio_count)
58 + ])
59 +
60 + prompt = ("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
61 + "<|im_start|>user\n"
62 + f"{audio_in_prompt}{question}<|im_end|>\n"
63 + "<|im_start|>assistant\n")
64 + stop_token_ids = None
65 + return llm, prompt, stop_token_ids
66 +
67 +
68 +model_example_map = {"ultravox": run_ultravox, "qwen2_audio": run_qwen2_audio}
69
70
71 def main(args):
@@ -54,7 +75,7 @@ def main(args):
75
76 audio_count = args.num_audios
77 llm, prompt, stop_token_ids = model_example_map[model](
57 - question_per_audio_count[audio_count - 1], audio_count)
78 + question_per_audio_count[audio_count], audio_count)
79
80 # We set temperature to 0.2 so that outputs can be different
81 # even when all prompts are identical when running batch inference.
@@ -62,16 +83,17 @@ def main(args):
83 max_tokens=64,
84 stop_token_ids=stop_token_ids)
85
65 - assert args.num_prompts > 0
66 - inputs = {
67 - "prompt": prompt,
68 - "multi_modal_data": {
86 + mm_data = {}
87 + if audio_count > 0:
88 + mm_data = {
89 "audio": [
90 asset.audio_and_sample_rate
91 for asset in audio_assets[:audio_count]
92 ]
73 - },
74 - }
93 + }
94 +
95 + assert args.num_prompts > 0
96 + inputs = {"prompt": prompt, "multi_modal_data": mm_data}
97 if args.num_prompts > 1:
98 # Batch inference
99 inputs = [inputs] * args.num_prompts
@@ -100,7 +122,7 @@ if __name__ == "__main__":
122 parser.add_argument("--num-audios",
123 type=int,
124 default=1,
103 - choices=[1, 2],
125 + choices=[0, 1, 2],
126 help="Number of audio items per prompt.")
127
128 args = parser.parse_args()
tests/distributed/test_pipeline_parallel.py
+1
@@ -199,6 +199,7 @@ MULTIMODAL_MODEL_SETTINGS = {
199 "microsoft/Phi-3-vision-128k-instruct": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
200 "mistralai/Pixtral-12B-2409": PPTestSettings.fast(tp_base=2, tokenizer_mode="mistral"), # noqa: E501
201 "Qwen/Qwen-VL-Chat": PPTestSettings.fast(trust_remote_code=True),
202 + "Qwen/Qwen2-Audio-7B-Instruct": PPTestSettings.fast(),
203 "Qwen/Qwen2-VL-2B-Instruct": PPTestSettings.fast(),
204 "fixie-ai/ultravox-v0_3": PPTestSettings.fast(),
205 }
vllm/entrypoints/chat_utils.py
+4 -1
@@ -196,7 +196,10 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
196 elif modality == "audio":
197 if model_type == "ultravox":
198 return "<|reserved_special_token_0|>"
199 - raise TypeError(f"Unknown {modality} model type: {model_type}")
199 + if model_type == "qwen2_audio":
200 + return (f"Audio {current_count}: "
201 + f"<|audio_bos|><|AUDIO|><|audio_eos|>")
202 + raise TypeError(f"Unknown model type: {model_type}")
203 elif modality == "video":
204 if model_type == "qwen2_vl":
205 return "<|vision_start|><|video_pad|><|vision_end|>"
vllm/model_executor/models/qwen2_audio.py new
+462
@@ -0,0 +1,462 @@
1 +# coding=utf-8
2 +# Copyright 2024 The Qwen team.
3 +# Copyright 2023 The vLLM team.
4 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
5 +#
6 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
7 +# and OPT implementations in this library. It has been modified from its
8 +# original forms to accommodate minor architectural differences compared
9 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
10 +#
11 +# Licensed under the Apache License, Version 2.0 (the "License");
12 +# you may not use this file except in compliance with the License.
13 +# You may obtain a copy of the License at
14 +#
15 +# http://www.apache.org/licenses/LICENSE-2.0
16 +#
17 +# Unless required by applicable law or agreed to in writing, software
18 +# distributed under the License is distributed on an "AS IS" BASIS,
19 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20 +# See the License for the specific language governing permissions and
21 +# limitations under the License.
22 +"""Inference-only Qwen2-Audio model compatible with HuggingFace weights."""
23 +from functools import lru_cache
24 +from typing import Iterable, List, Mapping, Optional, Tuple, TypedDict, Union
25 +
26 +import librosa
27 +import numpy as np
28 +import torch
29 +import torch.nn as nn
30 +from transformers import Qwen2AudioConfig, Qwen2AudioEncoder
31 +
32 +from vllm.attention import AttentionMetadata
33 +from vllm.config import CacheConfig, MultiModalConfig
34 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
35 + token_inputs)
36 +from vllm.logger import init_logger
37 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
38 +from vllm.model_executor.layers.quantization.base_config import (
39 + QuantizationConfig)
40 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
41 +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
42 +from vllm.model_executor.model_loader.weight_utils import (
43 + default_weight_loader, maybe_remap_kv_scale_name)
44 +from vllm.model_executor.models.qwen2 import Qwen2Model
45 +from vllm.model_executor.sampling_metadata import SamplingMetadata
46 +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalInputs
47 +from vllm.sequence import IntermediateTensors, SequenceData
48 +
49 +from .interfaces import SupportsMultiModal, SupportsPP
50 +
51 +logger = init_logger(__name__)
52 +
53 +_KEYS_TO_MODIFY_MAPPING = {
54 + "language_model.lm_head": "lm_head",
55 + "language_model.model": "language_model",
56 +}
57 +
58 +
59 +# # === Audio Inputs === #
60 +class Qwen2AudioInputs(TypedDict):
61 + input_features: torch.Tensor
62 + """Shape:
63 + `(num_audios, num_mel_bins, 3000)`
64 + """
65 +
66 + feature_attention_mask: torch.Tensor
67 + """Shape: `(num_audios, 3000)`
68 + """
69 +
70 +
71 +# === Audio Encoder === #
72 +
73 +
74 +class Qwen2AudioMultiModalProjector(nn.Module):
75 +
76 + def __init__(self, audio_hidden_size: int, text_hidden_size: int):
77 + super().__init__()
78 + self.linear = nn.Linear(audio_hidden_size, text_hidden_size, bias=True)
79 +
80 + def forward(self, audio_features):
81 + hidden_states = self.linear(audio_features)
82 + return hidden_states
83 +
84 +
85 +def dummy_data_for_qwen2_audio(ctx: InputContext, seq_len: int,
86 + mm_counts: Mapping[str, int]):
87 + num_audios = mm_counts["audio"]
88 + max_llm_audio_tokens = get_max_qwen2_audio_audio_tokens(ctx) * num_audios
89 + if seq_len - max_llm_audio_tokens - 2 < 0:
90 + raise RuntimeError(
91 + f"Qwen2-Audio cannot process {num_audios} audios in a prompt, "
92 + "please increase max_model_len or reduce audio limit by "
93 + "--limit-mm-per-prompt.")
94 +
95 + audio_token_index = ctx.model_config.hf_config.audio_token_index
96 +
97 + dummy_seqdata = SequenceData.from_prompt_token_counts(
98 + (audio_token_index, max_llm_audio_tokens),
99 + (0, seq_len - max_llm_audio_tokens),
100 + )
101 + dummy_audio = np.full((max_llm_audio_tokens * 2 * 2 * 160, ), 0.)
102 + return dummy_seqdata, {"audio": [(dummy_audio, 16000)] * num_audios}
103 +
104 +
105 +def get_processor(
106 + processor_name: str,
107 + *args,
108 + trust_remote_code: bool = False,
109 + **kwargs,
110 +):
111 + """Gets a processor for the given model name via HuggingFace.
112 +
113 + Derived from `vllm.transformers_utils.image_processor.get_image_processor`.
114 + """
115 + # don't put this import at the top level
116 + # it will call torch.cuda.device_count()
117 + from transformers import AutoProcessor
118 +
119 + try:
120 + processor = AutoProcessor.from_pretrained(
121 + processor_name,
122 + *args,
123 + trust_remote_code=trust_remote_code,
124 + **kwargs)
125 + except ValueError as e:
126 + # If the error pertains to the processor class not existing or not
127 + # currently being imported, suggest using the --trust-remote-code flag.
128 + # Unlike AutoTokenizer, AutoProcessor does not separate such errors
129 + if not trust_remote_code:
130 + err_msg = (
131 + "Failed to load the processor. If the processor is "
132 + "a custom processor not yet available in the HuggingFace "
133 + "transformers library, consider setting "
134 + "`trust_remote_code=True` in LLM or using the "
135 + "`--trust-remote-code` flag in the CLI.")
136 + raise RuntimeError(err_msg) from e
137 + else:
138 + raise e
139 +
140 + return processor
141 +
142 +
143 +cached_get_processor = lru_cache(get_processor)
144 +
145 +
146 +def _get_feat_extract_output_lengths(input_lengths: torch.LongTensor):
147 + """
148 + Computes the output length of the convolutional layers
149 + and the output length of the audio encoder
150 + """
151 + input_lengths = (input_lengths - 1) // 2 + 1
152 + output_lengths = (input_lengths - 2) // 2 + 1
153 + return input_lengths, output_lengths
154 +
155 +
156 +def get_max_qwen2_audio_audio_tokens(ctx: InputContext) -> int:
157 + max_source_position = (
158 + ctx.model_config.hf_config.audio_config.max_source_positions)
159 + output_lengths = (max_source_position - 2) // 2 + 1
160 + return output_lengths
161 +
162 +
163 +def input_processor_for_qwen2_audio(
164 + ctx: InputContext, inputs: DecoderOnlyInputs) -> DecoderOnlyInputs:
165 + multi_modal_data = inputs.get("multi_modal_data")
166 + if multi_modal_data is None or "audio" not in multi_modal_data:
167 + return inputs
168 +
169 + audios = multi_modal_data["audio"]
170 + if not isinstance(audios, list):
171 + audios = [audios]
172 +
173 + if len(audios) == 0:
174 + return inputs
175 +
176 + processor = cached_get_processor(ctx.model_config.model)
177 + resampled_audios = [
178 + librosa.resample(audio,
179 + orig_sr=sampling_rate,
180 + target_sr=processor.feature_extractor.sampling_rate)
181 + for audio, sampling_rate in audios
182 + ]
183 + audio_input_lengths = np.array(
184 + [min(3000, _.shape[0] // 160 + 1) for _ in resampled_audios])
185 +
186 + audio_feat_lengths, audio_output_lengths = _get_feat_extract_output_lengths(
187 + audio_input_lengths)
188 +
189 + audio_token_index = ctx.model_config.hf_config.audio_token_index
190 +
191 + input_ids = inputs['prompt_token_ids']
192 +
193 + new_input_ids = []
194 + audio_num = input_ids.count(audio_token_index)
195 + assert len(audio_input_lengths) == audio_num, \
196 + (f'The text input contains {audio_num} audio tokens, '
197 + f'but {len(audio_input_lengths)} audios provided')
198 + start = 0
199 + for audio_idx in range(audio_num):
200 + end = input_ids.index(audio_token_index, start)
201 + new_input_ids.extend(input_ids[start:end]) # text part
202 +
203 + new_input_ids.extend([audio_token_index] *
204 + audio_output_lengths[audio_idx])
205 + start = end + 1
206 + new_input_ids.extend(input_ids[start:])
207 +
208 + return token_inputs(
209 + prompt_token_ids=new_input_ids,
210 + prompt=inputs['prompt'],
211 + multi_modal_data=multi_modal_data,
212 + )
213 +
214 +
215 +def input_mapper_for_qwen2_audio(
216 + ctx: InputContext,
217 + multi_modal_data: Union[np.ndarray, List[np.ndarray]],
218 +) -> MultiModalInputs:
219 + """Input mapper for Qwen2-Audio."""
220 + if not isinstance(multi_modal_data, list):
221 + multi_modal_data = [multi_modal_data]
222 +
223 + if len(multi_modal_data) == 0:
224 + return MultiModalInputs()
225 +
226 + processor = cached_get_processor(ctx.model_config.model)
227 + audio_feature_extractor = processor.feature_extractor
228 + if audio_feature_extractor is None:
229 + raise RuntimeError(
230 + "No HuggingFace audio_feature_extractor is available "
231 + "to process the audio object")
232 +
233 + try:
234 + resampled_audios = [
235 + librosa.resample(
236 + audio,
237 + orig_sr=sampling_rate,
238 + target_sr=processor.feature_extractor.sampling_rate)
239 + for audio, sampling_rate in multi_modal_data
240 + ]
241 + batch_data = audio_feature_extractor(resampled_audios,
242 + sampling_rate=16000,
243 + return_attention_mask=True,
244 + padding="max_length",
245 + return_tensors="pt").data
246 + batch_data["feature_attention_mask"] = batch_data.pop("attention_mask")
247 + except Exception:
248 + logger.error("Failed to process audio (%s)", multi_modal_data)
249 + raise
250 +
251 + return MultiModalInputs(batch_data)
252 +
253 +
254 +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_qwen2_audio)
255 +@INPUT_REGISTRY.register_input_processor(input_processor_for_qwen2_audio)
256 +@MULTIMODAL_REGISTRY.register_input_mapper("audio",
257 + input_mapper_for_qwen2_audio)
258 +@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
259 + "audio", get_max_qwen2_audio_audio_tokens)
260 +class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
261 + SupportsPP):
262 +
263 + def __init__(self,
264 + config: Qwen2AudioConfig,
265 + multimodal_config: MultiModalConfig,
266 + cache_config: Optional[CacheConfig] = None,
267 + quant_config: Optional[QuantizationConfig] = None) -> None:
268 + super().__init__()
269 +
270 + self.config = config
271 + self.multimodal_config = multimodal_config
272 +
273 + self.audio_tower = Qwen2AudioEncoder(config.audio_config)
274 + self.multi_modal_projector = Qwen2AudioMultiModalProjector(
275 + config.audio_config.d_model, config.text_config.hidden_size)
276 +
277 + self.quant_config = quant_config
278 +
279 + self.language_model = Qwen2Model(config.text_config, cache_config,
280 + quant_config)
281 + self.unpadded_vocab_size = config.text_config.vocab_size
282 + if config.text_config.tie_word_embeddings:
283 + self.lm_head = self.language_model.embed_tokens
284 + else:
285 + self.lm_head = ParallelLMHead(config.text_config.vocab_size,
286 + config.text_config.hidden_size,
287 + quant_config=quant_config)
288 + logit_scale = getattr(config, "logit_scale", 1.0)
289 + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
290 + config.text_config.vocab_size,
291 + logit_scale)
292 + self.sampler = Sampler()
293 +
294 + self.make_empty_intermediate_tensors = (
295 + self.language_model.make_empty_intermediate_tensors)
296 +
297 + def _validate_and_reshape_mm_tensor(self,
298 + mm_input: Union[torch.Tensor,
299 + List[torch.Tensor]],
300 + name: str) -> torch.Tensor:
301 + if not isinstance(mm_input, (torch.Tensor, list)):
302 + raise ValueError(f"Incorrect type of {name}. "
303 + f"Got type: {type(mm_input)}")
304 + if isinstance(mm_input, torch.Tensor):
305 + return torch.concat(list(mm_input))
306 + else:
307 + return torch.concat(mm_input)
308 +
309 + def _parse_and_validate_audio_input(
310 + self, **kwargs: object) -> Optional[Qwen2AudioInputs]:
311 + input_features = kwargs.pop('input_features', None)
312 + feature_attention_mask = kwargs.pop('feature_attention_mask', None)
313 + if input_features is None:
314 + return None
315 + input_features = self._validate_and_reshape_mm_tensor(
316 + input_features, 'input_features')
317 + feature_attention_mask = self._validate_and_reshape_mm_tensor(
318 + feature_attention_mask, 'feature_attention_mask')
319 + if not isinstance(input_features, (torch.Tensor, list)):
320 + raise ValueError("Incorrect type of audio input features. "
321 + f"Got type: {type(input_features)}")
322 + return Qwen2AudioInputs(input_features=input_features,
323 + feature_attention_mask=feature_attention_mask)
324 +
325 + def _process_audio_input(self,
326 + audio_input: Qwen2AudioInputs) -> torch.Tensor:
327 +
328 + input_features = audio_input["input_features"]
329 + feature_attention_mask = audio_input["feature_attention_mask"]
330 +
331 + audio_feat_lengths, audio_output_lengths = (
332 + self.audio_tower._get_feat_extract_output_lengths(
333 + feature_attention_mask.sum(-1)))
334 +
335 + batch_size, _, max_mel_seq_len = input_features.shape
336 + max_seq_len = (max_mel_seq_len - 2) // 2 + 1
337 + # Create a sequence tensor of shape (batch_size, max_seq_len)
338 + seq_range = (torch.arange(
339 + 0,
340 + max_seq_len,
341 + dtype=audio_feat_lengths.dtype,
342 + device=audio_feat_lengths.device).unsqueeze(0).expand(
343 + batch_size, max_seq_len))
344 + lengths_expand = audio_feat_lengths.unsqueeze(-1).expand(
345 + batch_size, max_seq_len)
346 + # Create mask
347 + padding_mask = seq_range >= lengths_expand
348 +
349 + audio_attention_mask_ = padding_mask.view(
350 + batch_size, 1, 1, max_seq_len).expand(batch_size, 1, max_seq_len,
351 + max_seq_len)
352 + audio_attention_mask = audio_attention_mask_.to(
353 + dtype=self.audio_tower.conv1.weight.dtype,
354 + device=self.audio_tower.conv1.weight.device)
355 + audio_attention_mask[audio_attention_mask_] = float("-inf")
356 +
357 + audio_outputs = self.audio_tower(input_features,
358 + attention_mask=audio_attention_mask)
359 + selected_audio_feature = audio_outputs.last_hidden_state
360 + audio_features = self.multi_modal_projector(selected_audio_feature)
361 + num_audios, max_audio_tokens, embed_dim = audio_features.shape
362 + audio_features_mask = torch.arange(max_audio_tokens).expand(
363 + num_audios, max_audio_tokens
364 + ).to(audio_output_lengths.device) < audio_output_lengths.unsqueeze(1)
365 + masked_audio_features = audio_features[audio_features_mask].view(
366 + -1, embed_dim)
367 +
368 + return masked_audio_features
369 +
370 + def forward(
371 + self,
372 + input_ids: torch.Tensor,
373 + positions: torch.Tensor,
374 + kv_caches: List[torch.Tensor],
375 + attn_metadata: AttentionMetadata,
376 + intermediate_tensors: Optional[IntermediateTensors] = None,
377 + **kwargs: object,
378 + ) -> Union[torch.Tensor, IntermediateTensors]:
379 + if intermediate_tensors is not None:
380 + input_ids = None
381 + inputs_embeds = None
382 + else:
383 + audio_input = self._parse_and_validate_audio_input(**kwargs)
384 +
385 + if audio_input is None:
386 + inputs_embeds = None
387 + else:
388 + inputs_embeds = self.language_model.embed_tokens(input_ids)
389 + masked_audio_features = self._process_audio_input(audio_input)
390 + # merge llm embeddings and audio features
391 + mask = (input_ids == self.config.audio_token_index)
392 + inputs_embeds[mask, :] = masked_audio_features
393 +
394 + input_ids = None
395 +
396 + hidden_states = self.language_model(
397 + input_ids=input_ids,
398 + positions=positions,
399 + kv_caches=kv_caches,
400 + attn_metadata=attn_metadata,
401 + intermediate_tensors=intermediate_tensors,
402 + inputs_embeds=inputs_embeds,
403 + )
404 + return hidden_states
405 +
406 + def compute_logits(self, hidden_states: torch.Tensor,
407 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
408 + logits = self.logits_processor(self.lm_head, hidden_states,
409 + sampling_metadata)
410 + return logits
411 +
412 + def sample(
413 + self,
414 + logits: torch.Tensor,
415 + sampling_metadata: SamplingMetadata,
416 + ) -> Optional[SamplerOutput]:
417 + next_tokens = self.sampler(logits, sampling_metadata)
418 + return next_tokens
419 +
420 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
421 + stacked_params_mapping = [
422 + # (param_name, shard_name, shard_id)
423 + ("qkv_proj", "q_proj", "q"),
424 + ("qkv_proj", "k_proj", "k"),
425 + ("qkv_proj", "v_proj", "v"),
426 + ("gate_up_proj", "gate_proj", 0),
427 + ("gate_up_proj", "up_proj", 1),
428 + ]
429 + params_dict = dict(self.named_parameters(remove_duplicate=False))
430 + for name, loaded_weight in weights:
431 + if "rotary_emb.inv_freq" in name:
432 + continue
433 + if (self.config.text_config.tie_word_embeddings
434 + and "lm_head.weight" in name):
435 + continue
436 + for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
437 + if key_to_modify in name:
438 + name = name.replace(key_to_modify, new_key)
439 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
440 + if weight_name not in name or 'audio' in name:
441 + continue
442 + name = name.replace(weight_name, param_name)
443 + # Skip loading extra bias for GPTQ models.
444 + if name.endswith(".bias") and name not in params_dict:
445 + continue
446 + param = params_dict[name]
447 + weight_loader = param.weight_loader
448 + weight_loader(param, loaded_weight, shard_id)
449 + break
450 + else:
451 + # Skip loading extra bias for GPTQ models.
452 + if name.endswith(".bias") and name not in params_dict:
453 + continue
454 + # Remapping the name of FP8 kv-scale.
455 + name = maybe_remap_kv_scale_name(name, params_dict)
456 + if name is None:
457 + continue
458 +
459 + param = params_dict[name]
460 + weight_loader = getattr(param, "weight_loader",
461 + default_weight_loader)
462 + weight_loader(param, loaded_weight)
vllm/model_executor/models/registry.py
+1
@@ -121,6 +121,7 @@ _MULTIMODAL_MODELS = {
121 "PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"), # noqa: E501
122 "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
123 "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"), # noqa: E501
124 + "Qwen2AudioForConditionalGeneration": ("qwen2_audio", "Qwen2AudioForConditionalGeneration"), # noqa: E501
125 "UltravoxModel": ("ultravox", "UltravoxModel"),
126 # [Encoder-decoder]
127 "MllamaForConditionalGeneration": ("mllama", "MllamaForConditionalGeneration"), # noqa: E501
vllm/model_executor/models/ultravox.py
+3
@@ -117,6 +117,9 @@ def input_mapper_for_ultravox(ctx: InputContext, data: object):
117 if not isinstance(data, list):
118 data = [data]
119
120 + if len(data) == 0:
121 + return MultiModalInputs()
122 +
123 # If the audio inputs are embeddings, no need for preprocessing
124 if is_list_of(data, torch.Tensor, check="all"):
125 return MultiModalInputs({"audio_embeds": data})