A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Model] Add Qwen2-Audio model support (#9248)
Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
Yunfei Chu committed
Oct 24, 2024 at 01:54 UTC
fc6c27462614924dca90898ef762d6c56c0874ba
7 files changed
+515
-17
docs/source/models/supported_models.rst
+6
@@ -459,6 +459,12 @@ Text Generation
459
- :code:`Qwen/Qwen-VL`, :code:`Qwen/Qwen-VL-Chat`, etc.
460
-
461
- ✅︎
462
+ * - :code:`Qwen2AudioForConditionalGeneration`
463
+ - Qwen2-Audio
464
+ - T + A\ :sup:`+`
465
+ - :code:`Qwen/Qwen2-Audio-7B-Instruct`
466
+ -
467
+ - ✅︎
468
* - :code:`Qwen2VLForConditionalGeneration`
469
- Qwen2-VL
470
- T + I\ :sup:`E+` + V\ :sup:`+`
examples/offline_inference_audio_language.py
+38
-16
@@ -12,14 +12,15 @@ from vllm.assets.audio import AudioAsset
12
from vllm.utils import FlexibleArgumentParser
13
14
audio_assets = [AudioAsset("mary_had_lamb"), AudioAsset("winning_call")]
15
-question_per_audio_count = [
16
- "What is recited in the audio?",
17
- "What sport and what nursery rhyme are referenced?"
18
-]
15
+question_per_audio_count = {
16
+ 0: "What is 1+1?",
17
+ 1: "What is recited in the audio?",
18
+ 2: "What sport and what nursery rhyme are referenced?"
19
+}
20
21
22
# Ultravox 0.3
22
-def run_ultravox(question, audio_count):
23
+def run_ultravox(question: str, audio_count: int):
24
model_name = "fixie-ai/ultravox-v0_3"
25
26
tokenizer = AutoTokenizer.from_pretrained(model_name)
@@ -42,9 +43,29 @@ def run_ultravox(question, audio_count):
43
return llm, prompt, stop_token_ids
44
45
45
-model_example_map = {
46
- "ultravox": run_ultravox,
47
-}
46
+# Qwen2-Audio
47
+def run_qwen2_audio(question: str, audio_count: int):
48
+ model_name = "Qwen/Qwen2-Audio-7B-Instruct"
49
+
50
+ llm = LLM(model=model_name,
51
+ max_model_len=4096,
52
+ max_num_seqs=5,
53
+ limit_mm_per_prompt={"audio": audio_count})
54
+
55
+ audio_in_prompt = "".join([
56
+ f"Audio {idx+1}: "
57
+ f"<|audio_bos|><|AUDIO|><|audio_eos|>\n" for idx in range(audio_count)
58
+ ])
59
+
60
+ prompt = ("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
61
+ "<|im_start|>user\n"
62
+ f"{audio_in_prompt}{question}<|im_end|>\n"
63
+ "<|im_start|>assistant\n")
64
+ stop_token_ids = None
65
+ return llm, prompt, stop_token_ids
66
+
67
+
68
+model_example_map = {"ultravox": run_ultravox, "qwen2_audio": run_qwen2_audio}
69
70
71
def main(args):
@@ -54,7 +75,7 @@ def main(args):
75
76
audio_count = args.num_audios
77
llm, prompt, stop_token_ids = model_example_map[model](
57
- question_per_audio_count[audio_count - 1], audio_count)
78
+ question_per_audio_count[audio_count], audio_count)
79
80
# We set temperature to 0.2 so that outputs can be different
81
# even when all prompts are identical when running batch inference.
@@ -62,16 +83,17 @@ def main(args):
83
max_tokens=64,
84
stop_token_ids=stop_token_ids)
85
65
- assert args.num_prompts > 0
66
- inputs = {
67
- "prompt": prompt,
68
- "multi_modal_data": {
86
+ mm_data = {}
87
+ if audio_count > 0:
88
+ mm_data = {
89
"audio": [
90
asset.audio_and_sample_rate
91
for asset in audio_assets[:audio_count]
92
]
73
- },
74
- }
93
+ }
94
+
95
+ assert args.num_prompts > 0
96
+ inputs = {"prompt": prompt, "multi_modal_data": mm_data}
97
if args.num_prompts > 1:
98
# Batch inference
99
inputs = [inputs] * args.num_prompts
@@ -100,7 +122,7 @@ if __name__ == "__main__":
122
parser.add_argument("--num-audios",
123
type=int,
124
default=1,
103
- choices=[1, 2],
125
+ choices=[0, 1, 2],
126
help="Number of audio items per prompt.")
127
128
args = parser.parse_args()
tests/distributed/test_pipeline_parallel.py
+1
@@ -199,6 +199,7 @@ MULTIMODAL_MODEL_SETTINGS = {
199
"microsoft/Phi-3-vision-128k-instruct": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
200
"mistralai/Pixtral-12B-2409": PPTestSettings.fast(tp_base=2, tokenizer_mode="mistral"), # noqa: E501
201
"Qwen/Qwen-VL-Chat": PPTestSettings.fast(trust_remote_code=True),
202
+ "Qwen/Qwen2-Audio-7B-Instruct": PPTestSettings.fast(),
203
"Qwen/Qwen2-VL-2B-Instruct": PPTestSettings.fast(),
204
"fixie-ai/ultravox-v0_3": PPTestSettings.fast(),
205
}
vllm/entrypoints/chat_utils.py
+4
-1
@@ -196,7 +196,10 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
196
elif modality == "audio":
197
if model_type == "ultravox":
198
return "<|reserved_special_token_0|>"
199
- raise TypeError(f"Unknown {modality} model type: {model_type}")
199
+ if model_type == "qwen2_audio":
200
+ return (f"Audio {current_count}: "
201
+ f"<|audio_bos|><|AUDIO|><|audio_eos|>")
202
+ raise TypeError(f"Unknown model type: {model_type}")
203
elif modality == "video":
204
if model_type == "qwen2_vl":
205
return "<|vision_start|><|video_pad|><|vision_end|>"
vllm/model_executor/models/qwen2_audio.py
new
+462
@@ -0,0 +1,462 @@
1
+# coding=utf-8
2
+# Copyright 2024 The Qwen team.
3
+# Copyright 2023 The vLLM team.
4
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
5
+#
6
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
7
+# and OPT implementations in this library. It has been modified from its
8
+# original forms to accommodate minor architectural differences compared
9
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
10
+#
11
+# Licensed under the Apache License, Version 2.0 (the "License");
12
+# you may not use this file except in compliance with the License.
13
+# You may obtain a copy of the License at
14
+#
15
+# http://www.apache.org/licenses/LICENSE-2.0
16
+#
17
+# Unless required by applicable law or agreed to in writing, software
18
+# distributed under the License is distributed on an "AS IS" BASIS,
19
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20
+# See the License for the specific language governing permissions and
21
+# limitations under the License.
22
+"""Inference-only Qwen2-Audio model compatible with HuggingFace weights."""
23
+from functools import lru_cache
24
+from typing import Iterable, List, Mapping, Optional, Tuple, TypedDict, Union
25
+
26
+import librosa
27
+import numpy as np
28
+import torch
29
+import torch.nn as nn
30
+from transformers import Qwen2AudioConfig, Qwen2AudioEncoder
31
+
32
+from vllm.attention import AttentionMetadata
33
+from vllm.config import CacheConfig, MultiModalConfig
34
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
35
+ token_inputs)
36
+from vllm.logger import init_logger
37
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
38
+from vllm.model_executor.layers.quantization.base_config import (
39
+ QuantizationConfig)
40
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
41
+from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
42
+from vllm.model_executor.model_loader.weight_utils import (
43
+ default_weight_loader, maybe_remap_kv_scale_name)
44
+from vllm.model_executor.models.qwen2 import Qwen2Model
45
+from vllm.model_executor.sampling_metadata import SamplingMetadata
46
+from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalInputs
47
+from vllm.sequence import IntermediateTensors, SequenceData
48
+
49
+from .interfaces import SupportsMultiModal, SupportsPP
50
+
51
+logger = init_logger(__name__)
52
+
53
+_KEYS_TO_MODIFY_MAPPING = {
54
+ "language_model.lm_head": "lm_head",
55
+ "language_model.model": "language_model",
56
+}
57
+
58
+
59
+# # === Audio Inputs === #
60
+class Qwen2AudioInputs(TypedDict):
61
+ input_features: torch.Tensor
62
+ """Shape:
63
+ `(num_audios, num_mel_bins, 3000)`
64
+ """
65
+
66
+ feature_attention_mask: torch.Tensor
67
+ """Shape: `(num_audios, 3000)`
68
+ """
69
+
70
+
71
+# === Audio Encoder === #
72
+
73
+
74
+class Qwen2AudioMultiModalProjector(nn.Module):
75
+
76
+ def __init__(self, audio_hidden_size: int, text_hidden_size: int):
77
+ super().__init__()
78
+ self.linear = nn.Linear(audio_hidden_size, text_hidden_size, bias=True)
79
+
80
+ def forward(self, audio_features):
81
+ hidden_states = self.linear(audio_features)
82
+ return hidden_states
83
+
84
+
85
+def dummy_data_for_qwen2_audio(ctx: InputContext, seq_len: int,
86
+ mm_counts: Mapping[str, int]):
87
+ num_audios = mm_counts["audio"]
88
+ max_llm_audio_tokens = get_max_qwen2_audio_audio_tokens(ctx) * num_audios
89
+ if seq_len - max_llm_audio_tokens - 2 < 0:
90
+ raise RuntimeError(
91
+ f"Qwen2-Audio cannot process {num_audios} audios in a prompt, "
92
+ "please increase max_model_len or reduce audio limit by "
93
+ "--limit-mm-per-prompt.")
94
+
95
+ audio_token_index = ctx.model_config.hf_config.audio_token_index
96
+
97
+ dummy_seqdata = SequenceData.from_prompt_token_counts(
98
+ (audio_token_index, max_llm_audio_tokens),
99
+ (0, seq_len - max_llm_audio_tokens),
100
+ )
101
+ dummy_audio = np.full((max_llm_audio_tokens * 2 * 2 * 160, ), 0.)
102
+ return dummy_seqdata, {"audio": [(dummy_audio, 16000)] * num_audios}
103
+
104
+
105
+def get_processor(
106
+ processor_name: str,
107
+ *args,
108
+ trust_remote_code: bool = False,
109
+ **kwargs,
110
+):
111
+ """Gets a processor for the given model name via HuggingFace.
112
+
113
+ Derived from `vllm.transformers_utils.image_processor.get_image_processor`.
114
+ """
115
+ # don't put this import at the top level
116
+ # it will call torch.cuda.device_count()
117
+ from transformers import AutoProcessor
118
+
119
+ try:
120
+ processor = AutoProcessor.from_pretrained(
121
+ processor_name,
122
+ *args,
123
+ trust_remote_code=trust_remote_code,
124
+ **kwargs)
125
+ except ValueError as e:
126
+ # If the error pertains to the processor class not existing or not
127
+ # currently being imported, suggest using the --trust-remote-code flag.
128
+ # Unlike AutoTokenizer, AutoProcessor does not separate such errors
129
+ if not trust_remote_code:
130
+ err_msg = (
131
+ "Failed to load the processor. If the processor is "
132
+ "a custom processor not yet available in the HuggingFace "
133
+ "transformers library, consider setting "
134
+ "`trust_remote_code=True` in LLM or using the "
135
+ "`--trust-remote-code` flag in the CLI.")
136
+ raise RuntimeError(err_msg) from e
137
+ else:
138
+ raise e
139
+
140
+ return processor
141
+
142
+
143
+cached_get_processor = lru_cache(get_processor)
144
+
145
+
146
+def _get_feat_extract_output_lengths(input_lengths: torch.LongTensor):
147
+ """
148
+ Computes the output length of the convolutional layers
149
+ and the output length of the audio encoder
150
+ """
151
+ input_lengths = (input_lengths - 1) // 2 + 1
152
+ output_lengths = (input_lengths - 2) // 2 + 1
153
+ return input_lengths, output_lengths
154
+
155
+
156
+def get_max_qwen2_audio_audio_tokens(ctx: InputContext) -> int:
157
+ max_source_position = (
158
+ ctx.model_config.hf_config.audio_config.max_source_positions)
159
+ output_lengths = (max_source_position - 2) // 2 + 1
160
+ return output_lengths
161
+
162
+
163
+def input_processor_for_qwen2_audio(
164
+ ctx: InputContext, inputs: DecoderOnlyInputs) -> DecoderOnlyInputs:
165
+ multi_modal_data = inputs.get("multi_modal_data")
166
+ if multi_modal_data is None or "audio" not in multi_modal_data:
167
+ return inputs
168
+
169
+ audios = multi_modal_data["audio"]
170
+ if not isinstance(audios, list):
171
+ audios = [audios]
172
+
173
+ if len(audios) == 0:
174
+ return inputs
175
+
176
+ processor = cached_get_processor(ctx.model_config.model)
177
+ resampled_audios = [
178
+ librosa.resample(audio,
179
+ orig_sr=sampling_rate,
180
+ target_sr=processor.feature_extractor.sampling_rate)
181
+ for audio, sampling_rate in audios
182
+ ]
183
+ audio_input_lengths = np.array(
184
+ [min(3000, _.shape[0] // 160 + 1) for _ in resampled_audios])
185
+
186
+ audio_feat_lengths, audio_output_lengths = _get_feat_extract_output_lengths(
187
+ audio_input_lengths)
188
+
189
+ audio_token_index = ctx.model_config.hf_config.audio_token_index
190
+
191
+ input_ids = inputs['prompt_token_ids']
192
+
193
+ new_input_ids = []
194
+ audio_num = input_ids.count(audio_token_index)
195
+ assert len(audio_input_lengths) == audio_num, \
196
+ (f'The text input contains {audio_num} audio tokens, '
197
+ f'but {len(audio_input_lengths)} audios provided')
198
+ start = 0
199
+ for audio_idx in range(audio_num):
200
+ end = input_ids.index(audio_token_index, start)
201
+ new_input_ids.extend(input_ids[start:end]) # text part
202
+
203
+ new_input_ids.extend([audio_token_index] *
204
+ audio_output_lengths[audio_idx])
205
+ start = end + 1
206
+ new_input_ids.extend(input_ids[start:])
207
+
208
+ return token_inputs(
209
+ prompt_token_ids=new_input_ids,
210
+ prompt=inputs['prompt'],
211
+ multi_modal_data=multi_modal_data,
212
+ )
213
+
214
+
215
+def input_mapper_for_qwen2_audio(
216
+ ctx: InputContext,
217
+ multi_modal_data: Union[np.ndarray, List[np.ndarray]],
218
+) -> MultiModalInputs:
219
+ """Input mapper for Qwen2-Audio."""
220
+ if not isinstance(multi_modal_data, list):
221
+ multi_modal_data = [multi_modal_data]
222
+
223
+ if len(multi_modal_data) == 0:
224
+ return MultiModalInputs()
225
+
226
+ processor = cached_get_processor(ctx.model_config.model)
227
+ audio_feature_extractor = processor.feature_extractor
228
+ if audio_feature_extractor is None:
229
+ raise RuntimeError(
230
+ "No HuggingFace audio_feature_extractor is available "
231
+ "to process the audio object")
232
+
233
+ try:
234
+ resampled_audios = [
235
+ librosa.resample(
236
+ audio,
237
+ orig_sr=sampling_rate,
238
+ target_sr=processor.feature_extractor.sampling_rate)
239
+ for audio, sampling_rate in multi_modal_data
240
+ ]
241
+ batch_data = audio_feature_extractor(resampled_audios,
242
+ sampling_rate=16000,
243
+ return_attention_mask=True,
244
+ padding="max_length",
245
+ return_tensors="pt").data
246
+ batch_data["feature_attention_mask"] = batch_data.pop("attention_mask")
247
+ except Exception:
248
+ logger.error("Failed to process audio (%s)", multi_modal_data)
249
+ raise
250
+
251
+ return MultiModalInputs(batch_data)
252
+
253
+
254
+@INPUT_REGISTRY.register_dummy_data(dummy_data_for_qwen2_audio)
255
+@INPUT_REGISTRY.register_input_processor(input_processor_for_qwen2_audio)
256
+@MULTIMODAL_REGISTRY.register_input_mapper("audio",
257
+ input_mapper_for_qwen2_audio)
258
+@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
259
+ "audio", get_max_qwen2_audio_audio_tokens)
260
+class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
261
+ SupportsPP):
262
+
263
+ def __init__(self,
264
+ config: Qwen2AudioConfig,
265
+ multimodal_config: MultiModalConfig,
266
+ cache_config: Optional[CacheConfig] = None,
267
+ quant_config: Optional[QuantizationConfig] = None) -> None:
268
+ super().__init__()
269
+
270
+ self.config = config
271
+ self.multimodal_config = multimodal_config
272
+
273
+ self.audio_tower = Qwen2AudioEncoder(config.audio_config)
274
+ self.multi_modal_projector = Qwen2AudioMultiModalProjector(
275
+ config.audio_config.d_model, config.text_config.hidden_size)
276
+
277
+ self.quant_config = quant_config
278
+
279
+ self.language_model = Qwen2Model(config.text_config, cache_config,
280
+ quant_config)
281
+ self.unpadded_vocab_size = config.text_config.vocab_size
282
+ if config.text_config.tie_word_embeddings:
283
+ self.lm_head = self.language_model.embed_tokens
284
+ else:
285
+ self.lm_head = ParallelLMHead(config.text_config.vocab_size,
286
+ config.text_config.hidden_size,
287
+ quant_config=quant_config)
288
+ logit_scale = getattr(config, "logit_scale", 1.0)
289
+ self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
290
+ config.text_config.vocab_size,
291
+ logit_scale)
292
+ self.sampler = Sampler()
293
+
294
+ self.make_empty_intermediate_tensors = (
295
+ self.language_model.make_empty_intermediate_tensors)
296
+
297
+ def _validate_and_reshape_mm_tensor(self,
298
+ mm_input: Union[torch.Tensor,
299
+ List[torch.Tensor]],
300
+ name: str) -> torch.Tensor:
301
+ if not isinstance(mm_input, (torch.Tensor, list)):
302
+ raise ValueError(f"Incorrect type of {name}. "
303
+ f"Got type: {type(mm_input)}")
304
+ if isinstance(mm_input, torch.Tensor):
305
+ return torch.concat(list(mm_input))
306
+ else:
307
+ return torch.concat(mm_input)
308
+
309
+ def _parse_and_validate_audio_input(
310
+ self, **kwargs: object) -> Optional[Qwen2AudioInputs]:
311
+ input_features = kwargs.pop('input_features', None)
312
+ feature_attention_mask = kwargs.pop('feature_attention_mask', None)
313
+ if input_features is None:
314
+ return None
315
+ input_features = self._validate_and_reshape_mm_tensor(
316
+ input_features, 'input_features')
317
+ feature_attention_mask = self._validate_and_reshape_mm_tensor(
318
+ feature_attention_mask, 'feature_attention_mask')
319
+ if not isinstance(input_features, (torch.Tensor, list)):
320
+ raise ValueError("Incorrect type of audio input features. "
321
+ f"Got type: {type(input_features)}")
322
+ return Qwen2AudioInputs(input_features=input_features,
323
+ feature_attention_mask=feature_attention_mask)
324
+
325
+ def _process_audio_input(self,
326
+ audio_input: Qwen2AudioInputs) -> torch.Tensor:
327
+
328
+ input_features = audio_input["input_features"]
329
+ feature_attention_mask = audio_input["feature_attention_mask"]
330
+
331
+ audio_feat_lengths, audio_output_lengths = (
332
+ self.audio_tower._get_feat_extract_output_lengths(
333
+ feature_attention_mask.sum(-1)))
334
+
335
+ batch_size, _, max_mel_seq_len = input_features.shape
336
+ max_seq_len = (max_mel_seq_len - 2) // 2 + 1
337
+ # Create a sequence tensor of shape (batch_size, max_seq_len)
338
+ seq_range = (torch.arange(
339
+ 0,
340
+ max_seq_len,
341
+ dtype=audio_feat_lengths.dtype,
342
+ device=audio_feat_lengths.device).unsqueeze(0).expand(
343
+ batch_size, max_seq_len))
344
+ lengths_expand = audio_feat_lengths.unsqueeze(-1).expand(
345
+ batch_size, max_seq_len)
346
+ # Create mask
347
+ padding_mask = seq_range >= lengths_expand
348
+
349
+ audio_attention_mask_ = padding_mask.view(
350
+ batch_size, 1, 1, max_seq_len).expand(batch_size, 1, max_seq_len,
351
+ max_seq_len)
352
+ audio_attention_mask = audio_attention_mask_.to(
353
+ dtype=self.audio_tower.conv1.weight.dtype,
354
+ device=self.audio_tower.conv1.weight.device)
355
+ audio_attention_mask[audio_attention_mask_] = float("-inf")
356
+
357
+ audio_outputs = self.audio_tower(input_features,
358
+ attention_mask=audio_attention_mask)
359
+ selected_audio_feature = audio_outputs.last_hidden_state
360
+ audio_features = self.multi_modal_projector(selected_audio_feature)
361
+ num_audios, max_audio_tokens, embed_dim = audio_features.shape
362
+ audio_features_mask = torch.arange(max_audio_tokens).expand(
363
+ num_audios, max_audio_tokens
364
+ ).to(audio_output_lengths.device) < audio_output_lengths.unsqueeze(1)
365
+ masked_audio_features = audio_features[audio_features_mask].view(
366
+ -1, embed_dim)
367
+
368
+ return masked_audio_features
369
+
370
+ def forward(
371
+ self,
372
+ input_ids: torch.Tensor,
373
+ positions: torch.Tensor,
374
+ kv_caches: List[torch.Tensor],
375
+ attn_metadata: AttentionMetadata,
376
+ intermediate_tensors: Optional[IntermediateTensors] = None,
377
+ **kwargs: object,
378
+ ) -> Union[torch.Tensor, IntermediateTensors]:
379
+ if intermediate_tensors is not None:
380
+ input_ids = None
381
+ inputs_embeds = None
382
+ else:
383
+ audio_input = self._parse_and_validate_audio_input(**kwargs)
384
+
385
+ if audio_input is None:
386
+ inputs_embeds = None
387
+ else:
388
+ inputs_embeds = self.language_model.embed_tokens(input_ids)
389
+ masked_audio_features = self._process_audio_input(audio_input)
390
+ # merge llm embeddings and audio features
391
+ mask = (input_ids == self.config.audio_token_index)
392
+ inputs_embeds[mask, :] = masked_audio_features
393
+
394
+ input_ids = None
395
+
396
+ hidden_states = self.language_model(
397
+ input_ids=input_ids,
398
+ positions=positions,
399
+ kv_caches=kv_caches,
400
+ attn_metadata=attn_metadata,
401
+ intermediate_tensors=intermediate_tensors,
402
+ inputs_embeds=inputs_embeds,
403
+ )
404
+ return hidden_states
405
+
406
+ def compute_logits(self, hidden_states: torch.Tensor,
407
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
408
+ logits = self.logits_processor(self.lm_head, hidden_states,
409
+ sampling_metadata)
410
+ return logits
411
+
412
+ def sample(
413
+ self,
414
+ logits: torch.Tensor,
415
+ sampling_metadata: SamplingMetadata,
416
+ ) -> Optional[SamplerOutput]:
417
+ next_tokens = self.sampler(logits, sampling_metadata)
418
+ return next_tokens
419
+
420
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
421
+ stacked_params_mapping = [
422
+ # (param_name, shard_name, shard_id)
423
+ ("qkv_proj", "q_proj", "q"),
424
+ ("qkv_proj", "k_proj", "k"),
425
+ ("qkv_proj", "v_proj", "v"),
426
+ ("gate_up_proj", "gate_proj", 0),
427
+ ("gate_up_proj", "up_proj", 1),
428
+ ]
429
+ params_dict = dict(self.named_parameters(remove_duplicate=False))
430
+ for name, loaded_weight in weights:
431
+ if "rotary_emb.inv_freq" in name:
432
+ continue
433
+ if (self.config.text_config.tie_word_embeddings
434
+ and "lm_head.weight" in name):
435
+ continue
436
+ for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
437
+ if key_to_modify in name:
438
+ name = name.replace(key_to_modify, new_key)
439
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
440
+ if weight_name not in name or 'audio' in name:
441
+ continue
442
+ name = name.replace(weight_name, param_name)
443
+ # Skip loading extra bias for GPTQ models.
444
+ if name.endswith(".bias") and name not in params_dict:
445
+ continue
446
+ param = params_dict[name]
447
+ weight_loader = param.weight_loader
448
+ weight_loader(param, loaded_weight, shard_id)
449
+ break
450
+ else:
451
+ # Skip loading extra bias for GPTQ models.
452
+ if name.endswith(".bias") and name not in params_dict:
453
+ continue
454
+ # Remapping the name of FP8 kv-scale.
455
+ name = maybe_remap_kv_scale_name(name, params_dict)
456
+ if name is None:
457
+ continue
458
+
459
+ param = params_dict[name]
460
+ weight_loader = getattr(param, "weight_loader",
461
+ default_weight_loader)
462
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/registry.py
+1
@@ -121,6 +121,7 @@ _MULTIMODAL_MODELS = {
121
"PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"), # noqa: E501
122
"QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
123
"Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"), # noqa: E501
124
+ "Qwen2AudioForConditionalGeneration": ("qwen2_audio", "Qwen2AudioForConditionalGeneration"), # noqa: E501
125
"UltravoxModel": ("ultravox", "UltravoxModel"),
126
# [Encoder-decoder]
127
"MllamaForConditionalGeneration": ("mllama", "MllamaForConditionalGeneration"), # noqa: E501
vllm/model_executor/models/ultravox.py
+3
@@ -117,6 +117,9 @@ def input_mapper_for_ultravox(ctx: InputContext, data: object):
117
if not isinstance(data, list):
118
data = [data]
119
120
+ if len(data) == 0:
121
+ return MultiModalInputs()
122
+
123
# If the audio inputs are embeddings, no need for preprocessing
124
if is_list_of(data, torch.Tensor, check="all"):
125
return MultiModalInputs({"audio_embeds": data})