@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8b3f0a99dd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 9, 10:00 UTC

[Models] Qwen3-ASR (#33312)

Signed-off-by: Roger Wang <hey@rogerw.io>

Roger Wang committed Jan 29, 2026 at 03:27 UTC 8b3f0a99dd501447e3f48ef675e51590e993a616
9 files changed +1269
docs/models/supported_models.md
+2
@@ -720,6 +720,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
720 | `Qwen3VLForConditionalGeneration` | Qwen3-VL | T + I<sup>E+</sup> + V<sup>E+</sup> | `Qwen/Qwen3-VL-4B-Instruct`, etc. | ✅︎ | ✅︎ |
721 | `Qwen3VLMoeForConditionalGeneration` | Qwen3-VL-MOE | T + I<sup>E+</sup> + V<sup>E+</sup> | `Qwen/Qwen3-VL-30B-A3B-Instruct`, etc. | ✅︎ | ✅︎ |
722 | `Qwen3OmniMoeThinkerForConditionalGeneration` | Qwen3-Omni | T + I<sup>E+</sup> + V<sup>E+</sup> + A<sup>+</sup> | `Qwen/Qwen3-Omni-30B-A3B-Instruct`, `Qwen/Qwen3-Omni-30B-A3B-Thinking` | ✅︎ | ✅︎ |
723 +| `Qwen3ASRForConditionalGeneration` | Qwen3-ASR | T + A<sup>+</sup> | `Qwen/Qwen3-ASR-1.7B` | ✅︎ | ✅︎ |
724 | `RForConditionalGeneration` | R-VL-4B | T + I<sup>E+</sup> | `YannQi/R-4B` | | ✅︎ |
725 | `SkyworkR1VChatModel` | Skywork-R1V-38B | T + I | `Skywork/Skywork-R1V-38B` | | ✅︎ |
726 | `SmolVLMForConditionalGeneration` | SmolVLM2 | T + I | `SmolVLM2-2.2B-Instruct` | ✅︎ | |
@@ -769,6 +770,7 @@ Speech2Text models trained specifically for Automatic Speech Recognition.
770 | `Gemma3nForConditionalGeneration` | Gemma3n | `google/gemma-3n-E2B-it`, `google/gemma-3n-E4B-it`, etc. | | |
771 | `GlmAsrForConditionalGeneration` | GLM-ASR | `zai-org/GLM-ASR-Nano-2512` | ✅︎ | ✅︎ |
772 | `GraniteSpeechForConditionalGeneration` | Granite Speech | `ibm-granite/granite-speech-3.3-2b`, `ibm-granite/granite-speech-3.3-8b`, etc. | ✅︎ | ✅︎ |
773 +| `Qwen3ASRForConditionalGeneration` | Qwen3-ASR | `Qwen/Qwen3-ASR-1.7B`, etc. | | ✅︎ |
774 | `VoxtralForConditionalGeneration` | Voxtral (Mistral format) | `mistralai/Voxtral-Mini-3B-2507`, `mistralai/Voxtral-Small-24B-2507`, etc. | ✅︎ | ✅︎ |
775 | `WhisperForConditionalGeneration` | Whisper | `openai/whisper-small`, `openai/whisper-large-v3-turbo`, etc. | | |
776
examples/offline_inference/audio_language.py
+20
@@ -330,6 +330,25 @@ def run_qwen2_5_omni(question: str, audio_count: int):
330 )
331
332
333 +def run_qwen3_asr(question: str, audio_count: int) -> ModelRequestData:
334 + model_name = "Qwen/Qwen3-Asr-1.7B"
335 +
336 + audio_in_prompt = "<|audio_start|><|audio_pad|><|audio_end|>\n" * audio_count
337 + prompt = f"<|im_start|>user\n{audio_in_prompt}<|im_end|>\n<|im_start|>assistant\n"
338 +
339 + engine_args = EngineArgs(
340 + model=model_name,
341 + max_model_len=4096,
342 + max_num_seqs=5,
343 + limit_mm_per_prompt={"audio": audio_count},
344 + )
345 +
346 + return ModelRequestData(
347 + engine_args=engine_args,
348 + prompt=prompt,
349 + )
350 +
351 +
352 # Ultravox 0.5-1B
353 def run_ultravox(question: str, audio_count: int) -> ModelRequestData:
354 model_name = "fixie-ai/ultravox-v0_5-llama-3_2-1b"
@@ -442,6 +461,7 @@ model_example_map = {
461 "phi4_mm": run_phi4mm,
462 "qwen2_audio": run_qwen2_audio,
463 "qwen2_5_omni": run_qwen2_5_omni,
464 + "qwen3_asr": run_qwen3_asr,
465 "ultravox": run_ultravox,
466 "voxtral": run_voxtral,
467 "whisper": run_whisper,
tests/models/registry.py
+6
@@ -944,6 +944,12 @@ _MULTIMODAL_EXAMPLE_MODELS = {
944 max_model_len=4096,
945 min_transformers_version="4.57",
946 ),
947 + "Qwen3ASRForConditionalGeneration": _HfExamplesInfo(
948 + "Qwen/Qwen3-ASR-1.7B",
949 + max_model_len=4096,
950 + min_transformers_version="4.57",
951 + is_available_online=False,
952 + ),
953 "RForConditionalGeneration": _HfExamplesInfo("YannQi/R-4B", trust_remote_code=True),
954 "SkyworkR1VChatModel": _HfExamplesInfo(
955 "Skywork/Skywork-R1V-38B", trust_remote_code=True
vllm/model_executor/models/qwen3_asr.py new
+567
@@ -0,0 +1,567 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +# Copyright 2026 The Qwen team.
4 +# Copyright 2023 The vLLM team.
5 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6 +#
7 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8 +# and OPT implementations in this library. It has been modified from its
9 +# original forms to accommodate minor architectural differences compared
10 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11 +#
12 +# Licensed under the Apache License, Version 2.0 (the "License");
13 +# you may not use this file except in compliance with the License.
14 +# You may obtain a copy of the License at
15 +#
16 +# http://www.apache.org/licenses/LICENSE-2.0
17 +#
18 +# Unless required by applicable law or agreed to in writing, software
19 +# distributed under the License is distributed on an "AS IS" BASIS,
20 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21 +# See the License for the specific language governing permissions and
22 +# limitations under the License.
23 +"""Inference-only Qwen3-ASR model."""
24 +
25 +from collections.abc import Iterable, Mapping, Sequence
26 +from typing import Any, Literal, cast
27 +
28 +import numpy as np
29 +import torch
30 +import torch.nn as nn
31 +from transformers.feature_extraction_utils import BatchFeature
32 +from transformers.models.whisper import WhisperFeatureExtractor
33 +
34 +from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
35 +from vllm.config.multimodal import BaseDummyOptions
36 +from vllm.inputs.data import PromptType
37 +from vllm.logger import init_logger
38 +from vllm.model_executor.models.interfaces import (
39 + MultiModalEmbeddings,
40 + SupportsMRoPE,
41 + SupportsMultiModal,
42 + SupportsPP,
43 + SupportsTranscription,
44 +)
45 +from vllm.model_executor.models.module_mapping import MultiModelKeys
46 +from vllm.model_executor.models.qwen3 import Qwen3ForCausalLM
47 +from vllm.model_executor.models.qwen3_omni_moe_thinker import (
48 + Qwen2_5OmniAudioFeatureInputs,
49 + Qwen3OmniMoeAudioEncoder,
50 + Qwen3OmniMoeThinkerMultiModalProcessor,
51 +)
52 +from vllm.model_executor.models.utils import (
53 + AutoWeightsLoader,
54 + WeightsMapper,
55 + _merge_multimodal_embeddings,
56 + maybe_prefix,
57 +)
58 +from vllm.model_executor.models.whisper import ISO639_1_SUPPORTED_LANGS
59 +from vllm.multimodal import MULTIMODAL_REGISTRY
60 +from vllm.multimodal.inputs import (
61 + AudioItem,
62 + ModalityData,
63 + MultiModalDataDict,
64 + MultiModalFeatureSpec,
65 + MultiModalFieldConfig,
66 + MultiModalKwargsItems,
67 +)
68 +from vllm.multimodal.parse import (
69 + AudioProcessorItems,
70 + DictEmbeddingItems,
71 + ModalityDataItems,
72 + MultiModalDataItems,
73 + MultiModalDataParser,
74 +)
75 +from vllm.multimodal.processing import (
76 + BaseDummyInputsBuilder,
77 + BaseProcessingInfo,
78 + PromptReplacement,
79 + PromptUpdate,
80 +)
81 +from vllm.sequence import IntermediateTensors
82 +from vllm.tokenizers import cached_tokenizer_from_config
83 +from vllm.transformers_utils.configs.qwen3_asr import (
84 + Qwen3ASRConfig,
85 + Qwen3ASRThinkerConfig,
86 +)
87 +from vllm.transformers_utils.processor import cached_processor_from_config
88 +from vllm.transformers_utils.processors.qwen3_asr import (
89 + Qwen3ASRProcessor,
90 +)
91 +
92 +logger = init_logger(__name__)
93 +
94 +
95 +def _get_feat_extract_output_lengths(input_lengths: torch.Tensor):
96 + input_lengths_leave = input_lengths % 100
97 + feat_lengths = (input_lengths_leave - 1) // 2 + 1
98 + output_lengths = (
99 + ((feat_lengths - 1) // 2 + 1 - 1) // 2 + 1 + (input_lengths // 100) * 13
100 + )
101 + return output_lengths
102 +
103 +
104 +class Qwen3ASRProcessingInfo(BaseProcessingInfo):
105 + def get_hf_config(self):
106 + return self.ctx.get_hf_config(Qwen3ASRConfig).thinker_config
107 +
108 + def get_hf_processor(self, **kwargs: object) -> Qwen3ASRProcessor:
109 + processor = self.ctx.get_hf_processor(
110 + Qwen3ASRProcessor,
111 + use_fast=kwargs.pop("use_fast", True),
112 + **kwargs,
113 + )
114 + if not hasattr(processor, "audio_token"):
115 + processor.audio_token = "<|audio_pad|>"
116 + return processor
117 +
118 + def get_feature_extractor(self, **kwargs: object) -> WhisperFeatureExtractor:
119 + hf_processor = self.get_hf_processor(**kwargs)
120 + feature_extractor = hf_processor.feature_extractor
121 + assert isinstance(feature_extractor, WhisperFeatureExtractor)
122 + return feature_extractor
123 +
124 + def get_supported_mm_limits(self) -> Mapping[str, int | None]:
125 + return {"audio": None}
126 +
127 +
128 +class Qwen3ASRDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3ASRProcessingInfo]):
129 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
130 + num_audios = mm_counts.get("audio", 0)
131 +
132 + hf_processor = self.info.get_hf_processor()
133 + audio_token = hf_processor.audio_token
134 +
135 + return audio_token * num_audios
136 +
137 + def get_dummy_mm_data(
138 + self,
139 + seq_len: int,
140 + mm_counts: Mapping[str, int],
141 + mm_options: Mapping[str, BaseDummyOptions] | None = None,
142 + ) -> MultiModalDataDict:
143 + num_audios = mm_counts.get("audio", 0)
144 +
145 + feature_extractor = self.info.get_feature_extractor()
146 +
147 + target_audio_length = (
148 + min(
149 + feature_extractor.chunk_length,
150 + 30,
151 + )
152 + * feature_extractor.sampling_rate
153 + )
154 +
155 + audio_overrides = mm_options.get("audio") if mm_options else None
156 +
157 + return {
158 + "audio": self._get_dummy_audios(
159 + length=target_audio_length,
160 + num_audios=num_audios,
161 + overrides=audio_overrides,
162 + ),
163 + }
164 +
165 +
166 +def _qwen3asr_field_config(hf_inputs: Mapping[str, torch.Tensor]):
167 + audio_feature_lengths = hf_inputs.get("audio_feature_lengths", torch.empty((0,)))
168 + return dict(
169 + input_audio_features=MultiModalFieldConfig.flat_from_sizes(
170 + "audio", audio_feature_lengths, dim=1
171 + ),
172 + feature_attention_mask=MultiModalFieldConfig.batched("audio"),
173 + audio_feature_lengths=MultiModalFieldConfig.batched("audio"),
174 + )
175 +
176 +
177 +class Qwen3ASRMultiModalDataParser(MultiModalDataParser):
178 + def _parse_audio_data(
179 + self,
180 + data: dict[str, torch.Tensor] | ModalityData[AudioItem],
181 + ) -> ModalityDataItems[Any, Any] | None:
182 + if isinstance(data, dict):
183 + return DictEmbeddingItems(
184 + data,
185 + modality="audio",
186 + required_fields={"input_audio_features", "audio_feature_lengths"},
187 + fields_factory=_qwen3asr_field_config,
188 + )
189 +
190 + return super()._parse_audio_data(data)
191 +
192 +
193 +class Qwen3ASRMultiModalProcessor(
194 + Qwen3OmniMoeThinkerMultiModalProcessor,
195 +):
196 + def _get_data_parser(self) -> MultiModalDataParser:
197 + feature_extractor = self.info.get_feature_extractor()
198 + return Qwen3ASRMultiModalDataParser(
199 + target_sr=feature_extractor.sampling_rate,
200 + )
201 +
202 + def _get_mm_fields_config(
203 + self,
204 + hf_inputs: BatchFeature,
205 + hf_processor_mm_kwargs: Mapping[str, object],
206 + ) -> Mapping[str, MultiModalFieldConfig]:
207 + return _qwen3asr_field_config(hf_inputs)
208 +
209 + def _get_prompt_updates(
210 + self,
211 + mm_items: MultiModalDataItems,
212 + hf_processor_mm_kwargs: Mapping[str, Any],
213 + out_mm_kwargs: MultiModalKwargsItems,
214 + ) -> Sequence[PromptUpdate]:
215 + processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
216 + tokenizer = self.info.get_tokenizer()
217 + vocab = tokenizer.get_vocab()
218 +
219 + audio_token = processor.audio_token
220 + audio_token_id = vocab[audio_token]
221 +
222 + out_mm_data = out_mm_kwargs.get_data()
223 + audio_feature_lengths = out_mm_data.get("audio_feature_lengths")
224 + feature_attention_mask = out_mm_data.get("feature_attention_mask")
225 + if audio_feature_lengths is None and feature_attention_mask is None:
226 + audio_output_lengths = []
227 + elif audio_feature_lengths is not None:
228 + audio_output_lens = _get_feat_extract_output_lengths(audio_feature_lengths)
229 + audio_output_lengths = audio_output_lens.tolist()
230 + elif feature_attention_mask is not None:
231 + assert isinstance(feature_attention_mask, torch.Tensor)
232 + audio_output_lens = _get_feat_extract_output_lengths(
233 + feature_attention_mask.sum(-1)
234 + )
235 + audio_output_lengths = audio_output_lens.tolist()
236 +
237 + def get_replacement_qwen2_audio(item_idx: int):
238 + num_features = audio_output_lengths[item_idx]
239 + if num_features == 0:
240 + audios = mm_items.get_items("audio", AudioProcessorItems)
241 + audio = audios.get(item_idx)
242 + raise ValueError(
243 + f"The audio {audio} (len={len(audio)}) is too short "
244 + "to be represented inside the model"
245 + )
246 +
247 + return [audio_token_id] * num_features
248 +
249 + return [
250 + PromptReplacement(
251 + modality="audio",
252 + target=audio_token,
253 + replacement=get_replacement_qwen2_audio,
254 + ),
255 + ]
256 +
257 +
258 +@MULTIMODAL_REGISTRY.register_processor(
259 + Qwen3ASRMultiModalProcessor,
260 + info=Qwen3ASRProcessingInfo,
261 + dummy_inputs=Qwen3ASRDummyInputsBuilder,
262 +)
263 +class Qwen3ASRForConditionalGeneration(
264 + nn.Module,
265 + SupportsMultiModal,
266 + SupportsPP,
267 + SupportsMRoPE,
268 + SupportsTranscription,
269 +):
270 + supported_languages = ISO639_1_SUPPORTED_LANGS
271 +
272 + hf_to_vllm_mapper = WeightsMapper(
273 + orig_to_new_prefix={
274 + "thinker.lm_head.": "language_model.lm_head.",
275 + "thinker.model.": "language_model.model.",
276 + "thinker.": "",
277 + }
278 + )
279 +
280 + @classmethod
281 + def get_placeholder_str(cls, modality: str, i: int) -> str | None:
282 + if modality.startswith("audio"):
283 + return "<|audio_start|><|audio_pad|><|audio_end|>"
284 +
285 + raise ValueError("Only audio modality is supported")
286 +
287 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
288 + super().__init__()
289 + self.vllm_config = vllm_config # needed for torch compile forward context
290 + thinker_config: Qwen3ASRThinkerConfig = (
291 + vllm_config.model_config.hf_config.thinker_config
292 + )
293 + quant_config = vllm_config.quant_config
294 + multimodal_config = vllm_config.model_config.multimodal_config
295 + self.config = thinker_config
296 + self.multimodal_config = multimodal_config
297 +
298 + self.audio_tower = Qwen3OmniMoeAudioEncoder(
299 + thinker_config.audio_config,
300 + prefix=maybe_prefix(prefix, "audio_tower"),
301 + )
302 + self.quant_config = quant_config
303 +
304 + self.language_model = Qwen3ForCausalLM(
305 + vllm_config=vllm_config.with_hf_config(
306 + thinker_config.text_config, architectures=["Qwen3ForCausalLM"]
307 + ),
308 + prefix=maybe_prefix(prefix, "language_model"),
309 + )
310 +
311 + self.make_empty_intermediate_tensors = (
312 + self.language_model.make_empty_intermediate_tensors
313 + )
314 +
315 + def _parse_and_validate_audio_input(
316 + self, **kwargs: object
317 + ) -> Qwen2_5OmniAudioFeatureInputs | None:
318 + input_audio_features = kwargs.pop("input_audio_features", None)
319 + audio_feature_lengths = kwargs.pop("audio_feature_lengths", None)
320 + feature_attention_mask = kwargs.pop("feature_attention_mask", None)
321 + if input_audio_features is None:
322 + return None
323 +
324 + return Qwen2_5OmniAudioFeatureInputs(
325 + type="audio_features",
326 + input_features=input_audio_features,
327 + audio_feature_lengths=audio_feature_lengths,
328 + feature_attention_mask=feature_attention_mask,
329 + )
330 +
331 + def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
332 + mm_input_by_modality = {}
333 +
334 + # Preserve the order of modalities if there are multiple of them
335 + # from the order of kwargs.
336 + for input_key in kwargs:
337 + if (
338 + input_key in ("input_audio_features")
339 + and "audio" not in mm_input_by_modality
340 + ):
341 + mm_input_by_modality["audio"] = self._parse_and_validate_audio_input(
342 + **kwargs
343 + )
344 + return mm_input_by_modality
345 +
346 + def _process_audio_input(
347 + self,
348 + audio_input: Qwen2_5OmniAudioFeatureInputs,
349 + audio_hashes: list[str] | None = None,
350 + cached_audio_features: torch.Tensor | None = None,
351 + ) -> torch.Tensor:
352 + input_features = audio_input["input_features"]
353 + audio_feature_lengths = audio_input["audio_feature_lengths"]
354 +
355 + audio_output_lengths = _get_feat_extract_output_lengths(audio_feature_lengths)
356 +
357 + audio_features = self.audio_tower(
358 + input_features.to(self.audio_tower.dtype),
359 + feature_lens=audio_feature_lengths,
360 + aftercnn_lens=audio_output_lengths,
361 + )
362 + return audio_features.split(audio_output_lengths.tolist())
363 +
364 + def get_language_model(self) -> torch.nn.Module:
365 + return self.language_model
366 +
367 + def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings | None:
368 + mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)
369 + if not mm_input_by_modality:
370 + return []
371 +
372 + # The result multimodal_embeddings is tuple of tensors, with each
373 + # tensor correspoending to a multimodal data item (image or video).
374 + multimodal_embeddings: tuple[torch.Tensor, ...] = ()
375 +
376 + # NOTE: It is important to iterate over the keys in this dictionary
377 + # to preserve the order of the modalities.
378 + for modality in mm_input_by_modality:
379 + multimodal_input = mm_input_by_modality[modality]
380 + if modality == "audio":
381 + audio_embeddings = self._process_audio_input(multimodal_input)
382 + multimodal_embeddings += tuple(audio_embeddings)
383 + return multimodal_embeddings
384 +
385 + def embed_input_ids(
386 + self,
387 + input_ids: torch.Tensor,
388 + multimodal_embeddings: MultiModalEmbeddings | None = None,
389 + *,
390 + is_multimodal: torch.Tensor | None = None,
391 + handle_oov_mm_token: bool = False,
392 + ) -> torch.Tensor:
393 + inputs_embeds = self._embed_text_input_ids(
394 + input_ids,
395 + self.language_model.embed_input_ids,
396 + is_multimodal=is_multimodal,
397 + handle_oov_mm_token=handle_oov_mm_token,
398 + )
399 +
400 + if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
401 + return inputs_embeds
402 +
403 + inputs_embeds = _merge_multimodal_embeddings(
404 + inputs_embeds=inputs_embeds,
405 + multimodal_embeddings=multimodal_embeddings,
406 + is_multimodal=is_multimodal,
407 + )
408 +
409 + return inputs_embeds
410 +
411 + def forward(
412 + self,
413 + input_ids: torch.Tensor,
414 + positions: torch.Tensor,
415 + intermediate_tensors: IntermediateTensors | None = None,
416 + inputs_embeds: torch.Tensor | None = None,
417 + **kwargs: object,
418 + ) -> torch.Tensor | IntermediateTensors:
419 + if intermediate_tensors is not None:
420 + inputs_embeds = None
421 +
422 + hidden_states = self.language_model.model(
423 + input_ids,
424 + positions,
425 + intermediate_tensors,
426 + inputs_embeds=inputs_embeds,
427 + )
428 +
429 + return hidden_states
430 +
431 + def compute_logits(
432 + self,
433 + hidden_states: torch.Tensor,
434 + ) -> torch.Tensor | None:
435 + return self.language_model.compute_logits(hidden_states)
436 +
437 + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
438 + loader = AutoWeightsLoader(
439 + self,
440 + skip_prefixes=["talker.", "code2wav."],
441 + )
442 + loaded_weights = loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
443 +
444 + return loaded_weights
445 +
446 + def get_mrope_input_positions(
447 + self,
448 + input_tokens: list[int],
449 + mm_features: list[MultiModalFeatureSpec],
450 + ) -> tuple[torch.Tensor, int]:
451 + seq_len = len(input_tokens)
452 +
453 + if not mm_features:
454 + # No audio features, just return linear positions
455 + llm_positions = (
456 + torch.arange(seq_len, dtype=torch.long).view(1, -1).expand(3, -1)
457 + )
458 + return llm_positions.clone(), 0
459 +
460 + llm_pos_ids_list: list[torch.Tensor] = []
461 + st = 0
462 +
463 + for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
464 + offset = mm_feature.mm_position.offset
465 +
466 + # Get audio feature length from mm_feature data
467 + audio_feature_length = mm_feature.data["audio_feature_lengths"].data
468 + if isinstance(audio_feature_length, torch.Tensor):
469 + audio_feature_length = audio_feature_length.item()
470 + audio_len = _get_feat_extract_output_lengths(
471 + torch.tensor(audio_feature_length)
472 + ).item()
473 +
474 + # Text segment before audio (includes audio_start token)
475 + text_len = offset - st
476 + st_idx = llm_pos_ids_list[-1].max() + 1 if llm_pos_ids_list else 0
477 + text_positions = (
478 + torch.arange(text_len, dtype=torch.long).view(1, -1).expand(3, -1)
479 + + st_idx
480 + )
481 + llm_pos_ids_list.append(text_positions)
482 + st_idx = st_idx + text_len
483 +
484 + # Audio token segment
485 + audio_positions = (
486 + torch.arange(audio_len, dtype=torch.long).view(1, -1).expand(3, -1)
487 + + st_idx
488 + )
489 + llm_pos_ids_list.append(audio_positions)
490 +
491 + st = offset + audio_len
492 +
493 + # Handle remaining text (includes audio_end and any trailing text)
494 + if st < seq_len:
495 + st_idx = llm_pos_ids_list[-1].max() + 1 if llm_pos_ids_list else 0
496 + text_len = seq_len - st
497 + final_text_positions = (
498 + torch.arange(text_len, dtype=torch.long).view(1, -1).expand(3, -1)
499 + + st_idx
500 + )
501 + llm_pos_ids_list.append(final_text_positions)
502 +
503 + llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)
504 + if llm_positions.shape[1] != seq_len:
505 + raise RuntimeError("Position ids length mismatch with input ids length")
506 +
507 + mrope_position_delta = (llm_positions.max() + 1 - seq_len).item()
508 + return llm_positions, mrope_position_delta
509 +
510 + def get_mm_mapping(self) -> MultiModelKeys:
511 + """
512 + Get the module prefix in multimodal models
513 + """
514 + return MultiModelKeys.from_string_field(
515 + language_model="language_model",
516 + tower_model=["audio_tower."],
517 + )
518 +
519 + @classmethod
520 + def get_speech_to_text_config(
521 + cls, model_config: ModelConfig, task_type: str
522 + ) -> SpeechToTextConfig:
523 + processor = cached_processor_from_config(model_config)
524 + feature_extractor: WhisperFeatureExtractor = processor.feature_extractor
525 + return SpeechToTextConfig(
526 + max_audio_clip_s=feature_extractor.chunk_length,
527 + sample_rate=feature_extractor.sampling_rate,
528 + )
529 +
530 + @classmethod
531 + def get_generation_prompt(
532 + cls,
533 + audio: np.ndarray,
534 + model_config: ModelConfig,
535 + stt_config: SpeechToTextConfig,
536 + language: str | None,
537 + task_type: Literal["transcribe", "translate"],
538 + request_prompt: str,
539 + to_language: str | None,
540 + ) -> PromptType:
541 + """Get the generation prompt to be used for transcription requests."""
542 + tokenizer = cached_tokenizer_from_config(model_config)
543 + audio_placeholder = cls.get_placeholder_str("audio", 0)
544 +
545 + if task_type not in ("transcribe", "translate"):
546 + raise ValueError(
547 + f"Unsupported task_type '{task_type}'. "
548 + "Supported task types are 'transcribe' and 'translate'."
549 + )
550 + full_lang_name_to = cls.supported_languages.get(to_language, to_language)
551 + if to_language is None:
552 + prompt = (
553 + f"<|im_start|>user\n{audio_placeholder}<|im_end|>\n"
554 + f"<|im_start|>assistant\n"
555 + )
556 + else:
557 + prompt = (
558 + f"<|im_start|>user\n{audio_placeholder}<|im_end|>\n"
559 + f"<|im_start|>assistant\nlanguage {full_lang_name_to}<asr_text>"
560 + )
561 +
562 + prompt_token_ids = tokenizer.encode(prompt)
563 + prompt_dict = {
564 + "prompt_token_ids": prompt_token_ids,
565 + "multi_modal_data": {"audio": audio},
566 + }
567 + return cast(PromptType, prompt_dict)
vllm/model_executor/models/registry.py
+4
@@ -436,6 +436,10 @@ _MULTIMODAL_MODELS = {
436 "qwen3_omni_moe_thinker",
437 "Qwen3OmniMoeThinkerForConditionalGeneration",
438 ),
439 + "Qwen3ASRForConditionalGeneration": (
440 + "qwen3_asr",
441 + "Qwen3ASRForConditionalGeneration",
442 + ),
443 "Qwen3VLForConditionalGeneration": ("qwen3_vl", "Qwen3VLForConditionalGeneration"), # noqa: E501
444 "Qwen3VLMoeForConditionalGeneration": (
445 "qwen3_vl_moe",
vllm/transformers_utils/config.py
+1
@@ -97,6 +97,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
97 ultravox="UltravoxConfig",
98 step3_vl="Step3VLConfig",
99 step3_text="Step3TextConfig",
100 + qwen3_asr="Qwen3ASRConfig",
101 qwen3_next="Qwen3NextConfig",
102 lfm2_moe="Lfm2MoeConfig",
103 tarsier2="Tarsier2Config",
vllm/transformers_utils/configs/__init__.py
+2
@@ -52,6 +52,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
52 "Step3VLConfig": "vllm.transformers_utils.configs.step3_vl",
53 "Step3VisionEncoderConfig": "vllm.transformers_utils.configs.step3_vl",
54 "Step3TextConfig": "vllm.transformers_utils.configs.step3_vl",
55 + "Qwen3ASRConfig": "vllm.transformers_utils.configs.qwen3_asr",
56 "Qwen3NextConfig": "vllm.transformers_utils.configs.qwen3_next",
57 "Tarsier2Config": "vllm.transformers_utils.configs.tarsier2",
58 # Special case: DeepseekV3Config is from HuggingFace Transformers
@@ -94,6 +95,7 @@ __all__ = [
95 "Step3VLConfig",
96 "Step3VisionEncoderConfig",
97 "Step3TextConfig",
98 + "Qwen3ASRConfig",
99 "Qwen3NextConfig",
100 "Tarsier2Config",
101 ]
vllm/transformers_utils/configs/qwen3_asr.py new
+436
@@ -0,0 +1,436 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +# ruff: noqa
5 +# mypy: ignore-errors
6 +# coding=utf-8
7 +# Copyright 2026 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved.
8 +#
9 +# Licensed under the Apache License, Version 2.0 (the "License");
10 +# you may not use this file except in compliance with the License.
11 +# You may obtain a copy of the License at
12 +#
13 +# http://www.apache.org/licenses/LICENSE-2.0
14 +#
15 +# Unless required by applicable law or agreed to in writing, software
16 +# distributed under the License is distributed on an "AS IS" BASIS,
17 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 +# See the License for the specific language governing permissions and
19 +# limitations under the License.
20 +from transformers.configuration_utils import PretrainedConfig
21 +from transformers.modeling_rope_utils import rope_config_validation
22 +from transformers.utils import logging
23 +
24 +logger = logging.get_logger(__name__)
25 +
26 +
27 +class Qwen3ASRAudioEncoderConfig(PretrainedConfig):
28 + r"""
29 + This is the configuration class to store the configuration of a [`Qwen3ASRAudioEncoder`]. It is used to instantiate a
30 + Qwen3-ASR audio encoder according to the specified arguments, defining the model architecture. Instantiating a
31 + configuration with the defaults will yield a similar configuration to that of the audio encoder of the Qwen2-Audio
32 + architecture.
33 +
34 + e.g. [Qwen/Qwen3-ASR-1.7B](https://huggingface.co/Qwen/Qwen3-ASR-1.7B)
35 +
36 + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
37 + documentation from [`PretrainedConfig`] for more information.
38 +
39 + Args:
40 + num_mel_bins (`int`, *optional*, defaults to 128):
41 + Number of mel features used per input features. Should correspond to the value used in the
42 + `Qwen3ASRProcessor` class.
43 + encoder_layers (`int`, *optional*, defaults to 32):
44 + Number of encoder layers.
45 + encoder_attention_heads (`int`, *optional*, defaults to 20):
46 + Number of attention heads for each attention layer in the Transformer encoder.
47 + encoder_ffn_dim (`int`, *optional*, defaults to 5120):
48 + Dimensionality of the "intermediate" (often named feed-forward) layer in encoder.
49 + d_model (`int`, *optional*, defaults to 1280):
50 + Dimensionality of the layers.
51 + dropout (`float`, *optional*, defaults to 0.0):
52 + The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
53 + attention_dropout (`float`, *optional*, defaults to 0.0):
54 + The dropout ratio for the attention probabilities.
55 + activation_function (`str`, *optional*, defaults to `"gelu"`):
56 + The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
57 + `"relu"`, `"silu"` and `"gelu_new"` are supported.
58 + activation_dropout (`float`, *optional*, defaults to 0.0):
59 + The dropout ratio for activations inside the fully connected layer.
60 + scale_embedding (`bool`, *optional*, defaults to `False`):
61 + Scale embeddings by diving by sqrt(d_model).
62 + initializer_range (`float`, *optional*, defaults to 0.02):
63 + The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
64 + max_source_positions (`int`, *optional*, defaults to 1500):
65 + The maximum sequence length of log-mel filter-bank features that this model might ever be used with.
66 + n_window (`int`, *optional*, defaults to 100):
67 + The chunk for conv and flash attn in AudioEncoder.
68 + output_dim (`int`, *optional*, defaults to 3584):
69 + The output dimension of AudioEncoder.
70 +
71 + Example:
72 +
73 + ```python
74 + >>> from transformers import Qwen3ASRAudioEncoderConfig, Qwen3ASRAudioEncoder
75 +
76 + >>> # Initializing a Qwen3ASRAudioEncoderConfig
77 + >>> configuration = Qwen3ASRAudioEncoderConfig()
78 +
79 + >>> # Initializing a Qwen3ASRAudioEncoder (with random weights)
80 + >>> model = Qwen3ASRAudioEncoder(configuration)
81 +
82 + >>> # Accessing the model configuration
83 + >>> configuration = model.config
84 + ```"""
85 +
86 + model_type = "qwen3_asr_audio_encoder"
87 +
88 + def __init__(
89 + self,
90 + num_mel_bins=128,
91 + encoder_layers=32,
92 + encoder_attention_heads=20,
93 + encoder_ffn_dim=5120,
94 + d_model=1280,
95 + dropout=0,
96 + attention_dropout=0,
97 + activation_function="gelu",
98 + activation_dropout=0,
99 + scale_embedding=False,
100 + initializer_range=0.02,
101 + max_source_positions=1500,
102 + n_window=100,
103 + output_dim=3584,
104 + n_window_infer=400,
105 + conv_chunksize=500,
106 + downsample_hidden_size=480,
107 + **kwargs,
108 + ):
109 + super().__init__(**kwargs)
110 +
111 + self.num_mel_bins = num_mel_bins
112 + self.d_model = d_model
113 + self.encoder_layers = encoder_layers
114 + self.encoder_attention_heads = encoder_attention_heads
115 + self.encoder_ffn_dim = encoder_ffn_dim
116 + self.dropout = dropout
117 + self.attention_dropout = attention_dropout
118 + self.activation_function = activation_function
119 + self.activation_dropout = activation_dropout
120 + self.num_hidden_layers = encoder_layers
121 + self.initializer_range = initializer_range
122 + self.scale_embedding = (
123 + scale_embedding # scale factor will be sqrt(d_model) if True
124 + )
125 + self.max_source_positions = max_source_positions
126 + self.n_window = n_window
127 + self.output_dim = output_dim
128 + self.n_window_infer = n_window_infer
129 + self.conv_chunksize = conv_chunksize
130 + self.downsample_hidden_size = downsample_hidden_size
131 +
132 +
133 +class Qwen3ASRTextConfig(PretrainedConfig):
134 + r"""
135 + This is the configuration class to store the configuration of a [`Qwen3ASRTextModel`]. It is used to instantiate a
136 + Qwen3-ASR model according to the specified arguments, defining the model architecture. Instantiating a configuration
137 + with the defaults will yield a similar configuration to that of
138 + Qwen3-ASR-1.7B [Qwen/Qwen3-ASR-1.7B](https://huggingface.co/Qwen/Qwen3-ASR-1.7B)
139 +
140 + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
141 + documentation from [`PretrainedConfig`] for more information.
142 +
143 + Args:
144 + vocab_size (`int`, *optional*, defaults to 151936):
145 + Vocabulary size of the Qwen3ASR model. Defines the number of different tokens that can be represented by the
146 + `inputs_ids` passed when calling [`Qwen3ASRModel`]
147 + hidden_size (`int`, *optional*, defaults to 4096):
148 + Dimension of the hidden representations.
149 + intermediate_size (`int`, *optional*, defaults to 22016):
150 + Dimension of the MLP representations.
151 + num_hidden_layers (`int`, *optional*, defaults to 32):
152 + Number of hidden layers in the Transformer encoder.
153 + num_attention_heads (`int`, *optional*, defaults to 32):
154 + Number of attention heads for each attention layer in the Transformer encoder.
155 + num_key_value_heads (`int`, *optional*, defaults to 32):
156 + This is the number of key_value heads that should be used to implement Grouped Query Attention. If
157 + `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
158 + `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
159 + converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
160 + by meanpooling all the original heads within that group. For more details, check out [this
161 + paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to `32`.
162 + head_dim (`int`, *optional*, defaults to 128):
163 + The dimension of the head. If not specified, will default to `hidden_size // num_attention_heads`.
164 + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
165 + The non-linear activation function (function or string) in the decoder.
166 + max_position_embeddings (`int`, *optional*, defaults to 128000):
167 + The maximum sequence length that this model might ever be used with.
168 + initializer_range (`float`, *optional*, defaults to 0.02):
169 + The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
170 + rms_norm_eps (`float`, *optional*, defaults to 1e-06):
171 + The epsilon used by the rms normalization layers.
172 + use_cache (`bool`, *optional*, defaults to `True`):
173 + Whether or not the model should return the last key/values attentions (not used by all models). Only
174 + relevant if `config.is_decoder=True`.
175 + tie_word_embeddings (`bool`, *optional*, defaults to `False`):
176 + Whether the model's input and output word embeddings should be tied.
177 + rope_theta (`float`, *optional*, defaults to 5000000.0):
178 + The base period of the RoPE embeddings.
179 + rope_scaling (`Dict`, *optional*):
180 + Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
181 + and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
182 + accordingly.
183 + Expected contents:
184 + `rope_type` (`str`):
185 + The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
186 + 'llama3'], with 'default' being the original RoPE implementation.
187 + `factor` (`float`, *optional*):
188 + Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In
189 + most scaling types, a `factor` of x will enable the model to handle sequences of length x *
190 + original maximum pre-trained length.
191 + `original_max_position_embeddings` (`int`, *optional*):
192 + Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during
193 + pretraining.
194 + `attention_factor` (`float`, *optional*):
195 + Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention
196 + computation. If unspecified, it defaults to value recommended by the implementation, using the
197 + `factor` field to infer the suggested value.
198 + `beta_fast` (`float`, *optional*):
199 + Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear
200 + ramp function. If unspecified, it defaults to 32.
201 + `beta_slow` (`float`, *optional*):
202 + Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear
203 + ramp function. If unspecified, it defaults to 1.
204 + `short_factor` (`list[float]`, *optional*):
205 + Only used with 'longrope'. The scaling factor to be applied to short contexts (<
206 + `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
207 + size divided by the number of attention heads divided by 2
208 + `long_factor` (`list[float]`, *optional*):
209 + Only used with 'longrope'. The scaling factor to be applied to long contexts (<
210 + `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
211 + size divided by the number of attention heads divided by 2
212 + `low_freq_factor` (`float`, *optional*):
213 + Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE
214 + `high_freq_factor` (`float`, *optional*):
215 + Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE
216 + attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
217 + Whether to use a bias in the query, key, value and output projection layers during self-attention.
218 + attention_dropout (`float`, *optional*, defaults to 0.0):
219 + The dropout ratio for the attention probabilities.
220 +
221 + ```python
222 + >>> from transformers import Qwen3ASRTextModel, Qwen3ASRTextConfig
223 +
224 + >>> # Initializing a Qwen3ASR style configuration
225 + >>> configuration = Qwen3ASRTextConfig()
226 +
227 + >>> # Initializing a model from the Qwen3-VL-7B style configuration
228 + >>> model = Qwen3ASRTextModel(configuration)
229 +
230 + >>> # Accessing the model configuration
231 + >>> configuration = model.config
232 + ```"""
233 +
234 + model_type = "qwen3_asr_text"
235 + base_config_key = "text_config"
236 +
237 + def __init__(
238 + self,
239 + vocab_size=151936,
240 + hidden_size=4096,
241 + intermediate_size=22016,
242 + num_hidden_layers=32,
243 + num_attention_heads=32,
244 + num_key_value_heads=32,
245 + head_dim=128,
246 + hidden_act="silu",
247 + max_position_embeddings=128000,
248 + initializer_range=0.02,
249 + rms_norm_eps=1e-6,
250 + use_cache=True,
251 + tie_word_embeddings=False,
252 + rope_theta=5000000.0,
253 + rope_scaling=None,
254 + attention_bias=False,
255 + attention_dropout=0.0,
256 + **kwargs,
257 + ):
258 + self.vocab_size = vocab_size
259 + self.max_position_embeddings = max_position_embeddings
260 + self.hidden_size = hidden_size
261 + self.intermediate_size = intermediate_size
262 + self.num_hidden_layers = num_hidden_layers
263 + self.num_attention_heads = num_attention_heads
264 +
265 + # for backward compatibility
266 + if num_key_value_heads is None:
267 + num_key_value_heads = num_attention_heads
268 +
269 + self.num_key_value_heads = num_key_value_heads
270 + self.head_dim = head_dim
271 + self.hidden_act = hidden_act
272 + self.initializer_range = initializer_range
273 + self.rms_norm_eps = rms_norm_eps
274 + self.use_cache = use_cache
275 + self.rope_theta = rope_theta
276 + self.rope_scaling = rope_scaling
277 + self.attention_bias = attention_bias
278 + self.attention_dropout = attention_dropout
279 + # Validate the correctness of rotary position embeddings parameters
280 + # BC: if there is a 'type' field, move it to 'rope_type'.
281 + if self.rope_scaling is not None and "type" in self.rope_scaling:
282 + self.rope_scaling["rope_type"] = self.rope_scaling["type"]
283 + rope_config_validation(self)
284 +
285 + super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
286 +
287 +
288 +class Qwen3ASRThinkerConfig(PretrainedConfig):
289 + r"""
290 + This is the configuration class to store the configuration of a [`Qwen3ASRThinker`]. It is used to instantiate a
291 + Qwen3-ASR-Thinker model according to the specified arguments, defining the model architecture. Instantiating a
292 + configuration with the defaults will yield a similar configuration to that of the thinker component of the Qwen3-Omni
293 + architecture.
294 +
295 + e.g. [Qwen/Qwen3-ASR-1.7B](https://huggingface.co/Qwen/Qwen3-ASR-1.7B)
296 +
297 + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
298 + documentation from [`PretrainedConfig`] for more information.
299 +
300 + Args:
301 + audio_config (`dict`, *optional*):
302 + The config dictionary of the audio backbone.
303 + text_config (`dict`, *optional*):
304 + The config dictionary of the text backbone.
305 + audio_token_id (`int`, *optional*, defaults to 151646):
306 + The audio token id to encode the audio prompt.
307 + audio_start_token_id (`int`, *optional*, defaults to 151647):
308 + The audio start token id to encode the audio prompt.
309 + user_token_id (`int`, *optional*, defaults to 872):
310 + The user token id to encode the user token.
311 + initializer_range (`float`, *optional*, defaults to 0.02):
312 + The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
313 +
314 + Example:
315 +
316 + ```python
317 + >>> from transformers import Qwen3ASRThinkerModel, Qwen3ASRThinkerConfig
318 +
319 + >>> # Initializing a default Qwen3ASRThinkerConfig
320 + >>> configuration = Qwen3ASRThinkerConfig()
321 +
322 + >>> # Initializing a model (with random weights) from the default configuration
323 + >>> model = Qwen3ASRThinkerModel(configuration)
324 +
325 + >>> # Accessing the model configuration
326 + >>> configuration = model.config
327 + ```"""
328 +
329 + model_type = "qwen3_asr_thinker"
330 +
331 + attribute_map = {}
332 + sub_configs = {
333 + "audio_config": Qwen3ASRAudioEncoderConfig,
334 + "text_config": Qwen3ASRTextConfig,
335 + }
336 +
337 + def __init__(
338 + self,
339 + audio_config=None,
340 + text_config=None,
341 + audio_token_id=151646,
342 + audio_start_token_id=151647,
343 + user_token_id=872,
344 + initializer_range=0.02,
345 + **kwargs,
346 + ):
347 + super().__init__(**kwargs)
348 + self.user_token_id = user_token_id
349 + self.audio_start_token_id = audio_start_token_id
350 + self.initializer_range = initializer_range
351 +
352 + if isinstance(audio_config, dict):
353 + audio_config = Qwen3ASRAudioEncoderConfig(**audio_config)
354 + elif audio_config is None:
355 + audio_config = Qwen3ASRAudioEncoderConfig()
356 + self.audio_config = audio_config
357 +
358 + if isinstance(text_config, dict):
359 + text_config = Qwen3ASRTextConfig(**text_config)
360 + elif text_config is None:
361 + text_config = Qwen3ASRTextConfig()
362 + self.text_config = text_config
363 + self.audio_token_id = audio_token_id
364 +
365 +
366 +class Qwen3ASRConfig(PretrainedConfig):
367 + """
368 + This is the configuration class to store the configuration of a [`Qwen3ASRForConditionalGeneration`]. It is used to instantiate a Qwen3ASR
369 + model according to the specified sub-models configurations, defining the model architecture.
370 +
371 + Instantiating a configuration with the defaults will yield a similar configuration to that of the
372 + [Qwen/Qwen3-ASR-1.7B](https://huggingface.co/Qwen/Qwen3-ASR-1.7B) architecture.
373 +
374 + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
375 + documentation from [`PretrainedConfig`] for more information.
376 +
377 + Args:
378 + thinker_config (`dict`, *optional*): Configuration of the underlying thinker sub-model.
379 + support_languages (`List[str]`, *optional*): The languages supported by the model.
380 +
381 + Example:
382 +
383 + ```python
384 + >>> from transformers import (
385 + ... Qwen3ASRThinkerConfig,
386 + ... Qwen3ASRForConditionalGeneration,
387 + ... Qwen3ASRConfig,
388 + ... )
389 +
390 + >>> # Initializing a Qwen3ASR style configuration
391 + >>> configuration = Qwen3ASRConfig()
392 +
393 + >>> # Initializing a model from the configuration
394 + >>> model = Qwen3ASRForConditionalGeneration(configuration)
395 +
396 + >>> # Accessing the model configuration
397 + >>> configuration = model.config
398 + ```"""
399 +
400 + model_type = "qwen3_asr"
401 + sub_configs = {
402 + "thinker_config": Qwen3ASRThinkerConfig,
403 + }
404 +
405 + def __init__(
406 + self,
407 + thinker_config=None,
408 + support_languages=None,
409 + **kwargs,
410 + ):
411 + super().__init__(**kwargs)
412 + if thinker_config is None:
413 + thinker_config = {}
414 + logger.info(
415 + "thinker_config is None. Initializing thinker model with default values"
416 + )
417 +
418 + self.thinker_config = Qwen3ASRThinkerConfig(**thinker_config)
419 + self.support_languages = support_languages
420 +
421 + def get_text_config(self, decoder=False) -> "PretrainedConfig":
422 + """
423 + Returns the config that is meant to be used with text IO. On most models, it is the original config instance
424 + itself. On specific composite models, it is under a set of valid names.
425 +
426 + Args:
427 + decoder (`Optional[bool]`, *optional*, defaults to `False`):
428 + If set to `True`, then only search for decoder config names.
429 + """
430 + # Overridden for deeply nested config like Qwen2.5-Omni. We don't have any omni model
431 + # except for Qwen yet. This has to be generalized if more deeply nested configs are
432 + # added. NOTE: currently method used only by vLLM
433 + return self.thinker_config.get_text_config()
434 +
435 +
436 +__all__ = ["Qwen3ASRConfig", "Qwen3ASRThinkerConfig", "Qwen3ASRAudioEncoderConfig"]
vllm/transformers_utils/processors/qwen3_asr.py new
+231
@@ -0,0 +1,231 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +# ruff: noqa
5 +# mypy: ignore-errors
6 +# coding=utf-8
7 +# Copyright 2026 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved.
8 +#
9 +# Licensed under the Apache License, Version 2.0 (the "License");
10 +# you may not use this file except in compliance with the License.
11 +# You may obtain a copy of the License at
12 +#
13 +# http://www.apache.org/licenses/LICENSE-2.0
14 +#
15 +# Unless required by applicable law or agreed to in writing, software
16 +# distributed under the License is distributed on an "AS IS" BASIS,
17 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 +# See the License for the specific language governing permissions and
19 +# limitations under the License.
20 +import regex as re
21 +
22 +import numpy as np
23 +
24 +from transformers import AutoProcessor
25 +from transformers.audio_utils import AudioInput
26 +from transformers.feature_extraction_utils import BatchFeature
27 +from transformers.processing_utils import ProcessingKwargs, ProcessorMixin
28 +from transformers.tokenization_utils_base import TextInput
29 +
30 +
31 +class Qwen3ASRProcessorKwargs(ProcessingKwargs, total=False):
32 + _defaults = {
33 + "text_kwargs": {
34 + "padding": False,
35 + "padding_side": "left",
36 + },
37 + "audio_kwargs": {
38 + "sampling_rate": 16000,
39 + "padding": True,
40 + "return_attention_mask": True,
41 + },
42 + }
43 +
44 +
45 +def _get_feat_extract_output_lengths(input_lengths):
46 + """
47 + Computes the output length of the convolutional layers and the output length of the audio encoder
48 + """
49 +
50 + input_lengths_leave = input_lengths % 100
51 + feat_lengths = (input_lengths_leave - 1) // 2 + 1
52 + output_lengths = (
53 + ((feat_lengths - 1) // 2 + 1 - 1) // 2 + 1 + (input_lengths // 100) * 13
54 + )
55 + return output_lengths
56 +
57 +
58 +class Qwen3ASRProcessor(ProcessorMixin):
59 + r"""
60 + Constructs a Qwen3ASR processor.
61 + [`Qwen3ASRProcessor`] offers all the functionalities of [`WhisperFeatureExtractor`], and [`Qwen2TokenizerFast`]. See the
62 + [`~Qwen3ASRProcessor.__call__`] and [`~Qwen3ASRProcessor.decode`] for more information.
63 +
64 + Args:
65 + feature_extractor ([`WhisperFeatureExtractor`], *optional*):
66 + The audio feature extractor.
67 + tokenizer ([`Qwen2TokenizerFast`], *optional*):
68 + The text tokenizer.
69 + chat_template (`Optional[str]`, *optional*):
70 + The Jinja template to use for formatting the conversation. If not provided, the default chat template is used.
71 + """
72 +
73 + attributes = ["feature_extractor", "tokenizer"]
74 + feature_extractor_class = "WhisperFeatureExtractor"
75 + tokenizer_class = ("Qwen2Tokenizer", "Qwen2TokenizerFast")
76 +
77 + def __init__(self, feature_extractor=None, tokenizer=None, chat_template=None):
78 + super().__init__(feature_extractor, tokenizer, chat_template=chat_template)
79 + self.audio_token = self.tokenizer.audio_token
80 + self.audio_bos_token = self.tokenizer.audio_bos_token
81 + self.audio_eos_token = self.tokenizer.audio_eos_token
82 +
83 + def __call__(
84 + self,
85 + text: TextInput = None,
86 + audio: AudioInput = None,
87 + **kwargs,
88 + ) -> BatchFeature:
89 + """
90 + Main method to prepare for the model one or several sequences(s) and audio(s). This method forwards the `text`
91 + and `kwargs` arguments to Qwen2TokenizerFast's [`~Qwen2TokenizerFast.__call__`] if `text` is not `None` to encode
92 + the text. To prepare the audio(s), this method forwards the `audio` and `kwargs` arguments to
93 + WhisperFeatureExtractor's [`~WhisperFeatureExtractor.__call__`] if `audio` is not `None`. Please refer to the doctsring
94 + of the above two methods for more information.
95 +
96 + Args:
97 + text (`str`, `List[str]`, `List[List[str]]`):
98 + The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
99 + (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
100 + `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
101 + audio (`np.ndarray`, `List[np.ndarray]`):
102 + The audio or batch of audio to be prepared. Each audio can be a NumPy array.
103 + """
104 +
105 + if text is None:
106 + raise ValueError("You need to specify either a `text` input to process.")
107 +
108 + output_kwargs = self._merge_kwargs(
109 + Qwen3ASRProcessorKwargs,
110 + tokenizer_init_kwargs=self.tokenizer.init_kwargs,
111 + **kwargs,
112 + )
113 +
114 + if audio is not None:
115 + output_kwargs["audio_kwargs"]["padding"] = True
116 + output_kwargs["audio_kwargs"]["truncation"] = False
117 + audio_inputs = self.feature_extractor(
118 + audio, **output_kwargs["audio_kwargs"]
119 + )
120 + audio_inputs["feature_attention_mask"] = audio_inputs.pop(
121 + "attention_mask"
122 + ) # rename feature_attention_mask to prevent conflicts later on
123 + audio_inputs["input_features"] = audio_inputs.pop(
124 + "input_features"
125 + ) # rename input_features to prevent conflicts later on
126 + audio_lengths = iter(
127 + _get_feat_extract_output_lengths(
128 + audio_inputs["feature_attention_mask"].sum(-1)
129 + )
130 + )
131 + else:
132 + audio_inputs = {}
133 + audio_lengths = iter([])
134 +
135 + if not isinstance(text, list):
136 + text = [text]
137 +
138 + text = self.replace_multimodal_special_tokens(
139 + text,
140 + audio_lengths,
141 + )
142 +
143 + texts_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
144 +
145 + return BatchFeature(
146 + data={**texts_inputs, **audio_inputs},
147 + tensor_type=kwargs.get("return_tensors"),
148 + )
149 +
150 + def replace_multimodal_special_tokens(
151 + self,
152 + text,
153 + audio_lengths,
154 + ):
155 + processed_text = []
156 + for sample in text:
157 + positions = []
158 + special_tokens = [re.escape(tok) for tok in [self.audio_token]]
159 + pattern = "|".join(special_tokens)
160 + positions = sorted(
161 + [
162 + (match.start(), match.group())
163 + for match in re.finditer(pattern, sample)
164 + ]
165 + )
166 + positions.sort(key=lambda x: x[0])
167 +
168 + for _, special_token in positions:
169 + if special_token == self.audio_token:
170 + sample = sample.replace(
171 + self.audio_token,
172 + "<|audio_placeholder|>" * next(audio_lengths),
173 + 1,
174 + )
175 +
176 + sample = sample.replace("<|audio_placeholder|>", self.audio_token)
177 + processed_text.append(sample)
178 + return processed_text
179 +
180 + def get_chunked_index(
181 + self, token_indices: np.ndarray, tokens_per_chunk: int
182 + ) -> list[tuple[int, int]]:
183 + """
184 + Splits token index list into chunks based on token value ranges.
185 +
186 + Given a list of token indices, returns a list of (start, end) index tuples representing
187 + slices of the list where the token values fall within successive ranges of `tokens_per_chunk`.
188 +
189 + For example, if `tokens_per_chunk` is 1000, the function will create chunks such that:
190 + - the first chunk contains token values < 1000,
191 + - the second chunk contains values >= 1000 and < 2000, and so on.
192 +
193 + Parameters:
194 + token_indices (`np.ndarray`): A monotonically increasing list of token index values.
195 + tokens_per_chunk (`int`): Number of tokens per chunk (used as the chunk size threshold).
196 +
197 + Returns:
198 + `list[tuple[int, int]]`: A list of tuples, each representing the start (inclusive)
199 + and end (exclusive) indices of a chunk in `token_indices`.
200 + """
201 +
202 + def _iter():
203 + i, start_idx = 0, 0 # skip bos token
204 + current_chunk = 1
205 + while i < len(token_indices): # skip eos token
206 + if token_indices[i] >= current_chunk * tokens_per_chunk:
207 + yield (start_idx, i)
208 + start_idx = i
209 + current_chunk += 1
210 + i += 1
211 + yield (start_idx, len(token_indices))
212 +
213 + return list(_iter())
214 +
215 + def apply_chat_template(self, conversations, chat_template=None, **kwargs):
216 + return super().apply_chat_template(conversations, chat_template, **kwargs)
217 +
218 + @property
219 + def model_input_names(self):
220 + tokenizer_input_names = self.tokenizer.model_input_names
221 + feature_extractor_input_names = self.feature_extractor.model_input_names
222 + return list(
223 + dict.fromkeys(
224 + tokenizer_input_names
225 + + feature_extractor_input_names
226 + + ["feature_attention_mask"]
227 + )
228 + )
229 +
230 +
231 +AutoProcessor.register("Qwen3ASRProcessor", Qwen3ASRProcessor)