A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 22:45 UTC
[Model] Add transcription support for Qwen3-Omni (#29828)
Signed-off-by: Muhammad Hashmi <mhashmi@berkeley.edu> Signed-off-by: NickLucche <nlucches@redhat.com> Co-authored-by: NickLucche <nlucches@redhat.com>
Muhammad Hashmi committed
Feb 4, 2026 at 13:17 UTC
535de06cb1d90ed1c48246a512e74c87fe1768e4
3 files changed
+104
-2
docs/contributing/model/transcription.md
+1
@@ -251,6 +251,7 @@ No extra registration is required beyond having your model class available via t
251
- Whisper encoder–decoder (audio-only): [vllm/model_executor/models/whisper.py](../../../vllm/model_executor/models/whisper.py)
252
- Voxtral decoder-only (audio embeddings + LLM): [vllm/model_executor/models/voxtral.py](../../../vllm/model_executor/models/voxtral.py). Make sure to have installed `mistral-common[audio]`.
253
- Gemma3n decoder-only with fixed instruction prompt: [vllm/model_executor/models/gemma3n_mm.py](../../../vllm/model_executor/models/gemma3n_mm.py)
254
+- Qwen3-Omni multimodal with audio embeddings: [vllm/model_executor/models/qwen3_omni_moe_thinker.py](../../../vllm/model_executor/models/qwen3_omni_moe_thinker.py)
255
256
## Test with the API
257
docs/models/supported_models.md
+1
@@ -781,6 +781,7 @@ Speech2Text models trained specifically for Automatic Speech Recognition.
781
| `GlmAsrForConditionalGeneration` | GLM-ASR | `zai-org/GLM-ASR-Nano-2512` | ✅︎ | ✅︎ |
782
| `GraniteSpeechForConditionalGeneration` | Granite Speech | `ibm-granite/granite-speech-3.3-2b`, `ibm-granite/granite-speech-3.3-8b`, etc. | ✅︎ | ✅︎ |
783
| `Qwen3ASRForConditionalGeneration` | Qwen3-ASR | `Qwen/Qwen3-ASR-1.7B`, etc. | | ✅︎ |
784
+| `Qwen3OmniMoeThinkerForConditionalGeneration` | Qwen3-Omni | `Qwen/Qwen3-Omni-30B-A3B-Instruct`, etc. | | ✅︎ |
785
| `VoxtralForConditionalGeneration` | Voxtral (Mistral format) | `mistralai/Voxtral-Mini-3B-2507`, `mistralai/Voxtral-Small-24B-2507`, etc. | ✅︎ | ✅︎ |
786
| `WhisperForConditionalGeneration` | Whisper | `openai/whisper-small`, `openai/whisper-large-v3-turbo`, etc. | | |
787
vllm/model_executor/models/qwen3_omni_moe_thinker.py
+102
-2
@@ -24,7 +24,7 @@
24
25
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
26
from functools import partial
27
-from typing import Any
27
+from typing import Any, Literal, cast
28
29
import numpy as np
30
import torch
@@ -48,8 +48,9 @@ from transformers import __version__ as TRANSFORMERS_VERSION
48
# isort: on
49
50
from vllm.compilation.decorators import support_torch_compile
51
-from vllm.config import VllmConfig
51
+from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
52
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
53
+from vllm.inputs.data import PromptType
54
from vllm.logger import init_logger
55
from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY
56
from vllm.model_executor.layers.attention.mm_encoder_attention import (
@@ -79,6 +80,7 @@ from vllm.multimodal.processing.processor import (
80
PromptUpdateDetails,
81
)
82
from vllm.sequence import IntermediateTensors
83
+from vllm.transformers_utils.processor import cached_processor_from_config
84
from vllm.v1.attention.backends.registry import AttentionBackendEnum
85
86
from .interfaces import (
@@ -86,6 +88,7 @@ from .interfaces import (
88
SupportsMRoPE,
89
SupportsMultiModal,
90
SupportsPP,
91
+ SupportsTranscription,
92
)
93
from .qwen2_5_omni_thinker import (
94
Qwen2_5OmniAudioFeatureInputs,
@@ -110,6 +113,29 @@ from .vision import get_vit_attn_backend
113
114
logger = init_logger(__name__)
115
116
+# Speech input languages supported by Qwen3-Omni
117
+# From: https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct
118
+ISO639_1_SUPPORTED_LANGS = {
119
+ "en": "English",
120
+ "zh": "Chinese",
121
+ "ko": "Korean",
122
+ "ja": "Japanese",
123
+ "de": "German",
124
+ "ru": "Russian",
125
+ "it": "Italian",
126
+ "fr": "French",
127
+ "es": "Spanish",
128
+ "pt": "Portuguese",
129
+ "ms": "Malay",
130
+ "nl": "Dutch",
131
+ "id": "Indonesian",
132
+ "tr": "Turkish",
133
+ "vi": "Vietnamese",
134
+ "yue": "Cantonese",
135
+ "ar": "Arabic",
136
+ "ur": "Urdu",
137
+}
138
+
139
140
def _get_feat_extract_output_lengths(input_lengths: torch.Tensor):
141
input_lengths_leave = input_lengths % 100
@@ -1572,6 +1598,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
1598
SupportsPP,
1599
SupportsMRoPE,
1600
Qwen3OmniMoeConditionalGenerationMixin,
1601
+ SupportsTranscription,
1602
):
1603
hf_to_vllm_mapper = WeightsMapper(
1604
orig_to_new_prefix={
@@ -1593,6 +1620,8 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
1620
],
1621
}
1622
1623
+ supported_languages = ISO639_1_SUPPORTED_LANGS
1624
+
1625
@classmethod
1626
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
1627
if modality.startswith("image"):
@@ -2085,6 +2114,77 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
2114
total_tokens = num_video + audio_len
2115
return np.concatenate(pos_ids_list, axis=1), total_tokens
2116
2117
+ @classmethod
2118
+ def get_speech_to_text_config(
2119
+ cls, model_config: ModelConfig, task_type: str
2120
+ ) -> SpeechToTextConfig:
2121
+ processor = cached_processor_from_config(
2122
+ model_config, processor_cls=Qwen3OmniMoeProcessor
2123
+ )
2124
+ return SpeechToTextConfig(
2125
+ max_audio_clip_s=processor.feature_extractor.chunk_length,
2126
+ sample_rate=processor.feature_extractor.sampling_rate,
2127
+ min_energy_split_window_size=None,
2128
+ )
2129
+
2130
+ @classmethod
2131
+ def get_generation_prompt(
2132
+ cls,
2133
+ audio: np.ndarray,
2134
+ stt_config: SpeechToTextConfig,
2135
+ model_config: ModelConfig,
2136
+ language: str | None,
2137
+ task_type: Literal["transcribe", "translate"],
2138
+ request_prompt: str,
2139
+ to_language: str | None,
2140
+ ) -> PromptType:
2141
+ """
2142
+ Construct a transcription/translation prompt for Qwen3-Omni.
2143
+ """
2144
+ # Transcribe this audio [into <language>] | for transcription
2145
+ # Translate this audio [from <language> into <to_language>] | for translation
2146
+ instruction = "Transcribe" if task_type == "transcribe" else "Translate"
2147
+ instruction += " this audio"
2148
+
2149
+ # Default to_language to English for translation
2150
+ if task_type == "translate" and to_language is None:
2151
+ to_language = "en"
2152
+
2153
+ # Get full language names from supported_languages mapping
2154
+ full_lang_name = cls.supported_languages.get(language, "")
2155
+ full_lang_name_to = cls.supported_languages.get(to_language, "")
2156
+
2157
+ if task_type == "transcribe" and full_lang_name:
2158
+ instruction += f" into {full_lang_name}"
2159
+ elif task_type == "translate":
2160
+ if full_lang_name:
2161
+ instruction += f" from {full_lang_name}"
2162
+ if full_lang_name_to:
2163
+ instruction += f" into {full_lang_name_to}"
2164
+
2165
+ instruction += "."
2166
+
2167
+ if request_prompt:
2168
+ instruction += f" {request_prompt}"
2169
+
2170
+ processor = cached_processor_from_config(
2171
+ model_config, processor_cls=Qwen3OmniMoeProcessor
2172
+ )
2173
+ # Audio placeholder format: <|audio_start|><|audio_pad|><|audio_end|>
2174
+ audio_placeholder = "<|audio_start|><|audio_pad|><|audio_end|>"
2175
+ user_content = f"{audio_placeholder}{instruction}"
2176
+
2177
+ messages = [{"role": "user", "content": user_content}]
2178
+ prompt = processor.apply_chat_template(
2179
+ messages,
2180
+ tokenize=False,
2181
+ add_generation_prompt=True,
2182
+ )
2183
+
2184
+ audio_data = (audio, stt_config.sample_rate)
2185
+ prompts_dict = {"multi_modal_data": {"audio": audio_data}, "prompt": prompt}
2186
+ return cast(PromptType, prompts_dict)
2187
+
2188
def get_mrope_input_positions(
2189
self,
2190
input_tokens: list[int],