A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 00:00 UTC
[Realtime] Add Qwen3-ASR realtime streaming support (#34613)
Signed-off-by: Thomas Pouget-Abadie <thomaspou@microsoft.com> Co-authored-by: Thomas Pouget-Abadie <thomaspou@microsoft.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
pougetat committed
Feb 20, 2026 at 19:59 UTC
11be2c74dc1eb08aaaeb260f84a31c2b36bbd454
5 files changed
+256
-1
tests/models/registry.py
+8
@@ -1023,6 +1023,14 @@ _MULTIMODAL_EXAMPLE_MODELS = {
1023
min_transformers_version="4.57",
1024
is_available_online=False,
1025
),
1026
+ "Qwen3ASRRealtimeGeneration": _HfExamplesInfo(
1027
+ "Qwen/Qwen3-ASR-1.7B",
1028
+ max_model_len=4096,
1029
+ min_transformers_version="4.57",
1030
+ enforce_eager=True,
1031
+ hf_overrides={"architectures": ["Qwen3ASRRealtimeGeneration"]},
1032
+ is_available_online=False,
1033
+ ),
1034
"RForConditionalGeneration": _HfExamplesInfo("YannQi/R-4B", trust_remote_code=True),
1035
"SkyworkR1VChatModel": _HfExamplesInfo(
1036
"Skywork/Skywork-R1V-38B", trust_remote_code=True
vllm/entrypoints/openai/realtime/connection.py
+1
-1
@@ -205,7 +205,7 @@ class RealtimeConnection:
205
206
sampling_params = SamplingParams.from_optional(
207
temperature=0.0,
208
- max_tokens=1,
208
+ max_tokens=self.serving.model_cls.realtime_max_tokens,
209
output_kind=RequestOutputKind.DELTA,
210
skip_clone=True,
211
)
vllm/model_executor/models/interfaces.py
+4
@@ -1063,6 +1063,10 @@ class SupportsRealtime(Protocol):
1063
1064
supports_realtime: ClassVar[Literal[True]] = True
1065
1066
+ realtime_max_tokens: ClassVar[int] = 1
1067
+ """Maximum tokens to generate per streaming audio segment.
1068
+ Override in subclasses based on the model's expected output length."""
1069
+
1070
@classmethod
1071
async def buffer_realtime_audio(
1072
cls,
vllm/model_executor/models/qwen3_asr_realtime.py
new
+239
@@ -0,0 +1,239 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+# Copyright 2026 The Qwen team.
4
+# Copyright 2023 The vLLM team.
5
+#
6
+# Licensed under the Apache License, Version 2.0 (the "License");
7
+# you may not use this file except in compliance with the License.
8
+# You may obtain a copy of the License at
9
+#
10
+# http://www.apache.org/licenses/LICENSE-2.0
11
+#
12
+# Unless required by applicable law or agreed to in writing, software
13
+# distributed under the License is distributed on an "AS IS" BASIS,
14
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+# See the License for the specific language governing permissions and
16
+# limitations under the License.
17
+"""Inference-only Qwen3-ASR realtime model."""
18
+
19
+import asyncio
20
+from collections.abc import AsyncGenerator, Mapping
21
+
22
+import numpy as np
23
+import torch
24
+
25
+from vllm.compilation.decorators import support_torch_compile
26
+from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
27
+from vllm.inputs.data import PromptType, TokensPrompt
28
+from vllm.logger import init_logger
29
+from vllm.model_executor.models.interfaces import (
30
+ SupportsRealtime,
31
+)
32
+from vllm.model_executor.models.qwen3_asr import (
33
+ Qwen3ASRDummyInputsBuilder,
34
+ Qwen3ASRForConditionalGeneration,
35
+ Qwen3ASRMultiModalProcessor,
36
+ Qwen3ASRProcessingInfo,
37
+ _get_feat_extract_output_lengths,
38
+)
39
+from vllm.multimodal import MULTIMODAL_REGISTRY
40
+from vllm.multimodal.cache import _I, BaseMultiModalProcessorCache
41
+from vllm.multimodal.inputs import MultiModalKwargsOptionalItems
42
+from vllm.multimodal.parse import MultiModalDataItems
43
+from vllm.multimodal.processing import BaseDummyInputsBuilder
44
+from vllm.multimodal.processing.processor import (
45
+ MultiModalPromptUpdates,
46
+ PlaceholderFeaturesInfo,
47
+)
48
+from vllm.tokenizers import cached_tokenizer_from_config
49
+from vllm.transformers_utils.processor import cached_processor_from_config
50
+
51
+logger = init_logger(__name__)
52
+
53
+_PRE_ALLOCATE_BUFFER_SIZE_IN_S = 60
54
+
55
+
56
+class Qwen3ASRRealtimeBuffer:
57
+ """Audio buffer for Qwen3-ASR realtime streaming.
58
+
59
+ Accumulates audio samples and yields segments when enough
60
+ audio has been buffered for processing.
61
+ """
62
+
63
+ def __init__(self, sampling_rate: int, segment_duration_s: float = 5.0):
64
+ self._sampling_rate = sampling_rate
65
+ self._segment_size = int(segment_duration_s * sampling_rate)
66
+
67
+ self._buffer_size = _PRE_ALLOCATE_BUFFER_SIZE_IN_S * sampling_rate
68
+ self._buffer: np.ndarray = np.empty(self._buffer_size, dtype=np.float32)
69
+ self._filled_len = 0
70
+
71
+ def write_audio(self, audio: np.ndarray) -> None:
72
+ put_end = self._filled_len + len(audio)
73
+ if put_end > self._buffer_size:
74
+ new_size = max(self._buffer_size * 2, put_end)
75
+ new_buffer = np.empty(new_size, dtype=np.float32)
76
+ new_buffer[: self._filled_len] = self._buffer[: self._filled_len]
77
+ self._buffer = new_buffer
78
+ self._buffer_size = new_size
79
+
80
+ self._buffer[self._filled_len : put_end] = audio
81
+ self._filled_len = put_end
82
+
83
+ def read_audio(self) -> np.ndarray | None:
84
+ if self._filled_len < self._segment_size:
85
+ return None
86
+
87
+ segment = self._buffer[: self._segment_size].copy()
88
+ remaining = self._filled_len - self._segment_size
89
+ if remaining > 0:
90
+ self._buffer[:remaining] = self._buffer[
91
+ self._segment_size : self._filled_len
92
+ ]
93
+ self._filled_len = remaining
94
+ return segment
95
+
96
+ def flush(self) -> np.ndarray | None:
97
+ if self._filled_len == 0:
98
+ return None
99
+ audio = self._buffer[: self._filled_len].copy()
100
+ self._filled_len = 0
101
+ return audio
102
+
103
+
104
+class Qwen3ASRRealtimeMultiModalProcessor(Qwen3ASRMultiModalProcessor):
105
+ def __init__(
106
+ self,
107
+ info: _I,
108
+ dummy_inputs: BaseDummyInputsBuilder[_I],
109
+ *,
110
+ cache: BaseMultiModalProcessorCache | None = None,
111
+ ) -> None:
112
+ super().__init__(info, dummy_inputs, cache=None)
113
+
114
+ def _maybe_apply_prompt_updates(
115
+ self,
116
+ mm_items: MultiModalDataItems,
117
+ prompt_ids: list[int],
118
+ mm_kwargs: MultiModalKwargsOptionalItems,
119
+ mm_prompt_updates: MultiModalPromptUpdates,
120
+ is_update_applied: bool,
121
+ ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
122
+ audios = mm_kwargs.get("audio", [])
123
+ assert len(audios) == 1, (
124
+ f"Expected only one audio input for realtime, got {len(audios)}"
125
+ )
126
+
127
+ audio_data = audios[0]
128
+ audio_feature_lengths = audio_data.get("audio_feature_lengths")
129
+ if audio_feature_lengths is not None:
130
+ if isinstance(audio_feature_lengths.data, torch.Tensor):
131
+ audio_len = _get_feat_extract_output_lengths(
132
+ audio_feature_lengths.data
133
+ ).item()
134
+ else:
135
+ audio_len = int(
136
+ _get_feat_extract_output_lengths(
137
+ torch.tensor(audio_feature_lengths.data)
138
+ ).item()
139
+ )
140
+ else:
141
+ audio_len = 0
142
+
143
+ # Get audio_pad token ID and expand placeholder in prompt_ids
144
+ # so that MRoPE position computation matches seq_len.
145
+ tokenizer = self.info.get_tokenizer()
146
+ audio_pad_id = tokenizer.convert_tokens_to_ids("<|audio_pad|>")
147
+
148
+ # Find the audio_pad token position and expand it to audio_len tokens
149
+ expanded_ids = list[int]()
150
+ pad_start_idx = -1
151
+ for i, tid in enumerate(prompt_ids):
152
+ if tid == audio_pad_id and pad_start_idx == -1:
153
+ pad_start_idx = i
154
+ expanded_ids.extend([audio_pad_id] * audio_len)
155
+ else:
156
+ expanded_ids.append(tid)
157
+
158
+ if pad_start_idx == -1:
159
+ pad_start_idx = 0
160
+
161
+ features_info = PlaceholderFeaturesInfo(
162
+ modality="audio",
163
+ item_idx=0,
164
+ start_idx=pad_start_idx,
165
+ tokens=audio_len * [audio_pad_id],
166
+ is_embed=None,
167
+ )
168
+ return expanded_ids, {"audio": [features_info]}
169
+
170
+
171
+# NOTE: A separate model class is required here because the multimodal
172
+# processor registry binds one processor per model class. The realtime
173
+# endpoint needs a different processor (Qwen3ASRRealtimeMultiModalProcessor)
174
+# than the base transcription endpoint, so we register it on this subclass.
175
+@MULTIMODAL_REGISTRY.register_processor(
176
+ Qwen3ASRRealtimeMultiModalProcessor,
177
+ info=Qwen3ASRProcessingInfo,
178
+ dummy_inputs=Qwen3ASRDummyInputsBuilder,
179
+)
180
+@support_torch_compile
181
+class Qwen3ASRRealtimeGeneration(Qwen3ASRForConditionalGeneration, SupportsRealtime):
182
+ realtime_max_tokens = 64
183
+
184
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
185
+ super().__init__(vllm_config=vllm_config, prefix=prefix)
186
+
187
+ @classmethod
188
+ async def buffer_realtime_audio(
189
+ cls,
190
+ audio_stream: AsyncGenerator[np.ndarray, None],
191
+ input_stream: asyncio.Queue[list[int]],
192
+ model_config: ModelConfig,
193
+ ) -> AsyncGenerator[PromptType, None]:
194
+ processor = cached_processor_from_config(model_config)
195
+ feature_extractor = processor.feature_extractor
196
+ sampling_rate = feature_extractor.sampling_rate
197
+ tokenizer = cached_tokenizer_from_config(model_config)
198
+
199
+ # Use a small segment size for low-latency streaming.
200
+ segment_duration_s = 5.0
201
+ buffer = Qwen3ASRRealtimeBuffer(
202
+ sampling_rate=sampling_rate,
203
+ segment_duration_s=segment_duration_s,
204
+ )
205
+
206
+ audio_placeholder = cls.get_placeholder_str("audio", 0)
207
+ prompt_template = (
208
+ f"<|im_start|>user\n{audio_placeholder}<|im_end|>\n<|im_start|>assistant\n"
209
+ )
210
+
211
+ prompt_token_ids = tokenizer.encode(prompt_template)
212
+
213
+ async for audio_chunk in audio_stream:
214
+ buffer.write_audio(audio_chunk)
215
+
216
+ while (segment := buffer.read_audio()) is not None:
217
+ yield TokensPrompt(
218
+ prompt_token_ids=prompt_token_ids,
219
+ multi_modal_data={"audio": segment},
220
+ )
221
+
222
+ remaining = buffer.flush()
223
+ if remaining is not None and len(remaining) > 0:
224
+ yield TokensPrompt(
225
+ prompt_token_ids=prompt_token_ids,
226
+ multi_modal_data={"audio": remaining},
227
+ )
228
+
229
+ @classmethod
230
+ def get_speech_to_text_config(
231
+ cls, model_config: ModelConfig, task_type: str
232
+ ) -> SpeechToTextConfig:
233
+ processor = cached_processor_from_config(model_config)
234
+ feature_extractor = processor.feature_extractor
235
+ return SpeechToTextConfig(
236
+ max_audio_clip_s=None,
237
+ sample_rate=feature_extractor.sampling_rate,
238
+ min_energy_split_window_size=None,
239
+ )
vllm/model_executor/models/registry.py
+4
@@ -468,6 +468,10 @@ _MULTIMODAL_MODELS = {
468
"qwen3_asr",
469
"Qwen3ASRForConditionalGeneration",
470
),
471
+ "Qwen3ASRRealtimeGeneration": (
472
+ "qwen3_asr_realtime",
473
+ "Qwen3ASRRealtimeGeneration",
474
+ ),
475
"Qwen3VLForConditionalGeneration": ("qwen3_vl", "Qwen3VLForConditionalGeneration"), # noqa: E501
476
"Qwen3VLMoeForConditionalGeneration": (
477
"qwen3_vl_moe",