A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 12:45 UTC
[Voxtral] Fix speech transcription api (#31388)
Signed-off-by: Patrick von Platen <patrick.v.platen@gmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: bk-201 <joy25810@foxmail.com> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: prashanth058 <prashanth.dannamaneni@uipath.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: bk-201 <joy25810@foxmail.com> Co-authored-by: prashanth058 <prashanth.dannamaneni@uipath.com> Co-authored-by: Anexdeus <5142168@mail.ru> Co-authored-by: Julien Denize <40604584+juliendenize@users.noreply.github.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@gmail.com>
Patrick von Platen committed
Jan 8, 2026 at 12:34 UTC
18d4e481d0ce8336f4e64da70f7b6076ad804971
5 files changed
+114
-27
vllm/config/speech_to_text.py
+3
-2
@@ -17,10 +17,11 @@ class SpeechToTextConfig:
17
16kHz audio input. The input audio will be automatically resampled to this
18
rate before processing."""
19
20
- max_audio_clip_s: int = 30
20
+ max_audio_clip_s: int | None = 30
21
"""Maximum duration in seconds for a single audio clip without chunking.
22
Audio longer than this will be split into smaller chunks if
23
- `allow_audio_chunking` evaluates to True, otherwise it will be rejected."""
23
+ `allow_audio_chunking` evaluates to True, otherwise it will be rejected.
24
+ `None` means audio duration can be unlimited and won't be chunked."""
25
26
overlap_chunk_second: int = 1
27
"""Overlap duration in seconds between consecutive audio chunks when
vllm/entrypoints/openai/speech_to_text.py
+13
-1
@@ -477,7 +477,15 @@ class OpenAISpeechToText(OpenAIServing):
477
}
478
segment_class: type[SpeechToTextSegment] = segments_types[self.task_type]
479
text = ""
480
+ chunk_size_in_s = self.asr_config.max_audio_clip_s
481
+ if chunk_size_in_s is None:
482
+ assert len(list_result_generator) == 1, (
483
+ "`max_audio_clip_s` is set to None, audio cannot be chunked"
484
+ )
485
for idx, result_generator in enumerate(list_result_generator):
486
+ start_time = (
487
+ float(idx * chunk_size_in_s) if chunk_size_in_s is not None else 0.0
488
+ )
489
async for op in result_generator:
490
if request.response_format == "verbose_json":
491
segments: list[SpeechToTextSegment] = (
@@ -485,7 +493,7 @@ class OpenAISpeechToText(OpenAIServing):
493
tokens=tuple(op.outputs[0].token_ids),
494
segment_class=segment_class,
495
request=request,
488
- start_time=idx * self.asr_config.max_audio_clip_s,
496
+ start_time=start_time,
497
)
498
)
499
@@ -653,6 +661,10 @@ class OpenAISpeechToText(OpenAIServing):
661
def _split_audio(
662
self, audio_data: np.ndarray, sample_rate: int
663
) -> list[np.ndarray]:
664
+ assert self.asr_config.max_audio_clip_s is not None, (
665
+ f"{self.asr_config.max_audio_clip_s=} cannot be None to"
666
+ " split audio into chunks."
667
+ )
668
chunk_size = sample_rate * self.asr_config.max_audio_clip_s
669
overlap_size = sample_rate * self.asr_config.overlap_chunk_second
670
chunks = []
vllm/model_executor/models/voxtral.py
+16
-8
@@ -17,7 +17,11 @@ from mistral_common.protocol.instruct.chunk import AudioChunk, RawAudio, TextChu
17
from mistral_common.protocol.instruct.messages import UserMessage
18
from mistral_common.protocol.instruct.request import ChatCompletionRequest
19
from mistral_common.protocol.transcription.request import TranscriptionRequest
20
-from mistral_common.tokens.tokenizers.audio import Audio, AudioEncoder
20
+from mistral_common.tokens.tokenizers.audio import (
21
+ Audio,
22
+ AudioEncoder,
23
+ TranscriptionFormat,
24
+)
25
from transformers import BatchFeature, TensorType, WhisperConfig
26
from transformers.tokenization_utils_base import TextInput
27
@@ -157,13 +161,17 @@ class VoxtralProcessorAdapter:
161
162
# pad if necessary
163
# TODO(Patrick) - remove once mistral-common is bumped
160
- sig = inspect.signature(self._audio_processor.pad)
161
- if "is_online_streaming" in sig.parameters:
162
- audio = self._audio_processor.pad(
163
- audio, self.sampling_rate, is_online_streaming=False
164
- )
165
- else:
166
- audio = self._audio_processor.pad(audio, self.sampling_rate)
164
+ if (
165
+ self._audio_processor.audio_config.transcription_format
166
+ != TranscriptionFormat.STREAMING
167
+ ):
168
+ sig = inspect.signature(self._audio_processor.pad)
169
+ if "is_online_streaming" in sig.parameters:
170
+ audio = self._audio_processor.pad(
171
+ audio, self.sampling_rate, is_online_streaming=False
172
+ )
173
+ else:
174
+ audio = self._audio_processor.pad(audio, self.sampling_rate)
175
176
audio_tokens = [self.begin_audio_token_id] + [
177
self.audio_token_id
vllm/model_executor/models/voxtral_streaming.py
+73
-12
@@ -3,10 +3,19 @@
3
4
import math
5
from collections.abc import Mapping
6
+from typing import Literal, cast
7
8
+import numpy as np
9
import torch
10
+from mistral_common.protocol.instruct.chunk import RawAudio
11
+from mistral_common.protocol.transcription.request import (
12
+ StreamingMode,
13
+ TranscriptionRequest,
14
+)
15
+from mistral_common.tokens.tokenizers.audio import Audio
16
9
-from vllm.config.vllm import VllmConfig
17
+from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
18
+from vllm.inputs.data import PromptType
19
from vllm.logger import init_logger
20
from vllm.model_executor.models.interfaces import MultiModalEmbeddings
21
from vllm.model_executor.models.voxtral import (
@@ -27,6 +36,7 @@ from vllm.multimodal.processing import (
36
)
37
from vllm.multimodal.profiling import BaseDummyInputsBuilder
38
from vllm.sequence import IntermediateTensors
39
+from vllm.tokenizers import cached_tokenizer_from_config
40
41
from .utils import (
42
_flatten_embeddings,
@@ -205,13 +215,17 @@ class VoxtralStreamingGeneration(VoxtralForConditionalGeneration):
215
"For streaming you must provide an audio input at every step."
216
)
217
208
- multiple_of = self.audio_config.raw_audio_length_per_tok
209
- assert all(
210
- (this_audio := audio.shape[0]) % multiple_of == 0 for audio in audio_inputs
211
- ), (
212
- f"Every input audio waveform has to be a multiple of {multiple_of}, but"
213
- f" one is {this_audio} with {(this_audio / multiple_of)=}."
214
- )
218
+ def _truncate_left(
219
+ sample: torch.Tensor, mult_of: int, pos: int
220
+ ) -> torch.Tensor:
221
+ assert pos in [0, 1], pos
222
+ if (ctx := sample.shape[pos] % mult_of) != 0:
223
+ sample = sample[ctx:] if pos == 0 else sample[:, ctx:]
224
+ assert sample.shape[pos] > 0, (
225
+ f"Sample is empty after truncation with ctx {ctx}"
226
+ )
227
+
228
+ return sample
229
230
mel_features = [
231
self.whisper_encoder.compute_whisper_melspec(audio).to(
@@ -219,11 +233,16 @@ class VoxtralStreamingGeneration(VoxtralForConditionalGeneration):
233
)
234
for audio in audio_inputs
235
]
236
+
237
+ # we truncate the left most mel feature
238
+ # if the sequence length in impair
239
+ mel_features = [_truncate_left(mel, 2, 1) for mel in mel_features]
240
+
241
seq_lens = [mel.shape[1] for mel in mel_features]
242
# [total_num_20ms_frames, hidden_size]
243
audio_embeddings = self.whisper_encoder.whisper_encoder.forward_conv(
244
mel_features
226
- )[0]
245
+ )
246
conv_stride = self.whisper_encoder.whisper_encoder.total_stride
247
audio_embeddings_per_sample = audio_embeddings.split(
248
[s // conv_stride for s in seq_lens], dim=0
@@ -231,13 +250,55 @@ class VoxtralStreamingGeneration(VoxtralForConditionalGeneration):
250
251
# audio_embeddings per sample need to be divisible by 4
252
pool_size = self.config.audio_config.block_pool_size
234
- assert all(
235
- (this_shape := sample.shape[0]) % pool_size == 0
253
+
254
+ audio_embeddings_per_sample = [
255
+ _truncate_left(sample, pool_size, 0)
256
for sample in audio_embeddings_per_sample
237
- ), f"Every audio embedding has to be a multiple of 4, but one is {this_shape}."
257
+ ]
258
259
audio_embeddings_per_sample = [
260
e.view(e.shape[0] // pool_size, e.shape[1] * pool_size)
261
for e in audio_embeddings_per_sample
262
]
263
return audio_embeddings_per_sample
264
+
265
+ @classmethod
266
+ def get_speech_to_text_config(
267
+ cls, model_config: ModelConfig, task_type: str
268
+ ) -> SpeechToTextConfig:
269
+ tokenizer = cached_tokenizer_from_config(model_config)
270
+ audio_config = tokenizer.instruct.audio_encoder.audio_config
271
+ sample_rate = audio_config.sampling_rate
272
+ return SpeechToTextConfig(
273
+ max_audio_clip_s=None, # only limited by memory
274
+ sample_rate=sample_rate,
275
+ min_energy_split_window_size=None,
276
+ )
277
+
278
+ @classmethod
279
+ # for speech-to-text transcription
280
+ def get_generation_prompt(
281
+ cls,
282
+ audio: np.ndarray,
283
+ model_config: ModelConfig,
284
+ stt_config: SpeechToTextConfig,
285
+ language: str | None,
286
+ task_type: Literal["transcribe", "translate"],
287
+ request_prompt: str,
288
+ to_language: str | None,
289
+ ) -> PromptType:
290
+ tokenizer = cached_tokenizer_from_config(model_config)
291
+ audio = Audio(audio, int(stt_config.sample_rate), format="wav") # lossless
292
+
293
+ req = TranscriptionRequest(
294
+ model=model_config.model,
295
+ audio=RawAudio.from_audio(audio),
296
+ language=language,
297
+ streaming=StreamingMode.OFFLINE,
298
+ )
299
+
300
+ tokenized = tokenizer.instruct.encode_transcription(req)
301
+ audio = (tokenized.audios[0].audio_array, stt_config.sample_rate)
302
+ prompts_dict = {"multi_modal_data": {"audio": audio}}
303
+ prompts_dict["prompt_token_ids"] = tokenized.tokens
304
+ return cast(PromptType, prompts_dict)
vllm/model_executor/models/whisper.py
+9
-4
@@ -469,8 +469,10 @@ class WhisperEncoder(nn.Module):
469
self.max_source_positions = config.max_source_positions
470
self.embed_scale = math.sqrt(embed_dim) if config.scale_embedding else 1.0
471
472
- is_causal = getattr(config, "is_causal", False)
473
- Conv1d = WhisperCausalConv1d if is_causal else partial(nn.Conv1d, padding=1)
472
+ self.is_causal = getattr(config, "is_causal", False)
473
+ Conv1d = (
474
+ WhisperCausalConv1d if self.is_causal else partial(nn.Conv1d, padding=1)
475
+ )
476
477
self.conv1 = Conv1d(self.num_mel_bins, embed_dim, kernel_size=3)
478
self.conv2 = Conv1d(embed_dim, embed_dim, stride=2, kernel_size=3)
@@ -485,7 +487,7 @@ class WhisperEncoder(nn.Module):
487
)
488
self.layer_norm = nn.LayerNorm(config.d_model)
489
488
- if is_causal and self.pos_embed_type != WhisperPosEmbedType.NOPE:
490
+ if self.is_causal and self.pos_embed_type != WhisperPosEmbedType.NOPE:
491
raise ValueError(
492
"Only NOPE position embeddings are supported "
493
f"for causal models, but got {self.pos_embed_type}"
@@ -536,8 +538,11 @@ class WhisperEncoder(nn.Module):
538
hidden_states.append(embeds)
539
input_is_batched = embeds.ndim > 2
540
# Input to MHA must be B x T x D
539
- if input_is_batched:
541
+ if input_is_batched or self.is_causal:
542
# Models using WhisperEncoder may handle batching internally.
543
+ # If WhisperEncoder is causal, sequences
544
+ # are not padded to have identical seq length (T)
545
+ # => concat over feature dim
546
hidden_states = torch.cat(hidden_states)
547
else:
548
hidden_states = torch.stack(hidden_states, dim=0)