A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 18:15 UTC
[Bugfix][Frontend] Fix audio transcription for MP4, M4A, and WebM formats (#35109)
Signed-off-by: seanmamasde <seanmamasde@gmail.com>
seanmamasde committed
Mar 14, 2026 at 23:44 UTC
84868e479374d6b7b8b162e6bc2a1873e6dec7e2
3 files changed
+114
-20
setup.py
+1
@@ -976,6 +976,7 @@ setup(
976
"soundfile",
977
"mistral_common[audio]",
978
"av",
979
+ "torchcodec",
980
], # Required for audio processing
981
"video": [], # Kept for backwards compatibility
982
"flashinfer": [], # Kept for backwards compatibility
vllm/entrypoints/openai/speech_to_text/speech_to_text.py
+7
-20
@@ -1,7 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import asyncio
4
-import io
4
import math
5
import time
6
import zlib
@@ -11,7 +10,6 @@ from typing import Final, Literal, TypeAlias, TypeVar, cast
10
11
import numpy as np
12
from fastapi import Request
14
-from soundfile import LibsndfileError
13
from transformers import PreTrainedTokenizerBase
14
15
import vllm.envs as envs
@@ -37,6 +35,7 @@ from vllm.entrypoints.openai.speech_to_text.protocol import (
35
TranslationSegment,
36
TranslationStreamResponse,
37
)
38
+from vllm.entrypoints.openai.speech_to_text.utils import load_audio_bytes
39
from vllm.entrypoints.utils import get_max_tokens
40
from vllm.exceptions import VLLMValidationError
41
from vllm.inputs import EncoderDecoderInputs, ProcessorInputs
@@ -56,14 +55,6 @@ try:
55
except ImportError:
56
librosa = PlaceholderModule("librosa") # type: ignore[assignment]
57
59
-# Public libsndfile error codes exposed via `soundfile.LibsndfileError.code`, soundfile
60
-# being librosa's main backend. Used to validate if an audio loading error is due to a
61
-# server error vs a client error (invalid audio file).
62
-# 1 = unrecognised format (file is not a supported audio container)
63
-# 3 = malformed file (corrupt or structurally invalid audio)
64
-# 4 = unsupported encoding (codec not supported by this libsndfile build)
65
-_BAD_SF_CODES = {1, 3, 4}
66
-
58
SpeechToTextResponse: TypeAlias = TranscriptionResponse | TranslationResponse
59
SpeechToTextResponseVerbose: TypeAlias = (
60
TranscriptionResponseVerbose | TranslationResponseVerbose
@@ -202,16 +193,12 @@ class OpenAISpeechToText(OpenAIServing):
193
value=len(audio_data) / 1024**2,
194
)
195
205
- with io.BytesIO(audio_data) as bytes_:
206
- try:
207
- # NOTE resample to model SR here for efficiency. This is also a
208
- # pre-requisite for chunking, as it assumes Whisper SR.
209
- y, sr = librosa.load(bytes_, sr=self.asr_config.sample_rate)
210
- except LibsndfileError as exc:
211
- # Distinguish client errors (invalid audio) from server errors
212
- if exc.code in _BAD_SF_CODES:
213
- raise ValueError("Invalid or unsupported audio file.") from exc
214
- raise
196
+ # Decode audio bytes. For container formats (MP4, M4A, WebM) that
197
+ # soundfile cannot detect from a BytesIO stream, _load_audio_bytes
198
+ # transparently falls back to ffmpeg via an in-memory fd.
199
+ # NOTE resample to model SR here for efficiency. This is also a
200
+ # pre-requisite for chunking, as it assumes Whisper SR.
201
+ y, sr = load_audio_bytes(audio_data, sr=self.asr_config.sample_rate)
202
203
duration = librosa.get_duration(y=y, sr=sr)
204
do_split_audio = (
vllm/entrypoints/openai/speech_to_text/utils.py
new
+106
@@ -0,0 +1,106 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Audio decoding utilities for the speech-to-text endpoints."""
4
+
5
+import io
6
+
7
+import numpy as np
8
+import torchaudio
9
+
10
+from vllm.logger import init_logger
11
+from vllm.utils.import_utils import PlaceholderModule
12
+
13
+try:
14
+ import librosa
15
+except ImportError:
16
+ librosa = PlaceholderModule("librosa") # type: ignore[assignment]
17
+
18
+try:
19
+ import soundfile as sf
20
+except ImportError:
21
+ sf = PlaceholderModule("soundfile") # type: ignore[assignment]
22
+
23
+logger = init_logger(__name__)
24
+
25
+# Public libsndfile error codes exposed via ``soundfile.LibsndfileError.code``.
26
+# soundfile is librosa's primary backend. These codes indicate that the audio
27
+# data itself is problematic (unrecognised container, corrupt file, or
28
+# unsupported encoding) rather than a transient server error.
29
+# 1 = unrecognised format, 3 = malformed file, 4 = unsupported encoding
30
+_BAD_SF_CODES = {1, 3, 4}
31
+
32
+
33
+def _decode_audio_bytes_torchaudio(
34
+ audio_data: bytes,
35
+ sr: int,
36
+) -> tuple[np.ndarray, int]:
37
+ """Decode audio bytes to mono float32 PCM via torchaudio, in-process.
38
+
39
+ ``torchaudio.load`` (backed by TorchCodec / FFmpeg) can decode
40
+ container formats (MP4, M4A, WebM) directly from a ``BytesIO``
41
+ buffer without spawning a subprocess. The decoded waveform is
42
+ down-mixed to mono and resampled to *sr* Hz, matching the return
43
+ convention of ``librosa.load``.
44
+ """
45
+ buf = io.BytesIO(audio_data)
46
+ waveform, orig_sr = torchaudio.load(buf)
47
+
48
+ # Down-mix to mono (average across channels).
49
+ if waveform.shape[0] > 1:
50
+ waveform = waveform.mean(dim=0, keepdim=True)
51
+
52
+ # Resample to the target sample rate when necessary.
53
+ if orig_sr != sr:
54
+ waveform = torchaudio.functional.resample(
55
+ waveform, orig_freq=orig_sr, new_freq=sr
56
+ )
57
+
58
+ # Squeeze channel dim → 1-D float32 numpy array (same as librosa.load).
59
+ y = waveform.squeeze(0).numpy()
60
+ if y.size == 0:
61
+ raise RuntimeError(
62
+ "torchaudio produced no audio samples (file may be empty or corrupt)"
63
+ )
64
+ return y, sr
65
+
66
+
67
+def load_audio_bytes(
68
+ audio_data: bytes,
69
+ sr: int | float,
70
+) -> tuple[np.ndarray, int]:
71
+ """Load audio from raw bytes, with an in-process torchaudio fallback.
72
+
73
+ First tries ``librosa.load(BytesIO(...))`` which works for formats
74
+ that *soundfile* can auto-detect (WAV, FLAC, MP3, OGG, ...). If
75
+ that fails with a ``LibsndfileError`` indicating an unrecognised or
76
+ unsupported format (typically container formats like MP4/M4A/WebM),
77
+ the bytes are decoded in-process via ``torchaudio`` (backed by
78
+ TorchCodec / FFmpeg) which handles these containers natively.
79
+ """
80
+ sr = int(sr)
81
+
82
+ # Fast path: librosa + soundfile (works for most formats).
83
+ try:
84
+ with io.BytesIO(audio_data) as buf:
85
+ return librosa.load(buf, sr=sr) # type: ignore[return-value]
86
+ except sf.LibsndfileError as exc:
87
+ # Only fall back for known format-detection failures.
88
+ # Re-raise anything else (e.g. corrupt but recognised format).
89
+ if exc.code not in _BAD_SF_CODES:
90
+ raise
91
+ logger.debug(
92
+ "librosa/soundfile could not decode audio from BytesIO "
93
+ "(code=%s: %s); falling back to torchaudio in-process decode",
94
+ exc.code,
95
+ exc,
96
+ )
97
+
98
+ # Fallback: torchaudio in-process decode (no subprocess overhead).
99
+ try:
100
+ return _decode_audio_bytes_torchaudio(audio_data, sr)
101
+ except Exception as ta_exc:
102
+ logger.debug(
103
+ "torchaudio fallback also failed: %s",
104
+ ta_exc,
105
+ )
106
+ raise ValueError("Invalid or unsupported audio file.") from ta_exc