A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 17:00 UTC
[Frontend] Remove `torchcodec` from audio dependency (#37061)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Mar 15, 2026 at 13:15 UTC
6590a3ecdafdc001f29c3820b80dd14d994e640c
3 files changed
+39
-109
setup.py
-1
@@ -977,7 +977,6 @@ setup(
977
"soundfile",
978
"mistral_common[audio]",
979
"av",
980
- "torchcodec",
980
], # Required for audio processing
981
"video": [], # Kept for backwards compatibility
982
"flashinfer": [], # Kept for backwards compatibility
vllm/entrypoints/openai/speech_to_text/speech_to_text.py
+39
-2
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import asyncio
4
+import io
5
import math
6
import time
7
import zlib
@@ -35,7 +36,6 @@ from vllm.entrypoints.openai.speech_to_text.protocol import (
36
TranslationSegment,
37
TranslationStreamResponse,
38
)
38
-from vllm.entrypoints.openai.speech_to_text.utils import load_audio_bytes
39
from vllm.entrypoints.utils import get_max_tokens
40
from vllm.exceptions import VLLMValidationError
41
from vllm.inputs import EncoderDecoderInputs, ProcessorInputs
@@ -43,6 +43,7 @@ from vllm.logger import init_logger
43
from vllm.logprobs import FlatLogprobs, Logprob
44
from vllm.model_executor.models import SupportsTranscription
45
from vllm.multimodal.audio import split_audio
46
+from vllm.multimodal.media.audio import extract_audio_from_video_bytes
47
from vllm.outputs import RequestOutput
48
from vllm.renderers.inputs import DictPrompt, EncoderDecoderDictPrompt
49
from vllm.renderers.inputs.preprocess import parse_enc_dec_prompt, parse_model_prompt
@@ -55,6 +56,19 @@ try:
56
except ImportError:
57
librosa = PlaceholderModule("librosa") # type: ignore[assignment]
58
59
+try:
60
+ import soundfile as sf
61
+except ImportError:
62
+ sf = PlaceholderModule("soundfile") # type: ignore[assignment]
63
+
64
+# Public libsndfile error codes exposed via `soundfile.LibsndfileError.code`, soundfile
65
+# being librosa's main backend. Used to validate if an audio loading error is due to a
66
+# server error vs a client error (invalid audio file).
67
+# 1 = unrecognised format (file is not a supported audio container)
68
+# 3 = malformed file (corrupt or structurally invalid audio)
69
+# 4 = unsupported encoding (codec not supported by this libsndfile build)
70
+_BAD_SF_CODES = {1, 3, 4}
71
+
72
SpeechToTextResponse: TypeAlias = TranscriptionResponse | TranslationResponse
73
SpeechToTextResponseVerbose: TypeAlias = (
74
TranscriptionResponseVerbose | TranslationResponseVerbose
@@ -198,7 +212,30 @@ class OpenAISpeechToText(OpenAIServing):
212
# transparently falls back to ffmpeg via an in-memory fd.
213
# NOTE resample to model SR here for efficiency. This is also a
214
# pre-requisite for chunking, as it assumes Whisper SR.
201
- y, sr = load_audio_bytes(audio_data, sr=self.asr_config.sample_rate)
215
+ try:
216
+ with io.BytesIO(audio_data) as buf:
217
+ y, sr = librosa.load(buf, sr=self.asr_config.sample_rate) # type: ignore[return-value]
218
+ except sf.LibsndfileError as exc:
219
+ # Only fall back for known format-detection failures.
220
+ # Re-raise anything else (e.g. corrupt but recognised format).
221
+ if exc.code not in _BAD_SF_CODES:
222
+ raise
223
+ logger.debug(
224
+ "librosa/soundfile could not decode audio from BytesIO "
225
+ "(code=%s: %s); falling back to pyav in-process decode",
226
+ exc.code,
227
+ exc,
228
+ )
229
+ try:
230
+ native_y, native_sr = extract_audio_from_video_bytes(audio_data)
231
+ sr = self.asr_config.sample_rate
232
+ y = librosa.resample(native_y, orig_sr=native_sr, target_sr=sr)
233
+ except Exception as pyav_exc:
234
+ logger.debug(
235
+ "pyAV fallback also failed: %s",
236
+ pyav_exc,
237
+ )
238
+ raise ValueError("Invalid or unsupported audio file.") from pyav_exc
239
240
duration = librosa.get_duration(y=y, sr=sr)
241
do_split_audio = (
vllm/entrypoints/openai/speech_to_text/utils.py
deleted
-106
@@ -1,106 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-"""Audio decoding utilities for the speech-to-text endpoints."""
4
-
5
-import io
6
-
7
-import numpy as np
8
-import torchaudio
9
-
10
-from vllm.logger import init_logger
11
-from vllm.utils.import_utils import PlaceholderModule
12
-
13
-try:
14
- import librosa
15
-except ImportError:
16
- librosa = PlaceholderModule("librosa") # type: ignore[assignment]
17
-
18
-try:
19
- import soundfile as sf
20
-except ImportError:
21
- sf = PlaceholderModule("soundfile") # type: ignore[assignment]
22
-
23
-logger = init_logger(__name__)
24
-
25
-# Public libsndfile error codes exposed via ``soundfile.LibsndfileError.code``.
26
-# soundfile is librosa's primary backend. These codes indicate that the audio
27
-# data itself is problematic (unrecognised container, corrupt file, or
28
-# unsupported encoding) rather than a transient server error.
29
-# 1 = unrecognised format, 3 = malformed file, 4 = unsupported encoding
30
-_BAD_SF_CODES = {1, 3, 4}
31
-
32
-
33
-def _decode_audio_bytes_torchaudio(
34
- audio_data: bytes,
35
- sr: int,
36
-) -> tuple[np.ndarray, int]:
37
- """Decode audio bytes to mono float32 PCM via torchaudio, in-process.
38
-
39
- ``torchaudio.load`` (backed by TorchCodec / FFmpeg) can decode
40
- container formats (MP4, M4A, WebM) directly from a ``BytesIO``
41
- buffer without spawning a subprocess. The decoded waveform is
42
- down-mixed to mono and resampled to *sr* Hz, matching the return
43
- convention of ``librosa.load``.
44
- """
45
- buf = io.BytesIO(audio_data)
46
- waveform, orig_sr = torchaudio.load(buf)
47
-
48
- # Down-mix to mono (average across channels).
49
- if waveform.shape[0] > 1:
50
- waveform = waveform.mean(dim=0, keepdim=True)
51
-
52
- # Resample to the target sample rate when necessary.
53
- if orig_sr != sr:
54
- waveform = torchaudio.functional.resample(
55
- waveform, orig_freq=orig_sr, new_freq=sr
56
- )
57
-
58
- # Squeeze channel dim → 1-D float32 numpy array (same as librosa.load).
59
- y = waveform.squeeze(0).numpy()
60
- if y.size == 0:
61
- raise RuntimeError(
62
- "torchaudio produced no audio samples (file may be empty or corrupt)"
63
- )
64
- return y, sr
65
-
66
-
67
-def load_audio_bytes(
68
- audio_data: bytes,
69
- sr: int | float,
70
-) -> tuple[np.ndarray, int]:
71
- """Load audio from raw bytes, with an in-process torchaudio fallback.
72
-
73
- First tries ``librosa.load(BytesIO(...))`` which works for formats
74
- that *soundfile* can auto-detect (WAV, FLAC, MP3, OGG, ...). If
75
- that fails with a ``LibsndfileError`` indicating an unrecognised or
76
- unsupported format (typically container formats like MP4/M4A/WebM),
77
- the bytes are decoded in-process via ``torchaudio`` (backed by
78
- TorchCodec / FFmpeg) which handles these containers natively.
79
- """
80
- sr = int(sr)
81
-
82
- # Fast path: librosa + soundfile (works for most formats).
83
- try:
84
- with io.BytesIO(audio_data) as buf:
85
- return librosa.load(buf, sr=sr) # type: ignore[return-value]
86
- except sf.LibsndfileError as exc:
87
- # Only fall back for known format-detection failures.
88
- # Re-raise anything else (e.g. corrupt but recognised format).
89
- if exc.code not in _BAD_SF_CODES:
90
- raise
91
- logger.debug(
92
- "librosa/soundfile could not decode audio from BytesIO "
93
- "(code=%s: %s); falling back to torchaudio in-process decode",
94
- exc.code,
95
- exc,
96
- )
97
-
98
- # Fallback: torchaudio in-process decode (no subprocess overhead).
99
- try:
100
- return _decode_audio_bytes_torchaudio(audio_data, sr)
101
- except Exception as ta_exc:
102
- logger.debug(
103
- "torchaudio fallback also failed: %s",
104
- ta_exc,
105
- )
106
- raise ValueError("Invalid or unsupported audio file.") from ta_exc