[feat] add torchcodec as audio loader and implement selective audio backend (#51826)
Signed-off-by: JaredforReal <w13431838023@gmail.com> Co-authored-by: saltman155 <saltman155@outlook.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Jared Wen committed
Sep 5, 2026 at 00:16 UTC
6a039f465e3770192085e551f642c7a218797d2d
4 files changed
+446
-67
docs/features/multimodal_inputs.md
+32
@@ -1204,6 +1204,38 @@ Full example: [examples/generate/multimodal/openai_chat_completion_client_for_mu
1204
export VLLM_AUDIO_FETCH_TIMEOUT=<timeout>
1205
```
1206
1207
+#### Audio Decoding Backend
1208
+
1209
+vLLM decodes audio bytes into waveforms using a selectable decoding backend:
1210
+
1211
+| Backend | Description |
1212
+| --- | --- |
1213
+| `auto` (default) | torchcodec, falling back to soundfile, then PyAV |
1214
+| `soundfile` | libsndfile only, no fallback |
1215
+| `pyav` | PyAV (FFmpeg) only, no fallback |
1216
+| `torchcodec` | TorchCodec (PyTorch-native) only, no fallback |
1217
+
1218
+Select the backend per server via `--media-io-kwargs`:
1219
+
1220
+```bash
1221
+vllm serve mistralai/Voxtral-Mini-3B-2507 \
1222
+ --media-io-kwargs '{"audio": {"audio_backend": "soundfile"}}'
1223
+```
1224
+
1225
+!!! tip
1226
+ `pyav` drives FFmpeg through a per-frame Python generator, so under
1227
+ concurrency the Python/C crossings contend on the GIL. `torchcodec`
1228
+ decodes each stream in a single call that releases the GIL for its whole
1229
+ duration, which is why `auto` prefers it when many requests decode audio
1230
+ concurrently, such as when audio tracks are extracted from video.
1231
+
1232
+!!! note
1233
+ `torchcodec` ships as a requirement on CUDA, CPU and XPU builds. On other
1234
+ platforms (e.g. ROCm, TPU) `auto` falls back to the soundfile → PyAV
1235
+ chain unless you install it manually. torchcodec also links against a
1236
+ system FFmpeg installation; if FFmpeg is missing, `auto` falls back the
1237
+ same way.
1238
+
1239
### Embedding Inputs
1240
1241
To input pre-computed embeddings belonging to a data type (i.e. image, video, or audio) directly to the language model,
tests/entrypoints/speech_to_text/transcription/test_transcription_validation_whisper.py
+1
-1
@@ -172,7 +172,7 @@ async def test_invalid_audio_file(whisper_client):
172
)
173
174
assert exc_info.value.status_code == 400
175
- assert "Invalid or unsupported audio file" in exc_info.value.message
175
+ assert "Invalid or corrupted audio data" in exc_info.value.message
176
177
178
@pytest.mark.asyncio
tests/multimodal/media/test_audio.py
+100
-1
@@ -10,7 +10,12 @@ import pytest
10
import soundfile as sf
11
12
from vllm.multimodal.media import AudioMediaIO
13
-from vllm.multimodal.media.audio import load_audio, load_audio_soundfile
13
+from vllm.multimodal.media import audio as audio_module
14
+from vllm.multimodal.media.audio import (
15
+ load_audio,
16
+ load_audio_soundfile,
17
+ load_audio_torchcodec,
18
+)
19
20
from ...conftest import AudioTestAssets
21
@@ -170,3 +175,97 @@ def test_load_audio_threads_max_decode_bytes():
175
max_duration_s=600,
176
max_decode_bytes=512 * 1024,
177
)
178
+
179
+
180
+@pytest.mark.parametrize("backend", ["soundfile", "pyav", "torchcodec"])
181
+def test_load_audio_backend_matches_default(backend, dummy_audio_bytes):
182
+ """Every explicit backend must decode to the same samples as `auto`."""
183
+ if backend == "torchcodec":
184
+ pytest.importorskip("torchcodec")
185
+ ref_audio, ref_sr = load_audio(BytesIO(dummy_audio_bytes), sr=None)
186
+ audio, sr = load_audio(BytesIO(dummy_audio_bytes), sr=None, backend=backend)
187
+ assert sr == ref_sr
188
+ # Decoders disagree only on codec encoder-delay/padding, so torchcodec may
189
+ # emit a few extra trailing samples (e.g. ~192 for Ogg Vorbis). Compare the
190
+ # overlapping region, which must agree to float32 precision.
191
+ n = min(ref_audio.shape[-1], audio.shape[-1])
192
+ assert n > 0
193
+ np.testing.assert_allclose(ref_audio[:n], audio[:n], atol=1e-4)
194
+
195
+
196
+def test_load_audio_unknown_backend_rejected(dummy_audio_bytes):
197
+ """An unknown backend must fail loudly instead of silently degrading."""
198
+ with pytest.raises(ValueError, match="Unknown audio backend"):
199
+ load_audio(BytesIO(dummy_audio_bytes), sr=None, backend="not_a_backend")
200
+
201
+
202
+def test_load_audio_auto_falls_back_without_torchcodec(dummy_audio_bytes):
203
+ """`auto` must fall back to the soundfile → PyAV chain when torchcodec
204
+ is not importable."""
205
+ ref_audio, ref_sr = load_audio_soundfile(BytesIO(dummy_audio_bytes), sr=None)
206
+ with patch.object(audio_module, "load_audio_torchcodec", side_effect=ImportError):
207
+ audio, sr = load_audio(BytesIO(dummy_audio_bytes), sr=None, backend="auto")
208
+ assert sr == ref_sr
209
+ np.testing.assert_array_equal(ref_audio, audio)
210
+
211
+
212
+def test_load_audio_auto_falls_back_without_ffmpeg(dummy_audio_bytes):
213
+ """torchcodec installed but system ffmpeg missing (`AudioDecoder is None`)
214
+ must surface as ImportError so `auto` falls back to soundfile → PyAV."""
215
+ ref_audio, ref_sr = load_audio_soundfile(BytesIO(dummy_audio_bytes), sr=None)
216
+ with patch.object(audio_module, "AudioDecoder", None):
217
+ with pytest.raises(ImportError, match="torchcodec audio backend"):
218
+ load_audio_torchcodec(BytesIO(dummy_audio_bytes), sr=None)
219
+ audio, sr = load_audio(BytesIO(dummy_audio_bytes), sr=None, backend="auto")
220
+ assert sr == ref_sr
221
+ np.testing.assert_array_equal(ref_audio, audio)
222
+
223
+
224
+def test_load_audio_auto_falls_back_when_libtorchcodec_unloadable(
225
+ dummy_audio_bytes,
226
+):
227
+ """torchcodec loads its ffmpeg-backed core lazily at decoder construction;
228
+ a "Could not load libtorchcodec" RuntimeError there (no system ffmpeg)
229
+ must also surface as ImportError so `auto` falls back."""
230
+
231
+ class _UnloadableDecoder:
232
+ def __init__(self, *args, **kwargs):
233
+ raise RuntimeError("Could not load libtorchcodec. Likely causes: ...")
234
+
235
+ ref_audio, ref_sr = load_audio_soundfile(BytesIO(dummy_audio_bytes), sr=None)
236
+ with patch.object(audio_module, "AudioDecoder", _UnloadableDecoder):
237
+ with pytest.raises(ImportError, match="torchcodec audio backend"):
238
+ load_audio_torchcodec(BytesIO(dummy_audio_bytes), sr=None)
239
+ audio, sr = load_audio(BytesIO(dummy_audio_bytes), sr=None, backend="auto")
240
+ assert sr == ref_sr
241
+ np.testing.assert_array_equal(ref_audio, audio)
242
+
243
+
244
+def test_audio_media_io_audio_backend_kwarg(dummy_audio_bytes):
245
+ """`audio_backend` selects the backend; unknown values fail at init."""
246
+ audio, sr = AudioMediaIO(audio_backend="pyav").load_bytes(dummy_audio_bytes)
247
+ assert isinstance(audio, np.ndarray)
248
+ assert sr == 16000
249
+ with pytest.raises(ValueError, match="Unknown audio_backend"):
250
+ AudioMediaIO(audio_backend="not_a_backend")
251
+
252
+
253
+def test_torchcodec_max_duration_rejected(dummy_audio_bytes):
254
+ """The decompression-bomb duration guard must hold for torchcodec too."""
255
+ pytest.importorskip("torchcodec")
256
+ with pytest.raises(ValueError, match="exceeds maximum allowed duration"):
257
+ load_audio_torchcodec(
258
+ BytesIO(dummy_audio_bytes), sr=None, max_duration_s=0.0001
259
+ )
260
+
261
+
262
+def test_torchcodec_max_decode_bytes_rejected(dummy_audio_bytes):
263
+ """The decompression-bomb memory guard must hold for torchcodec too."""
264
+ pytest.importorskip("torchcodec")
265
+ audio, _ = load_audio_torchcodec(BytesIO(dummy_audio_bytes), sr=None)
266
+ with pytest.raises(ValueError, match="VLLM_MAX_AUDIO_DECODE_BYTES"):
267
+ load_audio_torchcodec(
268
+ BytesIO(dummy_audio_bytes),
269
+ sr=None,
270
+ max_decode_bytes=audio.nbytes - 1,
271
+ )
vllm/multimodal/media/audio.py
+313
-65
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import math
4
+from collections.abc import Callable
5
from io import BytesIO
6
from pathlib import Path
7
@@ -35,6 +36,15 @@ try:
36
except ImportError:
37
soundfile = PlaceholderModule("soundfile") # type: ignore[assignment]
38
39
+try:
40
+ from torchcodec.decoders import AudioDecoder
41
+except (ImportError, RuntimeError) as exc:
42
+ # RuntimeError: torchcodec is installed but the system ffmpeg is missing.
43
+ AudioDecoder = None # type: ignore[assignment]
44
+ _torchcodec_import_exc: BaseException | None = exc
45
+else:
46
+ _torchcodec_import_exc = None
47
+
48
49
# Public libsndfile error codes exposed via `soundfile.LibsndfileError.code`,
50
# soundfile being the main audio loading backend. Used to validate if an audio
@@ -48,6 +58,22 @@ except ImportError:
58
# 4 = unsupported encoding (codec not supported by this libsndfile build)
59
_BAD_SF_CODES = {0, 1, 3, 4}
60
61
+# Audio decoding backends selectable via `load_audio(backend=...)` or
62
+# `--media-io-kwargs '{"audio": {"audio_backend": ...}}'`.
63
+# "auto" tries torchcodec, then soundfile, then PyAV.
64
+AUDIO_BACKENDS = ("auto", "soundfile", "pyav", "torchcodec")
65
+
66
+# Raised as ImportError when the torchcodec backend cannot be used, so
67
+# `load_audio(backend="auto")` falls back to the soundfile → PyAV chain.
68
+_TORCHCODEC_UNAVAILABLE_MSG = (
69
+ "torchcodec audio backend is unavailable (requires the torchcodec "
70
+ "package and a system ffmpeg installation)"
71
+)
72
+
73
+# Slack on the torchcodec decode window when enforcing `max_duration_s`, so an
74
+# over-long stream is rejected rather than truncated to the limit.
75
+_DURATION_GUARD_MARGIN_S = 1.0
76
+
77
78
def load_audio_pyav(
79
path: BytesIO | Path | str,
@@ -75,54 +101,56 @@ def load_audio_pyav(
101
``(waveform, sample_rate)`` where *waveform* is a 1-D float32
102
NumPy array and *sample_rate* is the native sample rate in Hz.
103
"""
78
- native_sr = None
104
try:
80
- with av.open(path) as container:
81
- if not container.streams.audio:
82
- raise ValueError("No audio stream found.")
83
- stream = container.streams.audio[0]
84
- stream.thread_type = "AUTO"
85
- native_sr = stream.rate
86
- sr = sr or native_sr
87
-
88
- # Early rejection from container/stream metadata to avoid
89
- # wasting resources on decoding decompression bombs.
90
- if max_duration_s is not None:
91
- metadata_duration_s = None
92
- if stream.duration and stream.time_base:
93
- metadata_duration_s = float(stream.duration * stream.time_base)
94
- elif container.duration:
95
- metadata_duration_s = container.duration / 1_000_000
96
- if (
97
- metadata_duration_s is not None
98
- and metadata_duration_s > max_duration_s
99
- ):
100
- raise ValueError(
101
- f"Audio exceeds maximum allowed duration of "
102
- f"{max_duration_s}s (metadata reports "
103
- f"{metadata_duration_s:.1f}s). Set "
104
- f"VLLM_MAX_AUDIO_DECODE_DURATION_S to "
105
- f"increase this limit."
106
- )
105
+ container = av.open(path)
106
+ except av.error.FFmpegError as e:
107
+ raise ValueError(
108
+ "Invalid or corrupted audio data. Ensure the input is a valid "
109
+ "audio or video file (e.g. a complete WAV, MP3, or MP4)."
110
+ ) from e
111
108
- max_samples = (
109
- int(sr * max_duration_s) if max_duration_s is not None else None
110
- )
111
- total_samples = 0
112
- total_decode_bytes = 0
113
-
114
- chunks: list[npt.NDArray] = []
115
- needs_resampling = not math.isclose(
116
- float(sr),
117
- float(native_sr),
118
- rel_tol=0.0,
119
- abs_tol=1e-6,
120
- )
121
- resampler = (
122
- av.AudioResampler(format="fltp", layout="mono", rate=sr)
123
- if needs_resampling
124
- else None
125
- )
112
+ with container:
113
+ if not container.streams.audio:
114
+ raise ValueError("No audio stream found.")
115
+ stream = container.streams.audio[0]
116
+ stream.thread_type = "AUTO"
117
+ native_sr = stream.rate
118
+ sr = sr or native_sr
119
+
120
+ # Early rejection from container/stream metadata to avoid
121
+ # wasting resources on decoding decompression bombs.
122
+ if max_duration_s is not None:
123
+ metadata_duration_s = None
124
+ if stream.duration and stream.time_base:
125
+ metadata_duration_s = float(stream.duration * stream.time_base)
126
+ elif container.duration:
127
+ metadata_duration_s = container.duration / 1_000_000
128
+ if metadata_duration_s is not None and metadata_duration_s > max_duration_s:
129
+ raise ValueError(
130
+ f"Audio exceeds maximum allowed duration of "
131
+ f"{max_duration_s}s (metadata reports "
132
+ f"{metadata_duration_s:.1f}s). Set "
133
+ f"VLLM_MAX_AUDIO_DECODE_DURATION_S to "
134
+ f"increase this limit."
135
+ )
136
+
137
+ max_samples = int(sr * max_duration_s) if max_duration_s is not None else None
138
+ total_samples = 0
139
+ total_decode_bytes = 0
140
+
141
+ chunks: list[npt.NDArray] = []
142
+ needs_resampling = not math.isclose(
143
+ float(sr),
144
+ float(native_sr),
145
+ rel_tol=0.0,
146
+ abs_tol=1e-6,
147
+ )
148
+ resampler = (
149
+ av.AudioResampler(format="fltp", layout="mono", rate=sr)
150
+ if needs_resampling
151
+ else None
152
+ )
153
+ try:
154
for frame in container.decode(stream):
155
if needs_resampling:
156
assert resampler is not None
@@ -157,16 +185,14 @@ def load_audio_pyav(
185
f"VLLM_MAX_AUDIO_DECODE_BYTES to increase this "
186
f"limit."
187
)
160
- except (ValueError, ImportError):
161
- raise
162
- except Exception as e:
163
- raise ValueError(
164
- "Invalid or corrupted video data when extracting audio. "
165
- "Ensure the input is valid video bytes (e.g. a complete MP4)."
166
- ) from e
188
+ except av.error.FFmpegError as e:
189
+ raise ValueError(
190
+ "Invalid or corrupted audio data. Ensure the input is a valid "
191
+ "audio or video file (e.g. a complete WAV, MP3, or MP4)."
192
+ ) from e
193
194
if not chunks:
169
- raise ValueError("No audio found in the video.")
195
+ raise ValueError("No audio found in the input.")
196
197
audio = np.concatenate(chunks, axis=-1).astype(np.float32)
198
if mono and audio.ndim > 1:
@@ -217,6 +243,163 @@ def load_audio_soundfile(
243
return y, native_sr
244
245
246
+def load_audio_torchcodec(
247
+ path: BytesIO | Path | str,
248
+ *,
249
+ sr: float | None = 22050,
250
+ mono: bool = True,
251
+ max_duration_s: float | None = None,
252
+ max_decode_bytes: int | None = None,
253
+) -> tuple[npt.NDArray, float]:
254
+ """Load an audio file using torchcodec, returning a float32 waveform.
255
+
256
+ Unlike :func:`load_audio_pyav`, which drives FFmpeg through a per-frame
257
+ Python generator, torchcodec decodes inside a single call that releases
258
+ the GIL. Resampling also happens in C++ (``sample_rate=sr``), so this
259
+ backend never falls back to PyAV or libsndfile.
260
+
261
+ Args:
262
+ path: A :class:`~io.BytesIO` buffer, a filesystem
263
+ :class:`~pathlib.Path`, or a string path.
264
+ sr: Output sample rate; ``None`` keeps the native rate. Must be an
265
+ integer when set.
266
+ mono: Average channels down to a single channel.
267
+ max_duration_s: Reject audio longer than this many seconds.
268
+ Prevents decompression-bomb attacks where a small compressed
269
+ file expands into gigabytes of PCM.
270
+ max_decode_bytes: Reject audio whose decoded PCM exceeds this many
271
+ bytes.
272
+
273
+ Returns:
274
+ ``(waveform, sample_rate)`` where *waveform* is a float32 NumPy
275
+ array (1-D when ``mono=True``) and *sample_rate* is the output
276
+ sample rate in Hz.
277
+ """
278
+ if AudioDecoder is None:
279
+ # Unify "torchcodec not installed" and "system ffmpeg missing" into
280
+ # ImportError so `load_audio(backend="auto")` falls back cleanly.
281
+ raise ImportError(_TORCHCODEC_UNAVAILABLE_MSG) from _torchcodec_import_exc
282
+ if sr is not None and sr != int(sr):
283
+ raise ValueError(f"torchcodec requires an integer sample rate, got {sr}")
284
+ sample_rate = int(sr) if sr is not None else None
285
+
286
+ # Opening the source and reading container metadata both touch untrusted
287
+ # input; torchcodec signals corrupt/unsupported data with RuntimeError.
288
+ try:
289
+ decoder = AudioDecoder(path, sample_rate=sample_rate)
290
+ metadata = decoder.metadata
291
+ except RuntimeError as e:
292
+ # torchcodec loads its ffmpeg-backed core lazily at construction, so
293
+ # an ffmpeg-less (or ABI-incompatible) install can fail here rather
294
+ # than at import. Treat that as backend-unavailable, not bad input.
295
+ if "Could not load libtorchcodec" in str(e):
296
+ raise ImportError(_TORCHCODEC_UNAVAILABLE_MSG) from e
297
+ raise ValueError(
298
+ "Invalid or corrupted audio data. Ensure the input is a valid "
299
+ "audio or video file (e.g. a complete WAV, MP3, or MP4)."
300
+ ) from e
301
+
302
+ # Pre-check the memory budget from container metadata so an obviously
303
+ # oversized stream is rejected before allocating the PCM buffer. Metadata
304
+ # is attacker-controlled, so this may only reject, never admit; the
305
+ # post-decode check below is authoritative.
306
+ if max_decode_bytes is not None:
307
+ est_duration_s = metadata.duration_seconds
308
+ # Estimate at the *output* rate: when `sr` is set torchcodec
309
+ # resamples to it during decode.
310
+ est_sample_rate = sr if sr is not None else metadata.sample_rate
311
+ est_num_channels = metadata.num_channels
312
+ if (
313
+ est_duration_s is not None
314
+ and est_sample_rate is not None
315
+ and est_num_channels is not None
316
+ ):
317
+ estimated_bytes = (
318
+ int(est_duration_s * est_sample_rate)
319
+ * est_num_channels
320
+ * np.dtype(np.float32).itemsize
321
+ )
322
+ if estimated_bytes > max_decode_bytes:
323
+ raise ValueError(
324
+ f"Audio would allocate "
325
+ f"{estimated_bytes / MiB_bytes:.0f} MiB of PCM "
326
+ f"(~{est_duration_s:.1f}s at {est_sample_rate}Hz x "
327
+ f"{est_num_channels}ch), exceeding the "
328
+ f"{max_decode_bytes / MiB_bytes:.0f} MiB limit. Set "
329
+ f"VLLM_MAX_AUDIO_DECODE_BYTES to increase this limit."
330
+ )
331
+
332
+ # Duration pre-check, same attacker-controlled → reject-only semantics.
333
+ if max_duration_s is not None:
334
+ duration_s = metadata.duration_seconds
335
+ if duration_s is not None and duration_s > max_duration_s:
336
+ raise ValueError(
337
+ f"Audio exceeds maximum allowed duration of "
338
+ f"{max_duration_s}s (metadata reports "
339
+ f"{duration_s:.1f}s). Set "
340
+ f"VLLM_MAX_AUDIO_DECODE_DURATION_S to "
341
+ f"increase this limit."
342
+ )
343
+
344
+ # Decode the stream. RuntimeError here covers a corrupt stream or a
345
+ # container that lies about its length. Bounding the range (when a
346
+ # duration limit is set) keeps an under-reported duration from expanding
347
+ # into unbounded PCM.
348
+ try:
349
+ if max_duration_s is None:
350
+ samples = decoder.get_all_samples()
351
+ else:
352
+ samples = decoder.get_samples_played_in_range(
353
+ 0.0, max_duration_s + _DURATION_GUARD_MARGIN_S
354
+ )
355
+ except RuntimeError as e:
356
+ raise ValueError(
357
+ "Invalid or corrupted audio data. Ensure the input is a valid "
358
+ "audio or video file (e.g. a complete WAV, MP3, or MP4)."
359
+ ) from e
360
+
361
+ # Authoritative post-decode checks: the metadata estimates above can be
362
+ # bypassed by a container that lies about its duration or size.
363
+ if max_duration_s is not None:
364
+ # Same `int()` semantics as load_audio_pyav's
365
+ # `total_samples > int(sr * max_duration_s)`.
366
+ max_samples = int(samples.sample_rate * max_duration_s)
367
+ if samples.data.shape[-1] > max_samples:
368
+ raise ValueError(
369
+ f"Audio exceeds maximum allowed duration of "
370
+ f"{max_duration_s}s (decoded {samples.data.shape[-1]} "
371
+ f"samples at {samples.sample_rate}Hz). Set "
372
+ f"VLLM_MAX_AUDIO_DECODE_DURATION_S to "
373
+ f"increase this limit."
374
+ )
375
+ if max_decode_bytes is not None and samples.data.nbytes > max_decode_bytes:
376
+ raise ValueError(
377
+ f"Audio decode exceeded "
378
+ f"{max_decode_bytes / MiB_bytes:.0f} MiB memory "
379
+ f"limit ({samples.data.nbytes / MiB_bytes:.0f} MiB decoded). "
380
+ f"Set VLLM_MAX_AUDIO_DECODE_BYTES to increase this limit."
381
+ )
382
+
383
+ out_sr = samples.sample_rate
384
+ audio = samples.data.numpy() # (num_channels, num_samples), float32
385
+ if audio.size == 0:
386
+ raise ValueError("No audio found in the input.")
387
+
388
+ if mono and audio.ndim > 1:
389
+ # Same reduction as load_audio_pyav, so both backends agree at the
390
+ # native sample rate, which is what AudioMediaIO requests.
391
+ audio = np.mean(audio, axis=0)
392
+
393
+ return audio, out_sr
394
+
395
+
396
+_AUDIO_LOADERS: dict[str, Callable[..., tuple[npt.NDArray, float]]] = {
397
+ "torchcodec": load_audio_torchcodec,
398
+ "pyav": load_audio_pyav,
399
+ "soundfile": load_audio_soundfile,
400
+}
401
+
402
+
403
def load_audio(
404
path: BytesIO | Path | str,
405
*,
@@ -224,7 +407,55 @@ def load_audio(
407
mono: bool = True,
408
max_duration_s: float | None = None,
409
max_decode_bytes: int | None = None,
410
+ backend: str | None = None,
411
):
412
+ """Load audio using the selected decoding backend.
413
+
414
+ Args:
415
+ backend: One of ``AUDIO_BACKENDS``. ``None`` (default) selects
416
+ ``"auto"``, which tries torchcodec, then falls back to the
417
+ soundfile → PyAV chain; the other values select a single
418
+ backend with no fallback.
419
+ """
420
+ backend = backend or "auto"
421
+ if backend not in AUDIO_BACKENDS:
422
+ raise ValueError(
423
+ f"Unknown audio backend {backend!r}. "
424
+ f"Available backends: {list(AUDIO_BACKENDS)}"
425
+ )
426
+
427
+ if backend != "auto":
428
+ return _AUDIO_LOADERS[backend](
429
+ path,
430
+ sr=sr,
431
+ mono=mono,
432
+ max_duration_s=max_duration_s,
433
+ max_decode_bytes=max_decode_bytes,
434
+ )
435
+
436
+ # "auto": torchcodec → soundfile → PyAV. Each backend is called by its
437
+ # bare name so tests that monkeypatch it take effect. Only an ImportError
438
+ # (backend missing) or, for soundfile, an unsupported-format error defers
439
+ # to the next; any other error (decode failure, guard ValueError) raises.
440
+ if isinstance(path, BytesIO):
441
+ path.seek(0)
442
+ try:
443
+ return load_audio_torchcodec(
444
+ path,
445
+ sr=sr,
446
+ mono=mono,
447
+ max_duration_s=max_duration_s,
448
+ max_decode_bytes=max_decode_bytes,
449
+ )
450
+ except ImportError as exc:
451
+ # Decode errors don't defer: torchcodec is FFmpeg-based like PyAV, so
452
+ # retrying would just fail the same way.
453
+ logger.warning(
454
+ "torchcodec unavailable (%r); falling back to soundfile/PyAV.", exc
455
+ )
456
+
457
+ if isinstance(path, BytesIO):
458
+ path.seek(0)
459
try:
460
return load_audio_soundfile(
461
path,
@@ -234,18 +465,21 @@ def load_audio(
465
max_decode_bytes=max_decode_bytes,
466
)
467
except ImportError as exc:
237
- # soundfile (or resampy) is not installed — fall through to pyav.
238
- # NOTE: this clause must stay BEFORE ``soundfile.LibsndfileError``
239
- # because when soundfile is a PlaceholderModule, evaluating
240
- # ``soundfile.LibsndfileError`` itself raises ImportError.
468
+ # Reach the LibsndfileError clause below only if exc is not an
469
+ # ImportError — i.e. soundfile is a real module — so evaluating
470
+ # ``soundfile.LibsndfileError`` here is safe even when soundfile is a
471
+ # PlaceholderModule (which would make that attribute access raise).
472
logger.error("Failed to load audio via soundfile: %r", exc)
473
except soundfile.LibsndfileError as exc:
243
- # Only fall back for known format-detection failures.
244
- # Re-raise anything else (e.g. corrupt but recognised format).
474
+ # libsndfile covers fewer containers than FFmpeg, so defer only on a
475
+ # format-detection failure; re-raise anything else (e.g. corrupt but
476
+ # recognised) since PyAV would hit the same corruption.
477
if exc.code not in _BAD_SF_CODES:
478
raise
247
- # soundfile may have advanced the BytesIO seek position before failing;
248
- # reset it so PyAV can read from the beginning.
479
+
480
+ # PyAV is terminal: nothing left to fall back to. Normalize an FFmpeg
481
+ # failure to ValueError; let a missing soundfile/PyAV surface its
482
+ # "install vllm[audio]" ImportError.
483
if isinstance(path, BytesIO):
484
path.seek(0)
485
try:
@@ -257,9 +491,9 @@ def load_audio(
491
max_decode_bytes=max_decode_bytes,
492
)
493
except ImportError:
260
- raise # Let PlaceholderModule's message ("install vllm[audio]") propagate.
261
- except Exception as pyav_exc:
262
- raise ValueError("Invalid or unsupported audio file.") from pyav_exc
494
+ raise
495
+ except av.error.FFmpegError as exc:
496
+ raise ValueError("Invalid or unsupported audio file.") from exc
497
498
499
class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
@@ -271,6 +505,18 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
505
def __init__(self, **kwargs) -> None:
506
super().__init__()
507
508
+ # Allow per-server override of the audio decoding backend, e.g.:
509
+ # --media-io-kwargs '{"audio": {"audio_backend": "torchcodec"}}'
510
+ # Validate eagerly so a misconfigured deployment fails at startup
511
+ # rather than per request.
512
+ backend = kwargs.pop("audio_backend", None)
513
+ if backend is not None and backend not in AUDIO_BACKENDS:
514
+ raise ValueError(
515
+ f"Unknown audio_backend {backend!r}. "
516
+ f"Available backends: {list(AUDIO_BACKENDS)}"
517
+ )
518
+ self.audio_backend = backend
519
+
520
# `kwargs` contains custom arguments from
521
# --media-io-kwargs for this modality, merged with
522
# per-request runtime media_io_kwargs via merge_kwargs().
@@ -298,6 +544,7 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
544
sr=None,
545
max_duration_s=envs.VLLM_MAX_AUDIO_DECODE_DURATION_S,
546
max_decode_bytes=envs.VLLM_MAX_AUDIO_DECODE_BYTES,
547
+ backend=self.audio_backend,
548
)
549
550
def load_base64(
@@ -321,6 +568,7 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
568
sr=None,
569
max_duration_s=envs.VLLM_MAX_AUDIO_DECODE_DURATION_S,
570
max_decode_bytes=envs.VLLM_MAX_AUDIO_DECODE_BYTES,
571
+ backend=self.audio_backend,
572
)
573
574
def encode_base64(