[Bugfix][Audio] Restore soundfile-first automatic decoding (#55642)
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Andreas Karatzas committed
Sep 7, 2026 at 01:21 UTC
f7f060d253fb67c9b83f45a22a1eeeba4d72a2be
3 files changed
+38
-29
docs/features/multimodal_inputs.md
+9
-7
@@ -1210,7 +1210,7 @@ vLLM decodes audio bytes into waveforms using a selectable decoding backend:
1210
1211
| Backend | Description |
1212
| --- | --- |
1213
-| `auto` (default) | torchcodec, falling back to soundfile, then PyAV |
1213
+| `auto` (default) | soundfile, falling back to torchcodec, then PyAV |
1214
| `soundfile` | libsndfile only, no fallback |
1215
| `pyav` | PyAV (FFmpeg) only, no fallback |
1216
| `torchcodec` | TorchCodec (PyTorch-native) only, no fallback |
@@ -1226,15 +1226,17 @@ vllm serve mistralai/Voxtral-Mini-3B-2507 \
1226
`pyav` drives FFmpeg through a per-frame Python generator, so under
1227
concurrency the Python/C crossings contend on the GIL. `torchcodec`
1228
decodes each stream in a single call that releases the GIL for its whole
1229
- duration, which is why `auto` prefers it when many requests decode audio
1230
- concurrently, such as when audio tracks are extracted from video.
1229
+ duration. Select it explicitly for concurrent decoding workloads.
1230
+ `auto` prefers soundfile for supported formats to preserve their existing
1231
+ decoding behavior, including encoder padding. Audio extracted from formats
1232
+ that soundfile cannot read, such as video containers, can use torchcodec
1233
+ through the fallback chain.
1234
1235
!!! note
1236
`torchcodec` ships as a requirement on CUDA, CPU and XPU builds. On other
1234
- platforms (e.g. ROCm, TPU) `auto` falls back to the soundfile → PyAV
1235
- chain unless you install it manually. torchcodec also links against a
1236
- system FFmpeg installation; if FFmpeg is missing, `auto` falls back the
1237
- same way.
1237
+ platforms (e.g. ROCm, TPU), install it manually to enable that backend.
1238
+ torchcodec also links against a system FFmpeg installation. If the package
1239
+ or FFmpeg is unavailable, `auto` uses the soundfile → PyAV chain.
1240
1241
### Embedding Inputs
1242
tests/multimodal/media/test_audio.py
+8
@@ -193,6 +193,14 @@ def test_load_audio_backend_matches_default(backend, dummy_audio_bytes):
193
np.testing.assert_allclose(ref_audio[:n], audio[:n], atol=1e-4)
194
195
196
+def test_load_audio_default_preserves_vorbis_length(dummy_audio_bytes):
197
+ """Default decoding must not append Vorbis padding to the speech waveform."""
198
+ expected, expected_sr = load_audio_soundfile(BytesIO(dummy_audio_bytes), sr=None)
199
+ audio, sr = load_audio(BytesIO(dummy_audio_bytes), sr=None)
200
+ assert sr == expected_sr
201
+ assert audio.shape == expected.shape
202
+
203
+
204
def test_load_audio_unknown_backend_rejected(dummy_audio_bytes):
205
"""An unknown backend must fail loudly instead of silently degrading."""
206
with pytest.raises(ValueError, match="Unknown audio backend"):
vllm/multimodal/media/audio.py
+21
-22
@@ -60,11 +60,11 @@ _BAD_SF_CODES = {0, 1, 3, 4}
60
61
# Audio decoding backends selectable via `load_audio(backend=...)` or
62
# `--media-io-kwargs '{"audio": {"audio_backend": ...}}'`.
63
-# "auto" tries torchcodec, then soundfile, then PyAV.
63
+# "auto" tries soundfile, then torchcodec, then PyAV.
64
AUDIO_BACKENDS = ("auto", "soundfile", "pyav", "torchcodec")
65
66
# Raised as ImportError when the torchcodec backend cannot be used, so
67
-# `load_audio(backend="auto")` falls back to the soundfile → PyAV chain.
67
+# `load_audio(backend="auto")` can continue to PyAV.
68
_TORCHCODEC_UNAVAILABLE_MSG = (
69
"torchcodec audio backend is unavailable (requires the torchcodec "
70
"package and a system ffmpeg installation)"
@@ -413,8 +413,8 @@ def load_audio(
413
414
Args:
415
backend: One of ``AUDIO_BACKENDS``. ``None`` (default) selects
416
- ``"auto"``, which tries torchcodec, then falls back to the
417
- soundfile → PyAV chain; the other values select a single
416
+ ``"auto"``, which tries soundfile, then torchcodec, then PyAV;
417
+ the other values select a single
418
backend with no fallback.
419
"""
420
backend = backend or "auto"
@@ -433,27 +433,11 @@ def load_audio(
433
max_decode_bytes=max_decode_bytes,
434
)
435
436
- # "auto": torchcodec → soundfile → PyAV. Each backend is called by its
436
+ # Keep soundfile first to preserve decoding and padding of supported formats.
437
+ # "auto": soundfile → torchcodec → PyAV. Each backend is called by its
438
# bare name so tests that monkeypatch it take effect. Only an ImportError
439
# (backend missing) or, for soundfile, an unsupported-format error defers
440
# to the next; any other error (decode failure, guard ValueError) raises.
440
- if isinstance(path, BytesIO):
441
- path.seek(0)
442
- try:
443
- return load_audio_torchcodec(
444
- path,
445
- sr=sr,
446
- mono=mono,
447
- max_duration_s=max_duration_s,
448
- max_decode_bytes=max_decode_bytes,
449
- )
450
- except ImportError as exc:
451
- # Decode errors don't defer: torchcodec is FFmpeg-based like PyAV, so
452
- # retrying would just fail the same way.
453
- logger.warning(
454
- "torchcodec unavailable (%r); falling back to soundfile/PyAV.", exc
455
- )
456
-
441
if isinstance(path, BytesIO):
442
path.seek(0)
443
try:
@@ -477,6 +461,21 @@ def load_audio(
461
if exc.code not in _BAD_SF_CODES:
462
raise
463
464
+ if isinstance(path, BytesIO):
465
+ path.seek(0)
466
+ try:
467
+ return load_audio_torchcodec(
468
+ path,
469
+ sr=sr,
470
+ mono=mono,
471
+ max_duration_s=max_duration_s,
472
+ max_decode_bytes=max_decode_bytes,
473
+ )
474
+ except ImportError as exc:
475
+ # Decode errors don't defer: torchcodec is FFmpeg-based like PyAV, so
476
+ # retrying would just fail the same way.
477
+ logger.warning("torchcodec unavailable (%r); falling back to PyAV.", exc)
478
+
479
# PyAV is terminal: nothing left to fall back to. Normalize an FFmpeg
480
# failure to ValueError; let a missing soundfile/PyAV surface its
481
# "install vllm[audio]" ImportError.