[Bugfix] Fix TorchCodec audio IO correctness (#58364)
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Isotr0py committed
Sep 30, 2026 at 00:34 UTC
31876d8ea166e9fbeb6782466954151fd7e06a91
3 files changed
+67
-36
docs/features/multimodal_inputs.md
+10
-9
@@ -1210,7 +1210,7 @@ vLLM decodes audio bytes into waveforms using a selectable decoding backend:
1210
1211
| Backend | Description |
1212
| --- | --- |
1213
-| `auto` (default) | soundfile, falling back to torchcodec, then PyAV |
1213
+| `auto` (default) | torchcodec, falling back to soundfile, then PyAV |
1214
| `soundfile` | libsndfile only, no fallback |
1215
| `pyav` | PyAV (FFmpeg) only, no fallback |
1216
| `torchcodec` | TorchCodec (PyTorch-native) only, no fallback |
@@ -1226,17 +1226,18 @@ vllm serve mistralai/Voxtral-Mini-3B-2507 \
1226
`pyav` drives FFmpeg through a per-frame Python generator, so under
1227
concurrency the Python/C crossings contend on the GIL. `torchcodec`
1228
decodes each stream in a single call that releases the GIL for its whole
1229
- duration. Select it explicitly for concurrent decoding workloads.
1230
- `auto` prefers soundfile for supported formats to preserve their existing
1231
- decoding behavior, including encoder padding. Audio extracted from formats
1232
- that soundfile cannot read, such as video containers, can use torchcodec
1233
- through the fallback chain.
1229
+ duration, which is why `auto` prefers it when many requests decode audio
1230
+ concurrently, such as when audio tracks are extracted from video.
1231
+ vLLM normalizes codec padding on this path (e.g. it trims trailing Vorbis
1232
+ padding that older system FFmpeg versions (< 5.0) fail to trim), so the
1233
+ decoded waveform matches the soundfile reference length.
1234
1235
!!! note
1236
`torchcodec` ships as a requirement on CUDA, CPU and XPU builds. On other
1237
- platforms (e.g. ROCm, TPU), install it manually to enable that backend.
1238
- torchcodec also links against a system FFmpeg installation. If the package
1239
- or FFmpeg is unavailable, `auto` uses the soundfile → PyAV chain.
1237
+ platforms (e.g. ROCm, TPU) `auto` falls back to the soundfile → PyAV
1238
+ chain unless you install it manually. torchcodec also links against a
1239
+ system FFmpeg installation; if FFmpeg is missing, `auto` falls back the
1240
+ same way.
1241
1242
### Embedding Inputs
1243
tests/multimodal/media/test_audio.py
+20
-3
@@ -185,9 +185,15 @@ def test_load_audio_backend_matches_default(backend, dummy_audio_bytes):
185
ref_audio, ref_sr = load_audio(BytesIO(dummy_audio_bytes), sr=None)
186
audio, sr = load_audio(BytesIO(dummy_audio_bytes), sr=None, backend=backend)
187
assert sr == ref_sr
188
- # Decoders disagree only on codec encoder-delay/padding, so torchcodec may
189
- # emit a few extra trailing samples (e.g. ~192 for Ogg Vorbis). Compare the
190
- # overlapping region, which must agree to float32 precision.
188
+ if backend == "torchcodec":
189
+ # Vorbis encoder padding is trimmed, so torchcodec must match the
190
+ # soundfile reference exactly, including the waveform length.
191
+ assert audio.shape == ref_audio.shape
192
+ np.testing.assert_allclose(ref_audio, audio, atol=1e-4)
193
+ return
194
+ # Decoders disagree only on codec encoder-delay/padding, so PyAV may
195
+ # emit a few extra trailing samples (e.g. ~192 for Ogg Vorbis). Compare
196
+ # the overlapping region, which must agree to float32 precision.
197
n = min(ref_audio.shape[-1], audio.shape[-1])
198
assert n > 0
199
np.testing.assert_allclose(ref_audio[:n], audio[:n], atol=1e-4)
@@ -201,6 +207,17 @@ def test_load_audio_default_preserves_vorbis_length(dummy_audio_bytes):
207
assert audio.shape == expected.shape
208
209
210
+def test_load_audio_torchcodec_preserves_vorbis_length(dummy_audio_bytes):
211
+ """Torchcodec must trim Vorbis trailing padding even on FFmpeg < 5.0,
212
+ matching the soundfile reference exactly."""
213
+ pytest.importorskip("torchcodec")
214
+ expected, expected_sr = load_audio_soundfile(BytesIO(dummy_audio_bytes), sr=None)
215
+ audio, sr = load_audio_torchcodec(BytesIO(dummy_audio_bytes), sr=None)
216
+ assert sr == expected_sr
217
+ assert audio.shape == expected.shape
218
+ np.testing.assert_allclose(audio, expected, atol=1e-4)
219
+
220
+
221
def test_load_audio_unknown_backend_rejected(dummy_audio_bytes):
222
"""An unknown backend must fail loudly instead of silently degrading."""
223
with pytest.raises(ValueError, match="Unknown audio backend"):
vllm/multimodal/media/audio.py
+37
-24
@@ -15,6 +15,7 @@ from vllm.exceptions import VLLMValidationError
15
from vllm.logger import init_logger
16
from vllm.multimodal.audio import resample_audio_pyav
17
from vllm.utils.import_utils import PlaceholderModule
18
+from vllm.utils.math_utils import cdiv
19
from vllm.utils.mem_constants import MiB_bytes
20
from vllm.utils.serial_utils import tensor2base64
21
from vllm.utils.sparse_utils import (
@@ -60,11 +61,11 @@ _BAD_SF_CODES = {0, 1, 3, 4}
61
62
# Audio decoding backends selectable via `load_audio(backend=...)` or
63
# `--media-io-kwargs '{"audio": {"audio_backend": ...}}'`.
63
-# "auto" tries soundfile, then torchcodec, then PyAV.
64
+# "auto" tries torchcodec, then soundfile, then PyAV.
65
AUDIO_BACKENDS = ("auto", "soundfile", "pyav", "torchcodec")
66
67
# Raised as ImportError when the torchcodec backend cannot be used, so
67
-# `load_audio(backend="auto")` can continue to PyAV.
68
+# `load_audio(backend="auto")` falls back to the soundfile → PyAV chain.
69
_TORCHCODEC_UNAVAILABLE_MSG = (
70
"torchcodec audio backend is unavailable (requires the torchcodec "
71
"package and a system ffmpeg installation)"
@@ -74,6 +75,9 @@ _TORCHCODEC_UNAVAILABLE_MSG = (
75
# over-long stream is rejected rather than truncated to the limit.
76
_DURATION_GUARD_MARGIN_S = 1.0
77
78
+# Largest Vorbis block size in native-rate samples
79
+_VORBIS_MAX_BLOCK_SAMPLES = 8192
80
+
81
82
def load_audio_pyav(
83
path: BytesIO | Path | str,
@@ -364,6 +368,16 @@ def load_audio_torchcodec(
368
"audio or video file (e.g. a complete WAV, MP3, or MP4)."
369
) from e
370
371
+ # FFmpeg < 5.0 misses Ogg end_trimming, leaving up to one Vorbis block of
372
+ # padding at the tail; we need to trim for valid audio length.
373
+ if metadata.codec == "vorbis" and metadata.duration_seconds is not None:
374
+ expected_samples = round(metadata.duration_seconds * samples.sample_rate)
375
+ padding = samples.data.shape[-1] - expected_samples
376
+ native_sr = metadata.sample_rate or samples.sample_rate
377
+ max_padding = cdiv(_VORBIS_MAX_BLOCK_SAMPLES * samples.sample_rate, native_sr)
378
+ if 0 < padding <= max_padding:
379
+ samples.data = samples.data[..., :expected_samples]
380
+
381
# Authoritative post-decode checks: the metadata estimates above can be
382
# bypassed by a container that lies about its duration or size.
383
if max_duration_s is not None:
@@ -425,8 +439,8 @@ def load_audio(
439
max_decode_bytes: Reject audio that would decode to more than this
440
many bytes.
441
backend: One of ``AUDIO_BACKENDS``. ``None`` (default) selects
428
- ``"auto"``, which tries soundfile, then torchcodec, then PyAV;
429
- the other values select a single
442
+ ``"auto"``, which tries torchcodec, then falls back to the
443
+ soundfile → PyAV chain; the other values select a single
444
backend with no fallback.
445
446
"""
@@ -446,11 +460,25 @@ def load_audio(
460
max_decode_bytes=max_decode_bytes,
461
)
462
449
- # Keep soundfile first to preserve decoding and padding of supported formats.
450
- # "auto": soundfile → torchcodec → PyAV. Each backend is called by its
451
- # bare name so tests that monkeypatch it take effect. Only an ImportError
452
- # (backend missing) or, for soundfile, an unsupported-format error defers
453
- # to the next; any other error (decode failure, guard ValueError) raises.
463
+ # "auto": torchcodec → soundfile → PyAV, each called by its bare name so
464
+ # monkeypatched tests take effect. Only ImportError (or, for soundfile, an
465
+ # unsupported-format error) defers to the next backend; anything else raises.
466
+ if isinstance(path, BytesIO):
467
+ path.seek(0)
468
+ try:
469
+ return load_audio_torchcodec(
470
+ path,
471
+ sr=sr,
472
+ mono=mono,
473
+ max_duration_s=max_duration_s,
474
+ max_decode_bytes=max_decode_bytes,
475
+ )
476
+ except ImportError as exc:
477
+ # Decode errors don't defer: FFmpeg-based PyAV would fail the same way.
478
+ logger.warning(
479
+ "torchcodec unavailable (%r); falling back to soundfile/PyAV.", exc
480
+ )
481
+
482
if isinstance(path, BytesIO):
483
path.seek(0)
484
try:
@@ -474,21 +502,6 @@ def load_audio(
502
if exc.code not in _BAD_SF_CODES:
503
raise
504
477
- if isinstance(path, BytesIO):
478
- path.seek(0)
479
- try:
480
- return load_audio_torchcodec(
481
- path,
482
- sr=sr,
483
- mono=mono,
484
- max_duration_s=max_duration_s,
485
- max_decode_bytes=max_decode_bytes,
486
- )
487
- except ImportError as exc:
488
- # Decode errors don't defer: torchcodec is FFmpeg-based like PyAV, so
489
- # retrying would just fail the same way.
490
- logger.warning("torchcodec unavailable (%r); falling back to PyAV.", exc)
491
-
505
# PyAV is terminal: nothing left to fall back to. Normalize an FFmpeg
506
# failure to ValueError; let a missing soundfile/PyAV surface its
507
# "install vllm[audio]" ImportError.