[multimodal][feat] add torchaudio backend to AudioResampler (#52598)
Signed-off-by: JaredforReal <w13431838023@gmail.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Jared Wen committed
Sep 10, 2026 at 14:42 UTC
912d2e79fe8571ded2fcc6ec669d26429f814d7b
3 files changed
+159
-6
tests/multimodal/test_audio.py
+81
-3
@@ -14,9 +14,11 @@ from vllm.multimodal.audio import (
14
AudioResampler,
15
AudioSpec,
16
ChannelReduction,
17
+ _get_torchaudio_resampler,
18
normalize_audio,
19
resample_audio_pyav,
20
resample_audio_scipy,
21
+ resample_audio_torchaudio,
22
split_audio,
23
)
24
@@ -76,6 +78,46 @@ def test_resample_audio_scipy_resamples_last_axis_for_multichannel():
78
assert np.isfinite(out).all()
79
80
81
+def test_resample_audio_torchaudio(dummy_audio):
82
+ out_down = resample_audio_torchaudio(dummy_audio, orig_sr=4, target_sr=2)
83
+ out_up = resample_audio_torchaudio(dummy_audio, orig_sr=2, target_sr=4)
84
+ out_same = resample_audio_torchaudio(dummy_audio, orig_sr=4, target_sr=4)
85
+
86
+ assert len(out_down) == 3
87
+ assert len(out_up) == 10
88
+ assert np.all(out_same == dummy_audio)
89
+
90
+
91
+def test_resample_audio_torchaudio_non_divisible_sample_rates():
92
+ audio = np.arange(441, dtype=float)
93
+ out = resample_audio_torchaudio(audio, orig_sr=44100, target_sr=16000)
94
+
95
+ expected_len = math.ceil(len(audio) * 16000 / 44100)
96
+ assert len(out) == expected_len
97
+
98
+ assert isinstance(out, np.ndarray)
99
+ assert np.isfinite(out).all()
100
+
101
+
102
+def test_resample_audio_torchaudio_resamples_last_axis_for_multichannel():
103
+ audio = np.arange(2 * 441, dtype=float).reshape(2, 441)
104
+ out = resample_audio_torchaudio(audio, orig_sr=44100, target_sr=16000)
105
+
106
+ expected_len = math.ceil(audio.shape[-1] * 16000 / 44100)
107
+ assert out.shape == (2, expected_len)
108
+ assert np.isfinite(out).all()
109
+
110
+
111
+def test_resample_audio_torchaudio_short_input_needs_no_padding():
112
+ # Unlike the PyAV resampler, torchaudio produces output for inputs
113
+ # shorter than libswresample's minimum frame size.
114
+ audio = np.arange(8, dtype=float)
115
+ out = resample_audio_torchaudio(audio, orig_sr=44100, target_sr=16000)
116
+
117
+ assert len(out) == math.ceil(len(audio) * 16000 / 44100)
118
+ assert np.isfinite(out).all()
119
+
120
+
121
def test_audio_resampler_pyav_calls_resample(dummy_audio):
122
resampler = AudioResampler(target_sr=22050, method="pyav")
123
with patch("vllm.multimodal.audio.resample_audio_pyav") as mock_resample:
@@ -98,10 +140,46 @@ def test_audio_resampler_scipy_calls_resample(dummy_audio):
140
assert np.all(out == dummy_audio)
141
142
143
+def test_audio_resampler_torchaudio(dummy_audio):
144
+ resampler = AudioResampler(target_sr=22050, method="torchaudio")
145
+ out_down = resampler.resample(dummy_audio, orig_sr=44100)
146
+
147
+ expected_len = math.ceil(len(dummy_audio) * 22050 / 44100)
148
+ assert len(out_down) == expected_len
149
+ assert np.isfinite(out_down).all()
150
+
151
+
152
+def test_resample_audio_torchaudio_caches_kernel():
153
+ # The sinc kernel is rebuilt per (orig_sr, target_sr) pair; repeated
154
+ # requests at the same pair must reuse the cached one.
155
+ _get_torchaudio_resampler.cache_clear()
156
+ audio = np.arange(441, dtype=float)
157
+
158
+ resample_audio_torchaudio(audio, orig_sr=44100, target_sr=16000)
159
+ resample_audio_torchaudio(audio, orig_sr=44100, target_sr=16000)
160
+ resample_audio_torchaudio(audio, orig_sr=48000, target_sr=16000)
161
+
162
+ info = _get_torchaudio_resampler.cache_info()
163
+ assert info.misses == 2
164
+ assert info.hits == 1
165
+
166
+
167
+def test_audio_resampler_default_is_torchaudio():
168
+ # Deliberate behavior change: torchaudio is the default resampling backend
169
+ # for both AudioResampler and the data parser that wraps it.
170
+ assert AudioResampler(target_sr=16000).method == "torchaudio"
171
+
172
+ from vllm.multimodal.parse import MultiModalDataParser
173
+
174
+ parser = MultiModalDataParser(target_sr=16000)
175
+ assert parser.audio_resampler.method == "torchaudio"
176
+
177
+
178
def test_audio_resampler_invalid_method(dummy_audio):
102
- resampler = AudioResampler(target_sr=22050, method="invalid")
103
- with pytest.raises(ValueError):
104
- resampler.resample(dummy_audio, orig_sr=44100)
179
+ # Validated eagerly so a bad method fails at construction, not on the
180
+ # first audio request.
181
+ with pytest.raises(ValueError, match="Invalid resampling method"):
182
+ AudioResampler(target_sr=22050, method="invalid")
183
184
185
def test_audio_resampler_no_target_sr(dummy_audio):
vllm/multimodal/audio.py
+72
-2
@@ -3,6 +3,7 @@
3
import math
4
from dataclasses import dataclass
5
from enum import Enum
6
+from functools import lru_cache
7
from typing import Literal
8
9
import numpy as np
@@ -10,6 +11,7 @@ import numpy.typing as npt
11
import torch
12
13
from vllm.utils.import_utils import PlaceholderModule
14
+from vllm.utils.torch_utils import set_default_torch_num_threads
15
16
try:
17
import av as av
@@ -26,6 +28,11 @@ try:
28
except ImportError:
29
soxr = PlaceholderModule("soxr") # type: ignore[assignment]
30
31
+try:
32
+ import torchaudio
33
+except ImportError:
34
+ torchaudio = PlaceholderModule("torchaudio") # type: ignore[assignment]
35
+
36
37
# ============================================================
38
# Aligned with `librosa.get_duration` function
@@ -274,14 +281,73 @@ def resample_audio_soxr(
281
return soxr.resample(audio, orig_sr_int, target_sr_int)
282
283
284
+@lru_cache(maxsize=32)
285
+def _get_torchaudio_resampler(
286
+ orig_sr: int, target_sr: int
287
+) -> "torchaudio.transforms.Resample":
288
+ # `torchaudio.transforms.Resample` precomputes its kernel for a fixed
289
+ # (orig_sr, target_sr) pair; cache instances so repeated requests at a
290
+ # common input rate skip the kernel rebuild.
291
+ return torchaudio.transforms.Resample(orig_sr, target_sr)
292
+
293
+
294
+def resample_audio_torchaudio(
295
+ audio: npt.NDArray[np.floating],
296
+ *,
297
+ orig_sr: float,
298
+ target_sr: float,
299
+) -> npt.NDArray[np.floating]:
300
+ """Resample audio using torchaudio's bandlimited sinc interpolation.
301
+
302
+ Unlike the PyAV resampler, this handles any input length without padding
303
+ and applies the kernel over the trailing axis, so 2D ``(channels,
304
+ samples)`` input needs no per-channel loop.
305
+
306
+ Args:
307
+ audio: Input audio. Can be:
308
+ - 1D array ``(samples,)``: mono audio
309
+ - 2D array ``(channels, samples)``: stereo audio
310
+ orig_sr: Original sample rate in Hz.
311
+ target_sr: Target sample rate in Hz.
312
+
313
+ Returns:
314
+ Resampled audio with the same shape as the input (1D → 1D, 2D → 2D).
315
+ """
316
+ orig_sr_int = int(round(orig_sr))
317
+ target_sr_int = int(round(target_sr))
318
+
319
+ if orig_sr_int == target_sr_int:
320
+ return audio
321
+
322
+ # The kernel is float32; cast the input to match (same coercion as the
323
+ # PyAV path).
324
+ tensor = torch.as_tensor(audio, dtype=torch.float32)
325
+ # Resampling runs in the API/server parent process. Keep it from
326
+ # touching OpenMP or oneDNN thread state: both poison subsequently
327
+ # forked engine-core processes, which then segfault on their first
328
+ # parallel CPU op.
329
+ with set_default_torch_num_threads(1), torch.backends.mkldnn.flags(enabled=False):
330
+ resampler = _get_torchaudio_resampler(orig_sr_int, target_sr_int)
331
+ return resampler(tensor).numpy()
332
+
333
+
334
class AudioResampler:
335
"""Resample audio data to a target sample rate."""
336
337
+ _METHODS = ("pyav", "scipy", "soxr", "torchaudio")
338
+
339
def __init__(
340
self,
341
target_sr: float | None = None,
283
- method: Literal["pyav", "scipy", "soxr"] = "pyav",
342
+ method: Literal["pyav", "scipy", "soxr", "torchaudio"] = "torchaudio",
343
):
344
+ # Eager validation so a bad method fails at construction rather than
345
+ # on the first audio request.
346
+ if method not in self._METHODS:
347
+ raise ValueError(
348
+ f"Invalid resampling method: {method!r}. "
349
+ f"Supported methods: {list(self._METHODS)}."
350
+ )
351
self.target_sr = target_sr
352
self.method = method
353
@@ -310,10 +376,14 @@ class AudioResampler:
376
)
377
elif self.method == "soxr":
378
return resample_audio_soxr(audio, orig_sr=orig_sr, target_sr=self.target_sr)
379
+ elif self.method == "torchaudio":
380
+ return resample_audio_torchaudio(
381
+ audio, orig_sr=orig_sr, target_sr=self.target_sr
382
+ )
383
else:
384
raise ValueError(
385
f"Invalid resampling method: {self.method}. "
316
- "Supported methods are 'pyav', 'scipy', and 'soxr'."
386
+ f"Supported methods are {list(self._METHODS)}."
387
)
388
389
vllm/multimodal/parse.py
+6
-1
@@ -547,6 +547,9 @@ class MultiModalDataParser:
547
Args:
548
target_sr (float, optional): Enables automatic resampling of audio
549
items to the model's expected sampling rate.
550
+ audio_resample_method (str): Backend used for the resampling above.
551
+ Defaults to torchaudio; models with specific needs may override
552
+ (e.g. phi4mm uses scipy).
553
target_channels (int, optional): Target number of audio channels.
554
If provided, normalizes audio to this many channels (e.g., 1 for mono).
555
If None, audio channels are passed through unchanged.
@@ -602,7 +605,9 @@ class MultiModalDataParser:
605
*,
606
target_sr: float | None = None,
607
target_channels: int | None = None,
605
- audio_resample_method: Literal["pyav", "scipy", "soxr"] = "pyav",
608
+ audio_resample_method: Literal["pyav", "scipy", "soxr", "torchaudio"] = (
609
+ "torchaudio"
610
+ ),
611
video_needs_metadata: bool = False,
612
expected_hidden_size: int | None = None,
613
allow_missing_mm_embeddings: bool = False,