A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 07:00 UTC
[Frontend] Remove librosa from audio dependency (#37058)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Mar 21, 2026 at 11:36 UTC
c7f98b4d0a63b32ed939e2b6dfaa8a626e9b46c4
18 files changed
+247
-188
requirements/test.in
+1
@@ -21,6 +21,7 @@ vocos # required for minicpmo_26 test
21
peft>=0.15.0 # required for phi-4-mm test
22
pqdm
23
ray[cgraph,default]>=2.48.0 # Ray Compiled Graph, required by pipeline parallelism tests
24
+resampy # required for audio tests
25
sentence-transformers>=5.2.0 # required for embedding tests
26
soundfile # required for audio tests
27
jiwer # required for audio tests
requirements/test.txt
+4
@@ -544,6 +544,7 @@ numba==0.61.2
544
# via
545
# -r requirements/test.in
546
# librosa
547
+ # resampy
548
numpy==2.2.6
549
# via
550
# -r requirements/test.in
@@ -584,6 +585,7 @@ numpy==2.2.6
585
# pyogrio
586
# pywavelets
587
# rasterio
588
+ # resampy
589
# rioxarray
590
# rouge-score
591
# runai-model-streamer
@@ -995,6 +997,8 @@ requests==2.32.3
997
# tiktoken
998
# transformers
999
# wandb
1000
+resampy==0.4.3
1001
+ # via -r requirements/test.in
1002
responses==0.25.3
1003
# via genai-perf
1004
rfc3339-validator==0.1.4
setup.py
+2
-2
@@ -987,11 +987,11 @@ setup(
987
"instanttensor": ["instanttensor >= 0.1.5"],
988
"runai": ["runai-model-streamer[s3,gcs,azure] >= 0.15.7"],
989
"audio": [
990
- "librosa",
990
+ "av",
991
+ "resampy",
992
"scipy",
993
"soundfile",
994
"mistral_common[audio]",
994
- "av",
995
], # Required for audio processing
996
"video": [], # Kept for backwards compatibility
997
"flashinfer": [], # Kept for backwards compatibility
tests/entrypoints/openai/speech_to_text/test_transcription_validation.py
+1
-1
@@ -152,5 +152,5 @@ async def test_basic_audio_foscolo(foscolo, rocm_aiter_fa_attention, model_name)
152
model_name,
153
foscolo,
154
language="it",
155
- expected_text="ove il mio corpo fanciulletto giacque",
155
+ expected_text="ove il mio corpo fanciulletto",
156
)
tests/entrypoints/openai/test_run_batch.py
+1
-1
@@ -275,7 +275,7 @@ INPUT_REASONING_BATCH = "\n".join(
275
]
276
)
277
278
-MINIMAL_WAV_BASE64 = "UklGRiQAAABXQVZFZm10IBAAAAABAAEAQB8AAEAfAAABAAgAZGF0YQAAAAA="
278
+MINIMAL_WAV_BASE64 = "UklGRigAAABXQVZFZm10IBAAAAABAAEAgD4AAAB9AAACABAAZGF0YQQAAAAAAP9/"
279
INPUT_TRANSCRIPTION_BATCH = (
280
json.dumps(
281
{
tests/models/multimodal/generation/vlm_utils/builders.py
+1
-4
@@ -323,10 +323,7 @@ def build_audio_inputs_from_test_info(
323
test_info.audio_idx_to_prompt,
324
test_info.prompt_formatter,
325
)
326
- resampler = AudioResampler(
327
- target_sr=16000,
328
- method="librosa",
329
- )
326
+ resampler = AudioResampler(target_sr=16000)
327
audios = [asset.audio_and_sample_rate for asset in audio_assets]
328
resampled_audios = [
329
(
tests/multimodal/media/test_audio.py
+16
-22
@@ -10,6 +10,8 @@ import pytest
10
11
from vllm.multimodal.media import AudioMediaIO
12
13
+from ...conftest import AudioTestAssets
14
+
15
pytestmark = pytest.mark.cpu_test
16
17
ASSETS_DIR = Path(__file__).parent.parent / "assets"
@@ -22,40 +24,32 @@ def dummy_audio():
24
25
26
@pytest.fixture
25
-def dummy_audio_bytes():
26
- return b"FAKEAUDIOBYTES"
27
+def dummy_audio_bytes(audio_assets: AudioTestAssets):
28
+ with open(audio_assets[0].get_local_path(), "rb") as f:
29
+ return f.read()
30
31
32
def test_audio_media_io_load_bytes(dummy_audio_bytes):
33
audio_io = AudioMediaIO()
31
- with patch("librosa.load") as mock_load:
32
- mock_load.return_value = (np.array([0.1, 0.2]), 16000)
33
- out = audio_io.load_bytes(dummy_audio_bytes)
34
- mock_load.assert_called_once()
35
- assert isinstance(out[0], np.ndarray)
36
- assert out[1] == 16000
34
+ out = audio_io.load_bytes(dummy_audio_bytes)
35
+ assert isinstance(out[0], np.ndarray)
36
+ assert out[1] == 16000
37
38
39
def test_audio_media_io_load_base64(dummy_audio_bytes):
40
audio_io = AudioMediaIO()
41
encoded = base64.b64encode(dummy_audio_bytes).decode("utf-8")
42
- with patch.object(AudioMediaIO, "load_bytes") as mock_load_bytes:
43
- mock_load_bytes.return_value = (np.array([0.1, 0.2]), 16000)
44
- out = audio_io.load_base64("audio/wav", encoded)
45
- mock_load_bytes.assert_called_once()
46
- assert isinstance(out[0], np.ndarray)
47
- assert out[1] == 16000
42
+ out = audio_io.load_base64("audio/wav", encoded)
43
+ assert isinstance(out[0], np.ndarray)
44
+ assert out[1] == 16000
45
46
50
-def test_audio_media_io_load_file():
47
+def test_audio_media_io_load_file(audio_assets: AudioTestAssets):
48
audio_io = AudioMediaIO()
52
- path = Path("/fake/path.wav")
53
- with patch("librosa.load") as mock_load:
54
- mock_load.return_value = (np.array([0.1, 0.2]), 16000)
55
- out = audio_io.load_file(path)
56
- mock_load.assert_called_once_with(path, sr=None)
57
- assert isinstance(out[0], np.ndarray)
58
- assert out[1] == 16000
49
+ path = audio_assets[0].get_local_path()
50
+ out = audio_io.load_file(path)
51
+ assert isinstance(out[0], np.ndarray)
52
+ assert out[1] == 16000
53
54
55
def test_audio_media_io_encode_base64(dummy_audio):
tests/multimodal/test_audio.py
+19
-19
@@ -14,7 +14,7 @@ from vllm.multimodal.audio import (
14
AudioSpec,
15
ChannelReduction,
16
normalize_audio,
17
- resample_audio_librosa,
17
+ resample_audio_pyav,
18
resample_audio_scipy,
19
split_audio,
20
)
@@ -25,14 +25,14 @@ def dummy_audio():
25
return np.array([0.0, 0.1, 0.2, 0.3, 0.4], dtype=float)
26
27
28
-def test_resample_audio_librosa(dummy_audio):
29
- with patch("vllm.multimodal.audio.librosa.resample") as mock_resample:
30
- mock_resample.return_value = dummy_audio * 2
31
- out = resample_audio_librosa(dummy_audio, orig_sr=44100, target_sr=22050)
32
- mock_resample.assert_called_once_with(
33
- dummy_audio, orig_sr=44100, target_sr=22050
34
- )
35
- assert np.all(out == dummy_audio * 2)
28
+def test_resample_audio_pyav(dummy_audio):
29
+ out_down = resample_audio_pyav(dummy_audio, orig_sr=4, target_sr=2)
30
+ out_up = resample_audio_pyav(dummy_audio, orig_sr=2, target_sr=4)
31
+ out_same = resample_audio_pyav(dummy_audio, orig_sr=4, target_sr=4)
32
+
33
+ assert len(out_down) == 3
34
+ assert len(out_up) == 10
35
+ assert np.all(out_same == dummy_audio)
36
37
38
def test_resample_audio_scipy(dummy_audio):
@@ -56,9 +56,9 @@ def test_resample_audio_scipy_non_integer_ratio(dummy_audio):
56
assert np.isfinite(out).all()
57
58
59
-def test_audio_resampler_librosa_calls_resample(dummy_audio):
60
- resampler = AudioResampler(target_sr=22050, method="librosa")
61
- with patch("vllm.multimodal.audio.resample_audio_librosa") as mock_resample:
59
+def test_audio_resampler_pyav_calls_resample(dummy_audio):
60
+ resampler = AudioResampler(target_sr=22050, method="pyav")
61
+ with patch("vllm.multimodal.audio.resample_audio_pyav") as mock_resample:
62
mock_resample.return_value = dummy_audio
63
out = resampler.resample(dummy_audio, orig_sr=44100)
64
mock_resample.assert_called_once_with(
@@ -423,13 +423,13 @@ class TestAudioPipelineE2E:
423
# Verify channel averaging: mean of [0.5, -0.5] = 0.0
424
np.testing.assert_array_almost_equal(audio_output, np.zeros(16000), decimal=5)
425
426
- def test_librosa_mono_passthrough_e2e(self):
427
- """Full pipeline: librosa mono format → preserved as mono."""
426
+ def test_pyav_mono_passthrough_e2e(self):
427
+ """Full pipeline: pyav mono format → preserved as mono."""
428
from vllm.multimodal.parse import MultiModalDataParser
429
430
- # Simulate librosa output: already mono (time,) format
431
- mono_librosa = np.random.randn(16000).astype(np.float32)
432
- assert mono_librosa.shape == (16000,)
430
+ # Simulate pyav output: already mono (time,) format
431
+ mono_pyav = np.random.randn(16000).astype(np.float32)
432
+ assert mono_pyav.shape == (16000,)
433
434
# Create parser with mono normalization
435
parser = MultiModalDataParser(
@@ -438,7 +438,7 @@ class TestAudioPipelineE2E:
438
)
439
440
# Process audio through the parser
441
- result = parser._parse_audio_data((mono_librosa, 16000))
441
+ result = parser._parse_audio_data((mono_pyav, 16000))
442
audio_output = result.get(0)
443
444
# Verify output is still mono 1D
@@ -446,7 +446,7 @@ class TestAudioPipelineE2E:
446
assert audio_output.shape == (16000,)
447
448
# Verify audio content is preserved
449
- np.testing.assert_array_almost_equal(audio_output, mono_librosa)
449
+ np.testing.assert_array_almost_equal(audio_output, mono_pyav)
450
451
def test_multichannel_5_1_surround_to_mono_e2e(self):
452
"""Full pipeline: 5.1 surround (6 channels) → mono output."""
vllm/assets/audio.py
+2
-7
@@ -8,15 +8,10 @@ from urllib.parse import urljoin
8
9
import numpy.typing as npt
10
11
-from vllm.utils.import_utils import PlaceholderModule
11
+from vllm.multimodal.media.audio import load_audio
12
13
from .base import VLLM_S3_BUCKET_URL, get_vllm_public_assets
14
15
-try:
16
- import librosa
17
-except ImportError:
18
- librosa = PlaceholderModule("librosa") # type: ignore[assignment]
19
-
15
ASSET_DIR = "multimodal_asset"
16
17
AudioAssetName = Literal["winning_call", "mary_had_lamb"]
@@ -33,7 +28,7 @@ class AudioAsset:
28
@property
29
def audio_and_sample_rate(self) -> tuple[npt.NDArray, float]:
30
audio_path = get_vllm_public_assets(filename=self.filename, s3_prefix=ASSET_DIR)
36
- return librosa.load(audio_path, sr=None)
31
+ return load_audio(audio_path, sr=None)
32
33
def get_local_path(self) -> Path:
34
return get_vllm_public_assets(filename=self.filename, s3_prefix=ASSET_DIR)
vllm/assets/video.py
+2
-7
@@ -10,15 +10,10 @@ import numpy.typing as npt
10
from huggingface_hub import hf_hub_download
11
from PIL import Image
12
13
-from vllm.utils.import_utils import PlaceholderModule
13
+from vllm.multimodal.media.audio import load_audio_pyav
14
15
from .base import get_cache_dir
16
17
-try:
18
- import librosa
19
-except ImportError:
20
- librosa = PlaceholderModule("librosa") # type: ignore[assignment]
21
-
17
18
@lru_cache
19
def download_video_asset(filename: str) -> str:
@@ -146,4 +141,4 @@ class VideoAsset:
141
142
See also: examples/offline_inference/qwen2_5_omni/only_thinker.py
143
"""
149
- return librosa.load(self.video_path, sr=sampling_rate)[0]
144
+ return load_audio_pyav(self.video_path, sr=sampling_rate)[0]
vllm/benchmarks/datasets.py
+2
-5
@@ -38,6 +38,7 @@ from typing_extensions import deprecated
38
from vllm.lora.request import LoRARequest
39
from vllm.lora.utils import get_adapter_absolute_path
40
from vllm.multimodal import MultiModalDataDict
41
+from vllm.multimodal.audio import get_audio_duration
42
from vllm.multimodal.image import convert_image_mode
43
from vllm.tokenizers import TokenizerLike
44
from vllm.utils.argparse_utils import FlexibleArgumentParser
@@ -54,10 +55,6 @@ try:
55
except ImportError:
56
pd = PlaceholderModule("pandas")
57
57
-try:
58
- import librosa
59
-except ImportError:
60
- librosa = PlaceholderModule("librosa")
58
59
logger = logging.getLogger(__name__)
60
@@ -3253,7 +3250,7 @@ class ASRDataset(HuggingFaceDataset):
3250
break
3251
audio = item["audio"]
3252
y, sr = audio["array"], audio["sampling_rate"]
3256
- duration_s = librosa.get_duration(y=y, sr=sr)
3253
+ duration_s = get_audio_duration(y=y, sr=sr)
3254
if duration_s < asr_min_audio_len_sec or duration_s > asr_max_audio_len_sec:
3255
skipped += 1
3256
continue
vllm/entrypoints/openai/speech_to_text/speech_to_text.py
+8
-46
@@ -42,32 +42,13 @@ from vllm.inputs import EncoderDecoderInputs, ProcessorInputs
42
from vllm.logger import init_logger
43
from vllm.logprobs import FlatLogprobs, Logprob
44
from vllm.model_executor.models import SupportsTranscription
45
-from vllm.multimodal.audio import split_audio
46
-from vllm.multimodal.media.audio import extract_audio_from_video_bytes
45
+from vllm.multimodal.audio import get_audio_duration, split_audio
46
+from vllm.multimodal.media.audio import load_audio
47
from vllm.outputs import RequestOutput
48
from vllm.renderers.inputs import DictPrompt, EncoderDecoderDictPrompt
49
from vllm.renderers.inputs.preprocess import parse_enc_dec_prompt, parse_model_prompt
50
from vllm.sampling_params import BeamSearchParams, SamplingParams
51
from vllm.tokenizers import get_tokenizer
52
-from vllm.utils.import_utils import PlaceholderModule
53
-
54
-try:
55
- import librosa
56
-except ImportError:
57
- librosa = PlaceholderModule("librosa") # type: ignore[assignment]
58
-
59
-try:
60
- import soundfile as sf
61
-except ImportError:
62
- sf = PlaceholderModule("soundfile") # type: ignore[assignment]
63
-
64
-# Public libsndfile error codes exposed via `soundfile.LibsndfileError.code`, soundfile
65
-# being librosa's main backend. Used to validate if an audio loading error is due to a
66
-# server error vs a client error (invalid audio file).
67
-# 1 = unrecognised format (file is not a supported audio container)
68
-# 3 = malformed file (corrupt or structurally invalid audio)
69
-# 4 = unsupported encoding (codec not supported by this libsndfile build)
70
-_BAD_SF_CODES = {1, 3, 4}
52
53
SpeechToTextResponse: TypeAlias = TranscriptionResponse | TranslationResponse
54
SpeechToTextResponseVerbose: TypeAlias = (
@@ -214,32 +195,13 @@ class OpenAISpeechToText(OpenAIServing):
195
# pre-requisite for chunking, as it assumes Whisper SR.
196
try:
197
with io.BytesIO(audio_data) as buf:
217
- y, sr = librosa.load(buf, sr=self.asr_config.sample_rate) # type: ignore[return-value]
218
- except sf.LibsndfileError as exc:
219
- # Only fall back for known format-detection failures.
220
- # Re-raise anything else (e.g. corrupt but recognised format).
221
- if exc.code not in _BAD_SF_CODES:
222
- raise
223
- logger.debug(
224
- "librosa/soundfile could not decode audio from BytesIO "
225
- "(code=%s: %s); falling back to pyav in-process decode",
226
- exc.code,
227
- exc,
228
- )
229
- try:
230
- native_y, native_sr = extract_audio_from_video_bytes(audio_data)
231
- sr = self.asr_config.sample_rate
232
- y = librosa.resample(native_y, orig_sr=native_sr, target_sr=sr)
233
- except Exception as pyav_exc:
234
- logger.debug(
235
- "pyAV fallback also failed: %s",
236
- pyav_exc,
237
- )
238
- raise ValueError("Invalid or unsupported audio file.") from pyav_exc
198
+ y, sr = load_audio(buf, sr=self.asr_config.sample_rate)
199
+ except Exception as exc:
200
+ raise ValueError("Invalid or unsupported audio file.") from exc
201
240
- duration = librosa.get_duration(y=y, sr=sr)
241
- do_split_audio = (
242
- self.asr_config.allow_audio_chunking
202
+ duration = get_audio_duration(y=y, sr=sr)
203
+ do_split_audio = self.asr_config.allow_audio_chunking and (
204
+ self.asr_config.max_audio_clip_s is not None
205
and duration > self.asr_config.max_audio_clip_s
206
)
207
vllm/model_executor/models/nano_nemotron_vl.py
+3
-2
@@ -12,6 +12,7 @@ import math
12
import warnings
13
from collections.abc import Iterable, Mapping, Sequence
14
from functools import cached_property
15
+from io import BytesIO
16
from typing import Annotated, Literal, TypeAlias
17
18
import torch
@@ -53,7 +54,7 @@ from vllm.multimodal.inputs import (
54
MultiModalKwargsItems,
55
VideoItem,
56
)
56
-from vllm.multimodal.media.audio import extract_audio_from_video_bytes
57
+from vllm.multimodal.media.audio import load_audio_pyav
58
from vllm.multimodal.parse import (
59
AudioProcessorItems,
60
ImageEmbeddingItems,
@@ -553,7 +554,7 @@ class NanoNemotronVLMultiModalProcessor(
554
"video must be loaded with keep_video_bytes=True (e.g. via "
555
"the chat API with a model that sets use_audio_in_video)."
556
)
556
- audio_items.append(extract_audio_from_video_bytes(video_bytes))
557
+ audio_items.append(load_audio_pyav(BytesIO(video_bytes)))
558
559
# Create a new VideoProcessorItems with metadata that does not contain
560
# the large video bytes, to avoid modifying the input `mm_items`.
vllm/multimodal/audio.py
+87
-9
@@ -12,17 +12,35 @@ import torch
12
from vllm.utils.import_utils import PlaceholderModule
13
14
try:
15
- import librosa
15
+ import av as av
16
except ImportError:
17
- librosa = PlaceholderModule("librosa") # type: ignore[assignment]
17
+ av = PlaceholderModule("av") # type: ignore[assignment]
18
19
+try:
20
+ import resampy
21
+except ImportError:
22
+ resampy = PlaceholderModule("resampy") # type: ignore[assignment]
23
24
try:
25
import scipy.signal as scipy_signal
26
except ImportError:
27
scipy_signal = PlaceholderModule("scipy").placeholder_attr("signal") # type: ignore[assignment]
28
29
+
30
# ============================================================
31
+# Aligned with `librosa.get_duration` function
32
+def get_audio_duration(*, y: npt.NDArray[np.floating], sr: float = 22050) -> float:
33
+ """Get the duration of an audio array in seconds.
34
+
35
+ Args:
36
+ y: Audio time series. Can be 1D (samples,) or 2D (channels, samples).
37
+ sr: Sample rate of the audio in Hz.
38
+
39
+ Returns:
40
+ Duration of the audio in seconds.
41
+ """
42
+ n_samples = y.shape[-1]
43
+ return float(n_samples) / sr
44
45
46
class ChannelReduction(str, Enum):
@@ -153,13 +171,71 @@ def normalize_audio(
171
# ============================================================
172
173
156
-def resample_audio_librosa(
174
+def resample_audio_pyav(
175
audio: npt.NDArray[np.floating],
176
*,
177
orig_sr: float,
178
target_sr: float,
179
) -> npt.NDArray[np.floating]:
162
- return librosa.resample(audio, orig_sr=orig_sr, target_sr=target_sr)
180
+ """Resample audio using PyAV (libswresample via FFmpeg).
181
+
182
+ Args:
183
+ audio: Input audio. Can be:
184
+ - 1D array ``(samples,)``: mono audio
185
+ - 2D array ``(channels, samples)``: stereo audio
186
+ orig_sr: Original sample rate in Hz.
187
+ target_sr: Target sample rate in Hz.
188
+
189
+ Returns:
190
+ Resampled audio with the same shape as the input (1D → 1D, 2D → 2D).
191
+ """
192
+ orig_sr_int = int(round(orig_sr))
193
+ target_sr_int = int(round(target_sr))
194
+
195
+ if orig_sr_int == target_sr_int:
196
+ return audio
197
+
198
+ if audio.ndim == 2:
199
+ # Resample each channel independently and re-stack.
200
+ return np.stack(
201
+ [
202
+ resample_audio_pyav(ch, orig_sr=orig_sr, target_sr=target_sr)
203
+ for ch in audio
204
+ ],
205
+ axis=0,
206
+ )
207
+
208
+ expected_len = int(math.ceil(audio.shape[-1] * target_sr_int / orig_sr_int))
209
+
210
+ # from_ndarray expects shape (channels, samples) for planar formats.
211
+ # libswresample requires a minimum number of input samples to produce
212
+ # output frames; pad short inputs with zeros so we always get output,
213
+ # then trim to the expected output length.
214
+ _MIN_SAMPLES = 1024
215
+ audio_f32 = np.asarray(audio, dtype=np.float32)
216
+ if len(audio_f32) < _MIN_SAMPLES:
217
+ audio_f32 = np.pad(audio_f32, (0, _MIN_SAMPLES - len(audio_f32)))
218
+ audio_f32 = audio_f32.reshape(1, -1)
219
+
220
+ resampler = av.AudioResampler(format="fltp", layout="mono", rate=target_sr_int)
221
+
222
+ frame = av.AudioFrame.from_ndarray(audio_f32, format="fltp", layout="mono")
223
+ frame.sample_rate = orig_sr_int
224
+
225
+ out_frames = resampler.resample(frame)
226
+ out_frames.extend(resampler.resample(None)) # flush buffered samples
227
+
228
+ result = np.concatenate([f.to_ndarray() for f in out_frames], axis=1).squeeze(0)
229
+ return result[:expected_len]
230
+
231
+
232
+def resample_audio_resampy(
233
+ audio: npt.NDArray[np.floating],
234
+ *,
235
+ orig_sr: float,
236
+ target_sr: float,
237
+) -> npt.NDArray[np.floating]:
238
+ return resampy.resample(audio, sr_orig=orig_sr, sr_new=target_sr)
239
240
241
def resample_audio_scipy(
@@ -167,7 +243,7 @@ def resample_audio_scipy(
243
*,
244
orig_sr: float,
245
target_sr: float,
170
-):
246
+) -> npt.NDArray[np.floating]:
247
if orig_sr > target_sr:
248
return scipy_signal.resample_poly(audio, 1, orig_sr // target_sr)
249
elif orig_sr < target_sr:
@@ -181,7 +257,7 @@ class AudioResampler:
257
def __init__(
258
self,
259
target_sr: float | None = None,
184
- method: Literal["librosa", "scipy"] = "librosa",
260
+ method: Literal["pyav", "resampy", "scipy"] = "resampy",
261
):
262
self.target_sr = target_sr
263
self.method = method
@@ -203,8 +279,10 @@ class AudioResampler:
279
abs_tol=1e-6,
280
):
281
return audio
206
- if self.method == "librosa":
207
- return resample_audio_librosa(
282
+ if self.method == "pyav":
283
+ return resample_audio_pyav(audio, orig_sr=orig_sr, target_sr=self.target_sr)
284
+ if self.method == "resampy":
285
+ return resample_audio_resampy(
286
audio, orig_sr=orig_sr, target_sr=self.target_sr
287
)
288
elif self.method == "scipy":
@@ -214,7 +292,7 @@ class AudioResampler:
292
else:
293
raise ValueError(
294
f"Invalid resampling method: {self.method}. "
217
- "Supported methods are 'librosa' and 'scipy'."
295
+ "Supported methods are 'pyav' and 'scipy'."
296
)
297
298
vllm/multimodal/media/audio.py
+96
-58
@@ -1,5 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import math
4
from io import BytesIO
5
from pathlib import Path
6
@@ -14,58 +15,80 @@ from vllm.utils.serial_utils import tensor2base64
15
from .base import MediaIO
16
17
try:
17
- import librosa
18
+ import av
19
except ImportError:
19
- librosa = PlaceholderModule("librosa") # type: ignore[assignment]
20
+ av = PlaceholderModule("av") # type: ignore[assignment]
21
22
try:
23
import soundfile
24
except ImportError:
25
soundfile = PlaceholderModule("soundfile") # type: ignore[assignment]
26
27
+
28
try:
27
- import av
29
+ import resampy
30
except ImportError:
29
- av = PlaceholderModule("av") # type: ignore[assignment]
31
+ resampy = PlaceholderModule("resampy") # type: ignore[assignment]
32
33
32
-def extract_audio_from_video_bytes(
33
- data: bytes,
34
-) -> tuple[npt.NDArray, float]:
35
- """Extract the audio track from raw video bytes using PyAV.
34
+# Public libsndfile error codes exposed via `soundfile.LibsndfileError.code`, soundfile
35
+# being librosa's main backend. Used to validate if an audio loading error is due to a
36
+# server error vs a client error (invalid audio file).
37
+# 1 = unrecognised format (file is not a supported audio container)
38
+# 3 = malformed file (corrupt or structurally invalid audio)
39
+# 4 = unsupported encoding (codec not supported by this libsndfile build)
40
+_BAD_SF_CODES = {1, 3, 4}
41
37
- PyAV wraps FFmpeg's C libraries in-process — no subprocess is
38
- spawned, which is critical to avoid crashing CUDA-active vLLM
39
- worker processes.
42
41
- The returned waveform is at the native sample rate of the video's
42
- audio stream. Resampling to a model-specific rate is left to the
43
- downstream :class:`AudioResampler` in the parsing pipeline.
43
+def load_audio_pyav(
44
+ path: BytesIO | Path | str,
45
+ *,
46
+ sr: float | None = 22050,
47
+ mono: bool = True,
48
+) -> tuple[npt.NDArray, float]:
49
+ """Load an audio file using PyAV (FFmpeg), returning float32 mono waveform.
50
+
51
+ Decodes the audio stream at its native sample rate. Channel reduction to
52
+ mono is performed by averaging across channels. Resampling to a
53
+ model-specific rate is left to the downstream :class:`AudioResampler`.
54
55
Args:
46
- data: Raw video file bytes (e.g. from an mp4 file).
56
+ path: A :class:`~io.BytesIO` buffer, a filesystem
57
+ :class:`~pathlib.Path`, or a string path.
58
59
Returns:
49
- A tuple of ``(waveform, sample_rate)`` suitable for use as an
50
- :class:`AudioItem`.
60
+ ``(waveform, sample_rate)`` where *waveform* is a 1-D float32
61
+ NumPy array and *sample_rate* is the native sample rate in Hz.
62
"""
52
- if data is None or len(data) == 0:
53
- raise ValueError(
54
- "Cannot extract audio: video bytes are missing or empty. "
55
- "Ensure video was loaded with keep_video_bytes=True for "
56
- "audio-in-video extraction."
57
- )
63
+ native_sr = None
64
try:
59
- with av.open(BytesIO(data)) as container:
65
+ with av.open(path) as container:
66
if not container.streams.audio:
61
- raise ValueError("No audio stream found in the video.")
67
+ raise ValueError("No audio stream found.")
68
stream = container.streams.audio[0]
69
+ stream.thread_type = "AUTO"
70
native_sr = stream.rate
71
+ sr = sr or native_sr
72
73
chunks: list[npt.NDArray] = []
66
- for frame in container.decode(audio=0):
67
- arr = frame.to_ndarray()
68
- chunks.append(arr.mean(axis=0) if arr.ndim > 1 else arr)
74
+ needs_resampling = not math.isclose(
75
+ float(sr),
76
+ float(native_sr),
77
+ rel_tol=0.0,
78
+ abs_tol=1e-6,
79
+ )
80
+ resampler = (
81
+ av.AudioResampler(format="fltp", layout="mono", rate=sr)
82
+ if needs_resampling
83
+ else None
84
+ )
85
+ for frame in container.decode(stream):
86
+ if needs_resampling:
87
+ assert resampler is not None
88
+ for out_frame in resampler.resample(frame):
89
+ chunks.append(out_frame.to_ndarray())
90
+ else:
91
+ chunks.append(frame.to_ndarray())
92
except ValueError:
93
raise
94
except Exception as e:
@@ -77,37 +100,54 @@ def extract_audio_from_video_bytes(
100
if not chunks:
101
raise ValueError("No audio found in the video.")
102
80
- audio = np.concatenate(chunks).astype(np.float32)
81
- return audio, float(native_sr)
103
+ audio = np.concatenate(chunks, axis=-1).astype(np.float32)
104
+ if mono and audio.ndim > 1:
105
+ audio = np.mean(audio, axis=0)
106
107
+ return audio, sr
108
84
-def is_video(data: bytes) -> bool:
85
- """Check if the fetched bytes are video"""
86
- if len(data) < 12:
87
- return False
109
89
- box_type = data[4:8]
90
- major_brand = data[8:12]
110
+def load_audio_soundfile(
111
+ path: BytesIO | Path | str,
112
+ *,
113
+ sr: float | None = 22050,
114
+ mono: bool = True,
115
+) -> tuple[np.ndarray, int]:
116
+ """Load audio via soundfile"""
117
+ with soundfile.SoundFile(path) as f:
118
+ native_sr = f.samplerate
119
+ y = f.read(dtype="float32", always_2d=False).T
120
92
- MP4_BRANDS = {
93
- b"mp41",
94
- b"mp42", # MP4
95
- b"isom", # ISO Base Media
96
- b"iso2",
97
- b"iso4",
98
- b"iso5",
99
- b"iso6",
100
- b"M4V ",
101
- b"M4A ", # Apple
102
- b"avc1", # H.264
103
- b"dash", # DASH
104
- b"mmp4",
105
- b"MSNV",
106
- }
121
+ if mono and y.ndim > 1:
122
+ y = np.mean(y, axis=tuple(range(y.ndim - 1)))
123
108
- is_avi = data[:4] == b"RIFF" and major_brand == b"AVI "
109
- is_mp4 = box_type == b"ftyp" and major_brand in MP4_BRANDS
110
- return is_mp4 or is_avi
124
+ if sr is not None and sr != native_sr:
125
+ y = resampy.resample(y, sr_orig=native_sr, sr_new=sr)
126
+ return y, int(sr)
127
+ return y, native_sr
128
+
129
+
130
+def load_audio(
131
+ path: BytesIO | Path | str,
132
+ *,
133
+ sr: float | None = 22050,
134
+ mono: bool = True,
135
+):
136
+ try:
137
+ return load_audio_soundfile(path, sr=sr, mono=mono)
138
+ except soundfile.LibsndfileError as exc:
139
+ # Only fall back for known format-detection failures.
140
+ # Re-raise anything else (e.g. corrupt but recognised format).
141
+ if exc.code not in _BAD_SF_CODES:
142
+ raise
143
+ # soundfile may have advanced the BytesIO seek position before failing;
144
+ # reset it so PyAV can read from the beginning.
145
+ if isinstance(path, BytesIO):
146
+ path.seek(0)
147
+ try:
148
+ return load_audio_pyav(path, sr=sr, mono=mono)
149
+ except Exception as pyav_exc:
150
+ raise ValueError("Invalid or unsupported audio file.") from pyav_exc
151
152
153
class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
@@ -128,9 +168,7 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
168
self.kwargs = kwargs
169
170
def load_bytes(self, data: bytes) -> tuple[npt.NDArray, float]:
131
- if is_video(data):
132
- return extract_audio_from_video_bytes(data)
133
- return librosa.load(BytesIO(data), sr=None)
171
+ return load_audio(BytesIO(data), sr=None)
172
173
def load_base64(
174
self,
@@ -140,7 +178,7 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
178
return self.load_bytes(pybase64.b64decode(data))
179
180
def load_file(self, filepath: Path) -> tuple[npt.NDArray, float]:
143
- return librosa.load(filepath, sr=None)
181
+ return load_audio(filepath, sr=None)
182
183
def encode_base64(
184
self,
vllm/multimodal/parse.py
+1
-1
@@ -497,7 +497,7 @@ class MultiModalDataParser:
497
*,
498
target_sr: float | None = None,
499
target_channels: int | None = None,
500
- audio_resample_method: Literal["librosa", "scipy"] = "librosa",
500
+ audio_resample_method: Literal["pyav", "scipy"] = "pyav",
501
video_needs_metadata: bool = False,
502
expected_hidden_size: int | None = None,
503
) -> None:
vllm/renderers/base.py
-3
@@ -172,9 +172,6 @@ class BaseRenderer(ABC, Generic[_T]):
172
173
For chat requests:
174
- Jinja2 template compilation
175
-
176
- For multi-modal requests:
177
- - Importing libraries such as librosa triggers JIT compilation.
175
"""
176
from vllm.entrypoints.chat_utils import ChatTemplateResolutionError
177
vllm/transformers_utils/processors/fireredasr2.py
+1
-1
@@ -188,7 +188,7 @@ class FireRedASR2FeatureExtractor(SequenceFeatureExtractor):
188
for speech in raw_speech:
189
"""
190
We must multiply by 32768 here because FireRedASR2 loads audio data
191
- using kaldiio.load_mat, while vLLM loads audio data using librosa.
191
+ using kaldiio.load_mat, while vLLM loads audio data using pyav.
192
"""
193
speech = speech * 32768
194
fbank = self.fbank(sampling_rate, speech)