A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 01:00 UTC
[Refactor] Drop direct dependency on librosa (#39079)
Signed-off-by: Nick Cao <ncao@redhat.com> Co-authored-by: Claude <noreply@anthropic.com>
Nick Cao committed
Apr 18, 2026 at 02:55 UTC
153ba7f0f3d882c4feb84cea63b6a4e9618c9537
13 files changed
+40
-44
docs/contributing/model/transcription.md
+3
-3
@@ -193,7 +193,7 @@ Provide a fast duration→token estimate to improve streaming usage statistics:
193
194
The API server takes care of basic audio I/O and optional chunking before building prompts:
195
196
-- Resampling: Input audio is resampled to `SpeechToTextConfig.sample_rate` using `librosa`.
196
+- Resampling: Input audio is resampled to `SpeechToTextConfig.sample_rate` using `AudioResampler`.
197
- Chunking: If `SpeechToTextConfig.allow_audio_chunking` is True and the duration exceeds `max_audio_clip_s`, the server splits the audio into overlapping chunks and generates a prompt per chunk. Overlap is controlled by `overlap_chunk_second`.
198
- Energy-aware splitting: When `min_energy_split_window_size` is set, the server finds low-energy regions to minimize cutting within words.
199
@@ -206,8 +206,8 @@ Relevant server logic:
206
async def _preprocess_speech_to_text(...):
207
language = self.model_cls.validate_language(request.language)
208
...
209
- y, sr = librosa.load(bytes_, sr=self.asr_config.sample_rate)
210
- duration = librosa.get_duration(y=y, sr=sr)
209
+ y, sr = load_audio(bytes_, sr=self.asr_config.sample_rate)
210
+ duration = get_audio_duration(y=y, sr=sr)
211
do_split_audio = (self.asr_config.allow_audio_chunking
212
and duration > self.asr_config.max_audio_clip_s)
213
chunks = [y] if not do_split_audio else self._split_audio(y, int(sr))
docs/features/multimodal_inputs.md
+3
-3
@@ -300,12 +300,12 @@ Full example: [examples/offline_inference/audio_language.py](../../examples/offl
300
Speech-to-text models like Whisper have a maximum audio length they can process (typically 30 seconds). For longer audio files, vLLM provides a utility to intelligently split audio into chunks at quiet points to minimize cutting through speech.
301
302
```python
303
-import librosa
303
from vllm import LLM, SamplingParams
304
from vllm.multimodal.audio import split_audio
305
+from vllm.multimodal.media.audio import load_audio
306
307
# Load long audio file
308
-audio, sr = librosa.load("long_audio.wav", sr=16000)
308
+audio, sr = load_audio("long_audio.wav", sr=16000)
309
310
# Split into chunks at low-energy (quiet) regions
311
chunks = split_audio(
@@ -832,7 +832,7 @@ Then, you can use the OpenAI client as follows:
832
base_url=openai_api_base,
833
)
834
835
- # Any format supported by librosa is supported
835
+ # Any format supported by soundfile/PyAV is supported
836
audio_url = AudioAsset("winning_call").url
837
audio_base64 = encode_base64_content_from_url(audio_url)
838
examples/online_serving/openai_chat_completion_client_for_multimodal.py
+3
-3
@@ -267,7 +267,7 @@ def run_audio(model: str, max_completion_tokens: int) -> None:
267
{
268
"type": "input_audio",
269
"input_audio": {
270
- # Any format supported by librosa is supported
270
+ # Any format supported by soundfile/PyAV is supported
271
"data": audio_base64,
272
"format": "wav",
273
},
@@ -292,7 +292,7 @@ def run_audio(model: str, max_completion_tokens: int) -> None:
292
{
293
"type": "audio_url",
294
"audio_url": {
295
- # Any format supported by librosa is supported
295
+ # Any format supported by soundfile/PyAV is supported
296
"url": audio_url
297
},
298
},
@@ -316,7 +316,7 @@ def run_audio(model: str, max_completion_tokens: int) -> None:
316
{
317
"type": "audio_url",
318
"audio_url": {
319
- # Any format supported by librosa is supported
319
+ # Any format supported by soundfile/PyAV is supported
320
"url": f"data:audio/ogg;base64,{audio_base64}"
321
},
322
},
examples/online_serving/openai_realtime_client.py
+2
-3
@@ -12,7 +12,6 @@ model, for example:
12
Requirements:
13
- vllm with audio support
14
- websockets
15
-- librosa
15
- numpy
16
17
The script:
@@ -26,12 +25,12 @@ import argparse
25
import asyncio
26
import json
27
29
-import librosa
28
import numpy as np
29
import pybase64 as base64
30
import websockets
31
32
from vllm.assets.audio import AudioAsset
33
+from vllm.multimodal.media.audio import load_audio
34
35
36
def audio_to_pcm16_base64(audio_path: str) -> str:
@@ -39,7 +38,7 @@ def audio_to_pcm16_base64(audio_path: str) -> str:
38
Load an audio file and convert it to base64-encoded PCM16 @ 16kHz.
39
"""
40
# Load audio and resample to 16kHz mono
42
- audio, _ = librosa.load(audio_path, sr=16000, mono=True)
41
+ audio, _ = load_audio(audio_path, sr=16000, mono=True)
42
# Convert to PCM16
43
pcm16 = (audio * 32767).astype(np.int16)
44
# Encode as base64
tests/entrypoints/openai/correctness/test_transcription_api_correctness.py
+3
-3
@@ -13,7 +13,6 @@ import io
13
import time
14
from statistics import mean, median
15
16
-import librosa
16
import pytest
17
import soundfile
18
import torch
@@ -21,6 +20,7 @@ from datasets import load_dataset
20
from evaluate import load
21
from transformers.models.whisper.english_normalizer import EnglishTextNormalizer
22
23
+from vllm.multimodal.audio import get_audio_duration
24
from vllm.tokenizers import get_tokenizer
25
26
from ....models.registry import HF_EXAMPLE_MODELS
@@ -84,7 +84,7 @@ async def process_dataset(model, client, data, concurrent_request):
84
trust_remote_code=model_info.trust_remote_code,
85
)
86
87
- # Warmup call as the first `librosa.load` server-side is quite slow.
87
+ # Warmup call as the first `load_audio` server-side is quite slow.
88
audio, sr = data[0]["audio"]["array"], data[0]["audio"]["sampling_rate"]
89
_ = await bound_transcribe(sem, client, tokenizer, (audio, sr), "")
90
@@ -118,7 +118,7 @@ def print_performance_metrics(results, total_time):
118
119
def add_duration(sample):
120
y, sr = sample["audio"]["array"], sample["audio"]["sampling_rate"]
121
- sample["duration_ms"] = librosa.get_duration(y=y, sr=sr) * 1000
121
+ sample["duration_ms"] = get_audio_duration(y=y, sr=sr) * 1000
122
return sample
123
124
tests/entrypoints/openai/realtime/test_realtime_validation.py
+2
-2
@@ -5,7 +5,6 @@ import asyncio
5
import json
6
import warnings
7
8
-import librosa
8
import numpy as np
9
import pybase64 as base64
10
import pytest
@@ -14,6 +13,7 @@ import websockets
13
from tests.entrypoints.openai.conftest import add_attention_backend
14
from tests.utils import ROCM_ENV_OVERRIDES, ROCM_EXTRA_ARGS, RemoteOpenAIServer
15
from vllm.assets.audio import AudioAsset
16
+from vllm.multimodal.media.audio import load_audio
17
18
# Increase engine iteration timeout for ROCm where first-use JIT compilation
19
# can exceed the default 60s, causing a silent deadlock in feed_tokens.
@@ -56,7 +56,7 @@ async def send_event(ws, event: dict) -> None:
56
def mary_had_lamb_audio_chunks() -> list[str]:
57
"""Audio split into ~1 second chunks for streaming."""
58
path = AudioAsset("mary_had_lamb").get_local_path()
59
- audio, _ = librosa.load(str(path), sr=16000, mono=True)
59
+ audio, _ = load_audio(str(path), sr=16000, mono=True)
60
61
# Split into ~0.1 second chunks (1600 samples at 16kHz)
62
chunk_size = 1600
tests/entrypoints/openai/speech_to_text/test_transcription_validation_whisper.py
+2
-2
@@ -6,7 +6,6 @@ import asyncio
6
import io
7
import json
8
9
-import librosa
9
import numpy as np
10
import openai
11
import pytest
@@ -14,6 +13,7 @@ import pytest_asyncio
13
import soundfile as sf
14
15
from tests.utils import RemoteOpenAIServer
16
+from vllm.multimodal.media.audio import load_audio
17
from vllm.platforms import current_platform
18
19
MODEL_NAME = "openai/whisper-large-v3-turbo"
@@ -134,7 +134,7 @@ async def test_bad_requests(mary_had_lamb, whisper_client):
134
@pytest.mark.asyncio
135
async def test_long_audio_request(mary_had_lamb, whisper_client):
136
mary_had_lamb.seek(0)
137
- audio, sr = librosa.load(mary_had_lamb)
137
+ audio, sr = load_audio(mary_had_lamb)
138
# Add small silence after each audio for repeatability in the split process
139
audio = np.pad(audio, (0, 1600))
140
repeated_audio = np.tile(audio, 10)
tests/entrypoints/openai/speech_to_text/test_translation_validation.py
+2
-2
@@ -7,7 +7,6 @@ import io
7
import json
8
9
import httpx
10
-import librosa
10
import numpy as np
11
import openai
12
import pytest
@@ -17,6 +16,7 @@ import soundfile as sf
16
from tests.entrypoints.openai.conftest import add_attention_backend
17
from tests.utils import RemoteOpenAIServer
18
from vllm.logger import init_logger
19
+from vllm.multimodal.media.audio import load_audio
20
21
logger = init_logger(__name__)
22
@@ -264,7 +264,7 @@ async def test_long_audio_request(foscolo, client_and_model):
264
if model_name == "google/gemma-3n-E2B-it":
265
pytest.skip("Gemma3n does not support long audio requests")
266
foscolo.seek(0)
267
- audio, sr = librosa.load(foscolo)
267
+ audio, sr = load_audio(foscolo)
268
repeated_audio = np.tile(audio, 2)
269
# Repeated audio to buffer
270
buffer = io.BytesIO()
tests/models/multimodal/generation/test_phi4mm.py
+2
-2
@@ -4,7 +4,6 @@
4
import os
5
from collections.abc import Sequence
6
7
-import librosa
7
import pytest
8
import regex as re
9
from huggingface_hub import snapshot_download
@@ -14,6 +13,7 @@ from vllm.assets.image import ImageAsset
13
from vllm.logprobs import SampleLogprobs
14
from vllm.lora.request import LoRARequest
15
from vllm.multimodal.image import convert_image_mode, rescale_image_size
16
+from vllm.multimodal.media.audio import load_audio
17
18
from ....conftest import (
19
IMAGE_ASSETS,
@@ -290,7 +290,7 @@ def test_vision_speech_models(
290
num_logprobs: int,
291
) -> None:
292
# use the example speech question so that the model outputs are reasonable
293
- audio = librosa.load(speech_question, sr=None)
293
+ audio = load_audio(speech_question, sr=None)
294
image = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
295
296
inputs_vision_speech = [
tests/models/multimodal/generation/test_whisper.py
+3
-4
@@ -4,11 +4,11 @@
4
from collections.abc import Sequence
5
from typing import Any
6
7
-import librosa
7
import pytest
8
from transformers import AutoModelForSpeechSeq2Seq
9
10
from vllm.assets.audio import AudioAsset
11
+from vllm.multimodal.audio import AudioResampler
12
from vllm.platforms import current_platform
13
14
from ....conftest import HfRunner, PromptAudioInput, VllmRunner
@@ -93,13 +93,12 @@ def run_test(
93
def resampled_assets() -> list[tuple[Any, int]]:
94
audio_assets = [AudioAsset("mary_had_lamb"), AudioAsset("winning_call")]
95
sampled_assets = []
96
+ resampler = AudioResampler(target_sr=WHISPER_SAMPLE_RATE)
97
for asset in audio_assets:
98
audio, orig_sr = asset.audio_and_sample_rate
99
# Resample to Whisper's expected sample rate (16kHz)
100
if orig_sr != WHISPER_SAMPLE_RATE:
100
- audio = librosa.resample(
101
- audio, orig_sr=orig_sr, target_sr=WHISPER_SAMPLE_RATE
102
- )
101
+ audio = resampler.resample(audio, orig_sr=orig_sr)
102
sampled_assets.append(
103
(audio, WHISPER_SAMPLE_RATE),
104
)
tests/multimodal/media/test_audio.py
+2
-2
@@ -3,12 +3,12 @@
3
from pathlib import Path
4
from unittest.mock import patch
5
6
-import librosa
6
import numpy as np
7
import pybase64 as base64
8
import pytest
9
10
from vllm.multimodal.media import AudioMediaIO
11
+from vllm.multimodal.media.audio import load_audio
12
13
from ...conftest import AudioTestAssets
14
@@ -73,6 +73,6 @@ def test_audio_media_io_from_video(video_assets):
73
video_path = video_assets[0].video_path
74
with open(video_path, "rb") as f:
75
audio, sr = audio_io.load_bytes(f.read())
76
- audio_ref, sr_ref = librosa.load(video_path, sr=None)
76
+ audio_ref, sr_ref = load_audio(video_path, sr=None)
77
assert sr == sr_ref
78
np.testing.assert_allclose(audio_ref, audio, atol=1e-4)
vllm/multimodal/media/audio.py
+3
-3
@@ -29,9 +29,9 @@ except ImportError:
29
soundfile = PlaceholderModule("soundfile") # type: ignore[assignment]
30
31
32
-# Public libsndfile error codes exposed via `soundfile.LibsndfileError.code`, soundfile
33
-# being librosa's main backend. Used to validate if an audio loading error is due to a
34
-# server error vs a client error (invalid audio file).
32
+# Public libsndfile error codes exposed via `soundfile.LibsndfileError.code`,
33
+# soundfile being the main audio loading backend. Used to validate if an audio
34
+# loading error is due to a server error vs a client error (invalid audio file).
35
# 0 = sf_error(NULL) race condition: when multiple threads fail sf_open_virtual
36
# concurrently, one thread may clear the global error before another reads it,
37
# producing code=0 ("Garbled error message from libsndfile" in soundfile).
vllm/transformers_utils/processors/cohere_asr.py
+10
-12
@@ -4,11 +4,11 @@ import logging
4
import math
5
import random
6
7
-import librosa
7
import numpy as np
8
import torch
9
import torch.nn.functional as F
10
from torch import nn
11
+from torchaudio.functional import melscale_fbanks
12
from transformers import AutoFeatureExtractor, AutoProcessor, BatchFeature
13
from transformers.feature_extraction_sequence_utils import (
14
SequenceFeatureExtractor,
@@ -129,17 +129,15 @@ class FilterbankFeatures(nn.Module):
129
self.pad_min_duration = 0.0
130
self.pad_direction = "both"
131
132
- filterbanks = torch.tensor(
133
- librosa.filters.mel(
134
- sr=sample_rate,
135
- n_fft=self.n_fft,
136
- n_mels=nfilt,
137
- fmin=lowfreq,
138
- fmax=highfreq,
139
- norm=mel_norm,
140
- ),
141
- dtype=torch.float,
142
- ).unsqueeze(0)
132
+ filterbanks = melscale_fbanks(
133
+ n_freqs=self.n_fft // 2 + 1,
134
+ f_min=lowfreq,
135
+ f_max=highfreq,
136
+ n_mels=nfilt,
137
+ sample_rate=sample_rate,
138
+ norm=mel_norm,
139
+ mel_scale="slaney",
140
+ ).T.unsqueeze(0)
141
self.register_buffer("fb", filterbanks)
142
143
# Calculate maximum sequence length