A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Misc] Optional installation of audio related packages (#8063)
Roger Wang committed
Sep 1, 2024 at 14:46 UTC
5b86b19954d30acaebb24bc5441b184ae3fcf345
6 files changed
+29
-10
requirements-common.txt
+1
-3
@@ -22,9 +22,7 @@ typing_extensions >= 4.10
22
filelock >= 3.10.4 # filelock starts to support `mode` argument from 3.10.4
23
pyzmq
24
msgspec
25
-librosa # Required for audio processing
26
-soundfile # Required for audio processing
25
gguf == 0.9.1
26
importlib_metadata
27
mistral_common >= 1.3.4
30
-pyyaml
\ No newline at end of file
28
+pyyaml
requirements-test.txt
+3
-1
@@ -13,10 +13,12 @@ pytest-shard
13
awscli
14
einops # required for MPT, qwen-vl and Mamba
15
httpx
16
+librosa # required for audio test
17
peft
18
requests
19
ray
20
sentence-transformers # required for embedding
21
+soundfile # required for audio test
22
compressed-tensors==0.4.0 # required for compressed-tensors
23
timm # required for internvl test
24
transformers_stream_generator # required for qwen-vl test
@@ -30,4 +32,4 @@ aiohttp
32
33
# quantization
34
bitsandbytes==0.42.0
33
-buildkite-test-collector==0.1.8
\ No newline at end of file
35
+buildkite-test-collector==0.1.8
setup.py
+1
@@ -501,6 +501,7 @@ setup(
501
ext_modules=ext_modules,
502
extras_require={
503
"tensorizer": ["tensorizer>=2.9.0"],
504
+ "audio": ["librosa", "soundfile"] # Required for audio processing
505
},
506
cmdclass={"build_ext": cmake_build_ext} if len(ext_modules) > 0 else {},
507
package_data=package_data,
tests/models/test_ultravox.py
+2
-2
@@ -1,11 +1,9 @@
1
from typing import List, Optional, Tuple, Type
2
3
-import librosa
3
import numpy as np
4
import pytest
5
from transformers import AutoModel, AutoTokenizer, BatchEncoding
6
8
-from vllm.assets.audio import AudioAsset
7
from vllm.sequence import SampleLogprobs
8
from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
9
@@ -21,6 +19,7 @@ AudioTuple = Tuple[np.ndarray, int]
19
20
@pytest.fixture(scope="session")
21
def audio_and_sample_rate():
22
+ from vllm.assets.audio import AudioAsset
23
return AudioAsset("mary_had_lamb").audio_and_sample_rate
24
25
@@ -109,6 +108,7 @@ def run_test(
108
dtype=dtype,
109
postprocess_inputs=process,
110
auto_cls=AutoModel) as hf_model:
111
+ import librosa
112
113
hf_outputs_per_audio = [
114
hf_model.generate_greedy_logprobs_limit(
vllm/model_executor/models/ultravox.py
+5
-1
@@ -8,7 +8,6 @@ from functools import lru_cache
8
from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
9
TypedDict, Union, cast)
10
11
-import librosa
11
import numpy as np
12
import torch
13
import torch.utils.checkpoint
@@ -107,6 +106,11 @@ def input_mapper_for_ultravox(ctx: InputContext, data: object):
106
feature_extractor = whisper_feature_extractor(ctx)
107
108
if sr != feature_extractor.sampling_rate:
109
+ try:
110
+ import librosa
111
+ except ImportError:
112
+ raise ImportError(
113
+ "Please install vllm[audio] for audio support.") from None
114
audio = librosa.resample(audio,
115
orig_sr=sr,
116
target_sr=feature_extractor.sampling_rate)
vllm/multimodal/utils.py
+17
-3
@@ -1,11 +1,9 @@
1
import base64
2
from functools import lru_cache
3
from io import BytesIO
4
-from typing import List, Optional, Tuple, TypeVar, Union
4
+from typing import Any, List, Optional, Tuple, TypeVar, Union
5
6
-import librosa
6
import numpy as np
8
-import soundfile
7
from PIL import Image
8
9
from vllm.connections import global_http_connection
@@ -73,10 +71,22 @@ async def async_fetch_image(image_url: str,
71
return image.convert(image_mode)
72
73
74
+def try_import_audio_packages() -> Tuple[Any, Any]:
75
+ try:
76
+ import librosa
77
+ import soundfile
78
+ except ImportError:
79
+ raise ImportError(
80
+ "Please install vllm[audio] for audio support.") from None
81
+ return librosa, soundfile
82
+
83
+
84
def fetch_audio(audio_url: str) -> Tuple[np.ndarray, Union[int, float]]:
85
"""
86
Load audio from a URL.
87
"""
88
+ librosa, _ = try_import_audio_packages()
89
+
90
if audio_url.startswith("http"):
91
audio_bytes = global_http_connection.get_bytes(
92
audio_url, timeout=VLLM_AUDIO_FETCH_TIMEOUT)
@@ -95,6 +105,8 @@ async def async_fetch_audio(
105
"""
106
Asynchronously fetch audio from a URL.
107
"""
108
+ librosa, _ = try_import_audio_packages()
109
+
110
if audio_url.startswith("http"):
111
audio_bytes = await global_http_connection.async_get_bytes(
112
audio_url, timeout=VLLM_AUDIO_FETCH_TIMEOUT)
@@ -123,6 +135,8 @@ def encode_audio_base64(
135
sampling_rate: int,
136
) -> str:
137
"""Encode audio as base64."""
138
+ _, soundfile = try_import_audio_packages()
139
+
140
buffered = BytesIO()
141
soundfile.write(buffered, audio, sampling_rate, format="WAV")
142