@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5b86b19954
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Misc] Optional installation of audio related packages (#8063)

Roger Wang committed Sep 1, 2024 at 14:46 UTC 5b86b19954d30acaebb24bc5441b184ae3fcf345
6 files changed +29 -10
requirements-common.txt
+1 -3
@@ -22,9 +22,7 @@ typing_extensions >= 4.10
22 filelock >= 3.10.4 # filelock starts to support `mode` argument from 3.10.4
23 pyzmq
24 msgspec
25 -librosa # Required for audio processing
26 -soundfile # Required for audio processing
25 gguf == 0.9.1
26 importlib_metadata
27 mistral_common >= 1.3.4
30 -pyyaml
\ No newline at end of file
28 +pyyaml
requirements-test.txt
+3 -1
@@ -13,10 +13,12 @@ pytest-shard
13 awscli
14 einops # required for MPT, qwen-vl and Mamba
15 httpx
16 +librosa # required for audio test
17 peft
18 requests
19 ray
20 sentence-transformers # required for embedding
21 +soundfile # required for audio test
22 compressed-tensors==0.4.0 # required for compressed-tensors
23 timm # required for internvl test
24 transformers_stream_generator # required for qwen-vl test
@@ -30,4 +32,4 @@ aiohttp
32
33 # quantization
34 bitsandbytes==0.42.0
33 -buildkite-test-collector==0.1.8
\ No newline at end of file
35 +buildkite-test-collector==0.1.8
setup.py
+1
@@ -501,6 +501,7 @@ setup(
501 ext_modules=ext_modules,
502 extras_require={
503 "tensorizer": ["tensorizer>=2.9.0"],
504 + "audio": ["librosa", "soundfile"] # Required for audio processing
505 },
506 cmdclass={"build_ext": cmake_build_ext} if len(ext_modules) > 0 else {},
507 package_data=package_data,
tests/models/test_ultravox.py
+2 -2
@@ -1,11 +1,9 @@
1 from typing import List, Optional, Tuple, Type
2
3 -import librosa
3 import numpy as np
4 import pytest
5 from transformers import AutoModel, AutoTokenizer, BatchEncoding
6
8 -from vllm.assets.audio import AudioAsset
7 from vllm.sequence import SampleLogprobs
8 from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
9
@@ -21,6 +19,7 @@ AudioTuple = Tuple[np.ndarray, int]
19
20 @pytest.fixture(scope="session")
21 def audio_and_sample_rate():
22 + from vllm.assets.audio import AudioAsset
23 return AudioAsset("mary_had_lamb").audio_and_sample_rate
24
25
@@ -109,6 +108,7 @@ def run_test(
108 dtype=dtype,
109 postprocess_inputs=process,
110 auto_cls=AutoModel) as hf_model:
111 + import librosa
112
113 hf_outputs_per_audio = [
114 hf_model.generate_greedy_logprobs_limit(
vllm/model_executor/models/ultravox.py
+5 -1
@@ -8,7 +8,6 @@ from functools import lru_cache
8 from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
9 TypedDict, Union, cast)
10
11 -import librosa
11 import numpy as np
12 import torch
13 import torch.utils.checkpoint
@@ -107,6 +106,11 @@ def input_mapper_for_ultravox(ctx: InputContext, data: object):
106 feature_extractor = whisper_feature_extractor(ctx)
107
108 if sr != feature_extractor.sampling_rate:
109 + try:
110 + import librosa
111 + except ImportError:
112 + raise ImportError(
113 + "Please install vllm[audio] for audio support.") from None
114 audio = librosa.resample(audio,
115 orig_sr=sr,
116 target_sr=feature_extractor.sampling_rate)
vllm/multimodal/utils.py
+17 -3
@@ -1,11 +1,9 @@
1 import base64
2 from functools import lru_cache
3 from io import BytesIO
4 -from typing import List, Optional, Tuple, TypeVar, Union
4 +from typing import Any, List, Optional, Tuple, TypeVar, Union
5
6 -import librosa
6 import numpy as np
8 -import soundfile
7 from PIL import Image
8
9 from vllm.connections import global_http_connection
@@ -73,10 +71,22 @@ async def async_fetch_image(image_url: str,
71 return image.convert(image_mode)
72
73
74 +def try_import_audio_packages() -> Tuple[Any, Any]:
75 + try:
76 + import librosa
77 + import soundfile
78 + except ImportError:
79 + raise ImportError(
80 + "Please install vllm[audio] for audio support.") from None
81 + return librosa, soundfile
82 +
83 +
84 def fetch_audio(audio_url: str) -> Tuple[np.ndarray, Union[int, float]]:
85 """
86 Load audio from a URL.
87 """
88 + librosa, _ = try_import_audio_packages()
89 +
90 if audio_url.startswith("http"):
91 audio_bytes = global_http_connection.get_bytes(
92 audio_url, timeout=VLLM_AUDIO_FETCH_TIMEOUT)
@@ -95,6 +105,8 @@ async def async_fetch_audio(
105 """
106 Asynchronously fetch audio from a URL.
107 """
108 + librosa, _ = try_import_audio_packages()
109 +
110 if audio_url.startswith("http"):
111 audio_bytes = await global_http_connection.async_get_bytes(
112 audio_url, timeout=VLLM_AUDIO_FETCH_TIMEOUT)
@@ -123,6 +135,8 @@ def encode_audio_base64(
135 sampling_rate: int,
136 ) -> str:
137 """Encode audio as base64."""
138 + _, soundfile = try_import_audio_packages()
139 +
140 buffered = BytesIO()
141 soundfile.write(buffered, audio, sampling_rate, format="WAV")
142