A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Misc] Add placeholder module (#11501)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Dec 26, 2024 at 21:12 UTC
eec906d8114cd786315e49ab7f5a3093d1896880
14 files changed
+143
-100
tests/tensorizer_loader/test_tensorizer.py
+7
-2
@@ -9,7 +9,6 @@ import openai
9
import pytest
10
import torch
11
from huggingface_hub import snapshot_download
12
-from tensorizer import EncryptionParams
12
13
from vllm import SamplingParams
14
from vllm.engine.arg_utils import EngineArgs
@@ -23,12 +22,18 @@ from vllm.model_executor.model_loader.tensorizer import (TensorizerConfig,
22
serialize_vllm_model,
23
tensorize_vllm_model)
24
# yapf: enable
26
-from vllm.utils import import_from_path
25
+from vllm.utils import PlaceholderModule, import_from_path
26
27
from ..conftest import VllmRunner
28
from ..utils import VLLM_PATH, RemoteOpenAIServer
29
from .conftest import retry_until_skip
30
31
+try:
32
+ from tensorizer import EncryptionParams
33
+except ImportError:
34
+ tensorizer = PlaceholderModule("tensorizer") # type: ignore[assignment]
35
+ EncryptionParams = tensorizer.placeholder_attr("EncryptionParams")
36
+
37
EXAMPLES_PATH = VLLM_PATH / "examples"
38
39
prompts = [
vllm/assets/audio.py
+12
-7
@@ -1,11 +1,17 @@
1
from dataclasses import dataclass
2
-from typing import Literal, Tuple
2
+from typing import Literal
3
from urllib.parse import urljoin
4
5
-import librosa
6
-import numpy as np
5
+import numpy.typing as npt
6
8
-from vllm.assets.base import get_vllm_public_assets, vLLM_S3_BUCKET_URL
7
+from vllm.utils import PlaceholderModule
8
+
9
+from .base import VLLM_S3_BUCKET_URL, get_vllm_public_assets
10
+
11
+try:
12
+ import librosa
13
+except ImportError:
14
+ librosa = PlaceholderModule("librosa") # type: ignore[assignment]
15
16
ASSET_DIR = "multimodal_asset"
17
@@ -15,8 +21,7 @@ class AudioAsset:
21
name: Literal["winning_call", "mary_had_lamb"]
22
23
@property
18
- def audio_and_sample_rate(self) -> Tuple[np.ndarray, int]:
19
-
24
+ def audio_and_sample_rate(self) -> tuple[npt.NDArray, int]:
25
audio_path = get_vllm_public_assets(filename=f"{self.name}.ogg",
26
s3_prefix=ASSET_DIR)
27
y, sr = librosa.load(audio_path, sr=None)
@@ -25,4 +30,4 @@ class AudioAsset:
30
31
@property
32
def url(self) -> str:
28
- return urljoin(vLLM_S3_BUCKET_URL, f"{ASSET_DIR}/{self.name}.ogg")
33
+ return urljoin(VLLM_S3_BUCKET_URL, f"{ASSET_DIR}/{self.name}.ogg")
vllm/assets/base.py
+3
-4
@@ -4,9 +4,8 @@ from typing import Optional
4
5
import vllm.envs as envs
6
from vllm.connections import global_http_connection
7
-from vllm.envs import VLLM_IMAGE_FETCH_TIMEOUT
7
9
-vLLM_S3_BUCKET_URL = "https://vllm-public-assets.s3.us-west-2.amazonaws.com"
8
+VLLM_S3_BUCKET_URL = "https://vllm-public-assets.s3.us-west-2.amazonaws.com"
9
10
11
def get_cache_dir() -> Path:
@@ -32,8 +31,8 @@ def get_vllm_public_assets(filename: str,
31
if s3_prefix is not None:
32
filename = s3_prefix + "/" + filename
33
global_http_connection.download_file(
35
- f"{vLLM_S3_BUCKET_URL}/{filename}",
34
+ f"{VLLM_S3_BUCKET_URL}/{filename}",
35
asset_path,
37
- timeout=VLLM_IMAGE_FETCH_TIMEOUT)
36
+ timeout=envs.VLLM_IMAGE_FETCH_TIMEOUT)
37
38
return asset_path
vllm/assets/image.py
+1
-2
@@ -4,7 +4,7 @@ from typing import Literal
4
import torch
5
from PIL import Image
6
7
-from vllm.assets.base import get_vllm_public_assets
7
+from .base import get_vllm_public_assets
8
9
VLM_IMAGES_DIR = "vision_model_images"
10
@@ -15,7 +15,6 @@ class ImageAsset:
15
16
@property
17
def pil_image(self) -> Image.Image:
18
-
18
image_path = get_vllm_public_assets(filename=f"{self.name}.jpg",
19
s3_prefix=VLM_IMAGES_DIR)
20
return Image.open(image_path)
vllm/assets/video.py
+3
-6
@@ -2,13 +2,13 @@ from dataclasses import dataclass
2
from functools import lru_cache
3
from typing import List, Literal
4
5
+import cv2
6
import numpy as np
7
import numpy.typing as npt
8
from huggingface_hub import hf_hub_download
9
from PIL import Image
10
10
-from vllm.multimodal.video import (sample_frames_from_video,
11
- try_import_video_packages)
11
+from vllm.multimodal.video import sample_frames_from_video
12
13
from .base import get_cache_dir
14
@@ -19,7 +19,7 @@ def download_video_asset(filename: str) -> str:
19
Download and open an image from huggingface
20
repo: raushan-testing-hf/videos-test
21
"""
22
- video_directory = get_cache_dir() / "video-eample-data"
22
+ video_directory = get_cache_dir() / "video-example-data"
23
video_directory.mkdir(parents=True, exist_ok=True)
24
25
video_path = video_directory / filename
@@ -35,8 +35,6 @@ def download_video_asset(filename: str) -> str:
35
36
37
def video_to_ndarrays(path: str, num_frames: int = -1) -> npt.NDArray:
38
- cv2, _ = try_import_video_packages()
39
-
38
cap = cv2.VideoCapture(path)
39
if not cap.isOpened():
40
raise ValueError(f"Could not open video file {path}")
@@ -59,7 +57,6 @@ def video_to_ndarrays(path: str, num_frames: int = -1) -> npt.NDArray:
57
58
def video_to_pil_images_list(path: str,
59
num_frames: int = -1) -> List[Image.Image]:
62
- cv2, _ = try_import_video_packages()
60
frames = video_to_ndarrays(path, num_frames)
61
return [
62
Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
vllm/config.py
+1
-9
@@ -29,6 +29,7 @@ from vllm.transformers_utils.config import (
29
get_hf_text_config, get_pooling_config,
30
get_sentence_transformer_tokenizer_config, is_encoder_decoder,
31
try_get_generation_config, uses_mrope)
32
+from vllm.transformers_utils.s3_utils import S3Model
33
from vllm.transformers_utils.utils import is_s3
34
from vllm.utils import (GiB_bytes, LayerBlockType, cuda_device_count_stateless,
35
get_cpu_memory, print_warning_once, random_uuid,
@@ -372,15 +373,6 @@ class ModelConfig:
373
374
"""
375
if is_s3(model) or is_s3(tokenizer):
375
- try:
376
- from vllm.transformers_utils.s3_utils import S3Model
377
- except ImportError as err:
378
- raise ImportError(
379
- "Please install Run:ai optional dependency "
380
- "to use the S3 capabilities. "
381
- "You can install it with: pip install vllm[runai]"
382
- ) from err
383
-
376
if is_s3(model):
377
self.s3_model = S3Model()
378
self.s3_model.pull_files(model, allow_pattern=["*config.json"])
vllm/model_executor/model_loader/loader.py
+1
-10
@@ -48,6 +48,7 @@ from vllm.model_executor.model_loader.weight_utils import (
48
runai_safetensors_weights_iterator, safetensors_weights_iterator)
49
from vllm.model_executor.utils import set_weight_attrs
50
from vllm.platforms import current_platform
51
+from vllm.transformers_utils.s3_utils import glob as s3_glob
52
from vllm.transformers_utils.utils import is_s3
53
from vllm.utils import is_pin_memory_available
54
@@ -1269,16 +1270,6 @@ class RunaiModelStreamerLoader(BaseModelLoader):
1270
1271
If the model is not local, it will be downloaded."""
1272
is_s3_path = is_s3(model_name_or_path)
1272
- if is_s3_path:
1273
- try:
1274
- from vllm.transformers_utils.s3_utils import glob as s3_glob
1275
- except ImportError as err:
1276
- raise ImportError(
1277
- "Please install Run:ai optional dependency "
1278
- "to use the S3 capabilities. "
1279
- "You can install it with: pip install vllm[runai]"
1280
- ) from err
1281
-
1273
is_local = os.path.isdir(model_name_or_path)
1274
safetensors_pattern = "*.safetensors"
1275
index_file = SAFE_WEIGHTS_INDEX_NAME
vllm/model_executor/model_loader/tensorizer.py
+14
-11
@@ -19,9 +19,7 @@ from vllm.engine.llm_engine import LLMEngine
19
from vllm.logger import init_logger
20
from vllm.model_executor.layers.vocab_parallel_embedding import (
21
VocabParallelEmbedding)
22
-from vllm.utils import FlexibleArgumentParser
23
-
24
-tensorizer_error_msg = None
22
+from vllm.utils import FlexibleArgumentParser, PlaceholderModule
23
24
try:
25
from tensorizer import (DecryptionParams, EncryptionParams,
@@ -34,8 +32,19 @@ try:
32
open_stream,
33
mode=mode,
34
) for mode in ("rb", "wb+"))
37
-except ImportError as e:
38
- tensorizer_error_msg = str(e)
35
+except ImportError:
36
+ tensorizer = PlaceholderModule("tensorizer")
37
+ DecryptionParams = tensorizer.placeholder_attr("DecryptionParams")
38
+ EncryptionParams = tensorizer.placeholder_attr("EncryptionParams")
39
+ TensorDeserializer = tensorizer.placeholder_attr("TensorDeserializer")
40
+ TensorSerializer = tensorizer.placeholder_attr("TensorSerializer")
41
+ open_stream = tensorizer.placeholder_attr("stream_io.open_stream")
42
+ convert_bytes = tensorizer.placeholder_attr("utils.convert_bytes")
43
+ get_mem_usage = tensorizer.placeholder_attr("utils.get_mem_usage")
44
+ no_init_or_tensor = tensorizer.placeholder_attr("utils.no_init_or_tensor")
45
+
46
+ _read_stream = tensorizer.placeholder_attr("_read_stream")
47
+ _write_stream = tensorizer.placeholder_attr("_write_stream")
48
49
__all__ = [
50
'EncryptionParams', 'DecryptionParams', 'TensorDeserializer',
@@ -267,12 +276,6 @@ class TensorizerAgent:
276
"""
277
278
def __init__(self, tensorizer_config: TensorizerConfig, vllm_config):
270
- if tensorizer_error_msg is not None:
271
- raise ImportError(
272
- "Tensorizer is not installed. Please install tensorizer "
273
- "to use this feature with `pip install vllm[tensorizer]`. "
274
- "Error message: {}".format(tensorizer_error_msg))
275
-
279
self.tensorizer_config = tensorizer_config
280
self.tensorizer_args = (
281
self.tensorizer_config._construct_tensorizer_args())
vllm/model_executor/model_loader/weight_utils.py
+9
-8
@@ -25,7 +25,15 @@ from vllm.model_executor.layers.quantization import (QuantizationConfig,
25
get_quantization_config)
26
from vllm.model_executor.layers.quantization.schema import QuantParamSchema
27
from vllm.platforms import current_platform
28
-from vllm.utils import print_warning_once
28
+from vllm.utils import PlaceholderModule, print_warning_once
29
+
30
+try:
31
+ from runai_model_streamer import SafetensorsStreamer
32
+except ImportError:
33
+ runai_model_streamer = PlaceholderModule(
34
+ "runai_model_streamer") # type: ignore[assignment]
35
+ SafetensorsStreamer = runai_model_streamer.placeholder_attr(
36
+ "SafetensorsStreamer")
37
38
logger = init_logger(__name__)
39
@@ -414,13 +422,6 @@ def runai_safetensors_weights_iterator(
422
hf_weights_files: List[str]
423
) -> Generator[Tuple[str, torch.Tensor], None, None]:
424
"""Iterate over the weights in the model safetensor files."""
417
- try:
418
- from runai_model_streamer import SafetensorsStreamer
419
- except ImportError as err:
420
- raise ImportError(
421
- "Please install Run:ai optional dependency."
422
- "You can install it with: pip install vllm[runai]") from err
423
-
425
enable_tqdm = not torch.distributed.is_initialized(
426
) or torch.distributed.get_rank() == 0
427
with SafetensorsStreamer() as streamer:
vllm/multimodal/audio.py
+6
-18
@@ -1,13 +1,17 @@
1
-from typing import Any
2
-
1
import numpy as np
2
import numpy.typing as npt
3
4
from vllm.inputs.registry import InputContext
5
+from vllm.utils import PlaceholderModule
6
7
from .base import MultiModalPlugin
8
from .inputs import AudioItem, MultiModalData, MultiModalKwargs
9
10
+try:
11
+ import librosa
12
+except ImportError:
13
+ librosa = PlaceholderModule("librosa") # type: ignore[assignment]
14
+
15
16
class AudioPlugin(MultiModalPlugin):
17
"""Plugin for audio data."""
@@ -28,26 +32,10 @@ class AudioPlugin(MultiModalPlugin):
32
"There is no default maximum multimodal tokens")
33
34
31
-def try_import_audio_packages() -> tuple[Any, Any]:
32
- try:
33
- import librosa
34
- import soundfile
35
- except ImportError as exc:
36
- raise ImportError(
37
- "Please install vllm[audio] for audio support.") from exc
38
- return librosa, soundfile
39
-
40
-
35
def resample_audio(
36
audio: npt.NDArray[np.floating],
37
*,
38
orig_sr: float,
39
target_sr: float,
40
) -> npt.NDArray[np.floating]:
47
- try:
48
- import librosa
49
- except ImportError as exc:
50
- msg = "Please install vllm[audio] for audio support."
51
- raise ImportError(msg) from exc
52
-
41
return librosa.resample(audio, orig_sr=orig_sr, target_sr=target_sr)
vllm/multimodal/utils.py
+16
-10
@@ -13,10 +13,24 @@ import vllm.envs as envs
13
from vllm.connections import global_http_connection
14
from vllm.logger import init_logger
15
from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer
16
+from vllm.utils import PlaceholderModule
17
17
-from .audio import try_import_audio_packages
18
from .inputs import MultiModalDataDict, PlaceholderRange
19
-from .video import try_import_video_packages
19
+
20
+try:
21
+ import decord
22
+except ImportError:
23
+ decord = PlaceholderModule("decord") # type: ignore[assignment]
24
+
25
+try:
26
+ import librosa
27
+except ImportError:
28
+ librosa = PlaceholderModule("librosa") # type: ignore[assignment]
29
+
30
+try:
31
+ import soundfile
32
+except ImportError:
33
+ soundfile = PlaceholderModule("soundfile") # type: ignore[assignment]
34
35
logger = init_logger(__name__)
36
@@ -128,8 +142,6 @@ async def async_fetch_image(image_url: str,
142
143
144
def _load_video_from_bytes(b: bytes, num_frames: int = 32) -> npt.NDArray:
131
- _, decord = try_import_video_packages()
132
-
145
video_path = BytesIO(b)
146
vr = decord.VideoReader(video_path, num_threads=1)
147
total_frame_num = len(vr)
@@ -204,8 +216,6 @@ def fetch_audio(audio_url: str) -> Tuple[np.ndarray, Union[int, float]]:
216
"""
217
Load audio from a URL.
218
"""
207
- librosa, _ = try_import_audio_packages()
208
-
219
if audio_url.startswith("http"):
220
audio_bytes = global_http_connection.get_bytes(
221
audio_url,
@@ -226,8 +236,6 @@ async def async_fetch_audio(
236
"""
237
Asynchronously fetch audio from a URL.
238
"""
229
- librosa, _ = try_import_audio_packages()
230
-
239
if audio_url.startswith("http"):
240
audio_bytes = await global_http_connection.async_get_bytes(
241
audio_url,
@@ -286,8 +294,6 @@ def encode_audio_base64(
294
sampling_rate: int,
295
) -> str:
296
"""Encode audio as base64."""
289
- _, soundfile = try_import_audio_packages()
290
-
297
buffered = BytesIO()
298
soundfile.write(buffered, audio, sampling_rate, format="WAV")
299
vllm/multimodal/video.py
+1
-12
@@ -1,6 +1,7 @@
1
from functools import lru_cache
2
from typing import TYPE_CHECKING, Any, Dict, Optional
3
4
+import cv2
5
import numpy as np
6
import numpy.typing as npt
7
@@ -78,19 +79,7 @@ class VideoPlugin(ImagePlugin):
79
return 4096
80
81
81
-def try_import_video_packages() -> tuple[Any, Any]:
82
- try:
83
- import cv2
84
- import decord
85
- except ImportError as exc:
86
- raise ImportError(
87
- "Please install vllm[video] for video support.") from exc
88
- return cv2, decord
89
-
90
-
82
def resize_video(frames: npt.NDArray, size: tuple[int, int]) -> npt.NDArray:
92
- cv2, _ = try_import_video_packages()
93
-
83
num_frames, _, _, channels = frames.shape
84
new_height, new_width = size
85
resized_frames = np.empty((num_frames, new_height, new_width, channels),
vllm/transformers_utils/s3_utils.py
+6
-1
@@ -6,7 +6,12 @@ import tempfile
6
from pathlib import Path
7
from typing import Optional
8
9
-import boto3
9
+from vllm.utils import PlaceholderModule
10
+
11
+try:
12
+ import boto3
13
+except ImportError:
14
+ boto3 = PlaceholderModule("boto3") # type: ignore[assignment]
15
16
17
def _filter_allow(paths: list[str], patterns: list[str]) -> list[str]:
vllm/utils.py
+63
@@ -6,10 +6,12 @@ import datetime
6
import enum
7
import gc
8
import getpass
9
+import importlib.metadata
10
import importlib.util
11
import inspect
12
import ipaddress
13
import os
14
+import re
15
import signal
16
import socket
17
import subprocess
@@ -1550,6 +1552,67 @@ def import_from_path(module_name: str, file_path: Union[str, os.PathLike]):
1552
return module
1553
1554
1555
+@lru_cache(maxsize=None)
1556
+def get_vllm_optional_dependencies():
1557
+ metadata = importlib.metadata.metadata("vllm")
1558
+ requirements = metadata.get_all("Requires-Dist", [])
1559
+ extras = metadata.get_all("Provides-Extra", [])
1560
+
1561
+ return {
1562
+ extra: [
1563
+ re.split(r";|>=|<=|==", req)[0] for req in requirements
1564
+ if req.endswith(f'extra == "{extra}"')
1565
+ ]
1566
+ for extra in extras
1567
+ }
1568
+
1569
+
1570
+@dataclass(frozen=True)
1571
+class PlaceholderModule:
1572
+ """
1573
+ A placeholder object to use when a module does not exist.
1574
+
1575
+ This enables more informative errors when trying to access attributes
1576
+ of a module that does not exists.
1577
+ """
1578
+ name: str
1579
+
1580
+ def placeholder_attr(self, attr_path: str):
1581
+ return _PlaceholderModuleAttr(self, attr_path)
1582
+
1583
+ def __getattr__(self, key: str):
1584
+ name = self.name
1585
+
1586
+ try:
1587
+ importlib.import_module(self.name)
1588
+ except ImportError as exc:
1589
+ for extra, names in get_vllm_optional_dependencies().items():
1590
+ if name in names:
1591
+ msg = f"Please install vllm[{extra}] for {extra} support"
1592
+ raise ImportError(msg) from exc
1593
+
1594
+ raise exc
1595
+
1596
+ raise AssertionError("PlaceholderModule should not be used "
1597
+ "when the original module can be imported")
1598
+
1599
+
1600
+@dataclass(frozen=True)
1601
+class _PlaceholderModuleAttr:
1602
+ module: PlaceholderModule
1603
+ attr_path: str
1604
+
1605
+ def placeholder_attr(self, attr_path: str):
1606
+ return _PlaceholderModuleAttr(self.module,
1607
+ f"{self.attr_path}.{attr_path}")
1608
+
1609
+ def __getattr__(self, key: str):
1610
+ getattr(self.module, f"{self.attr_path}.{key}")
1611
+
1612
+ raise AssertionError("PlaceholderModule should not be used "
1613
+ "when the original module can be imported")
1614
+
1615
+
1616
# create a library to hold the custom op
1617
vllm_lib = Library("vllm", "FRAGMENT") # noqa
1618