@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / eec906d811
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Misc] Add placeholder module (#11501)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 26, 2024 at 21:12 UTC eec906d8114cd786315e49ab7f5a3093d1896880
14 files changed +143 -100
tests/tensorizer_loader/test_tensorizer.py
+7 -2
@@ -9,7 +9,6 @@ import openai
9 import pytest
10 import torch
11 from huggingface_hub import snapshot_download
12 -from tensorizer import EncryptionParams
12
13 from vllm import SamplingParams
14 from vllm.engine.arg_utils import EngineArgs
@@ -23,12 +22,18 @@ from vllm.model_executor.model_loader.tensorizer import (TensorizerConfig,
22 serialize_vllm_model,
23 tensorize_vllm_model)
24 # yapf: enable
26 -from vllm.utils import import_from_path
25 +from vllm.utils import PlaceholderModule, import_from_path
26
27 from ..conftest import VllmRunner
28 from ..utils import VLLM_PATH, RemoteOpenAIServer
29 from .conftest import retry_until_skip
30
31 +try:
32 + from tensorizer import EncryptionParams
33 +except ImportError:
34 + tensorizer = PlaceholderModule("tensorizer") # type: ignore[assignment]
35 + EncryptionParams = tensorizer.placeholder_attr("EncryptionParams")
36 +
37 EXAMPLES_PATH = VLLM_PATH / "examples"
38
39 prompts = [
vllm/assets/audio.py
+12 -7
@@ -1,11 +1,17 @@
1 from dataclasses import dataclass
2 -from typing import Literal, Tuple
2 +from typing import Literal
3 from urllib.parse import urljoin
4
5 -import librosa
6 -import numpy as np
5 +import numpy.typing as npt
6
8 -from vllm.assets.base import get_vllm_public_assets, vLLM_S3_BUCKET_URL
7 +from vllm.utils import PlaceholderModule
8 +
9 +from .base import VLLM_S3_BUCKET_URL, get_vllm_public_assets
10 +
11 +try:
12 + import librosa
13 +except ImportError:
14 + librosa = PlaceholderModule("librosa") # type: ignore[assignment]
15
16 ASSET_DIR = "multimodal_asset"
17
@@ -15,8 +21,7 @@ class AudioAsset:
21 name: Literal["winning_call", "mary_had_lamb"]
22
23 @property
18 - def audio_and_sample_rate(self) -> Tuple[np.ndarray, int]:
19 -
24 + def audio_and_sample_rate(self) -> tuple[npt.NDArray, int]:
25 audio_path = get_vllm_public_assets(filename=f"{self.name}.ogg",
26 s3_prefix=ASSET_DIR)
27 y, sr = librosa.load(audio_path, sr=None)
@@ -25,4 +30,4 @@ class AudioAsset:
30
31 @property
32 def url(self) -> str:
28 - return urljoin(vLLM_S3_BUCKET_URL, f"{ASSET_DIR}/{self.name}.ogg")
33 + return urljoin(VLLM_S3_BUCKET_URL, f"{ASSET_DIR}/{self.name}.ogg")
vllm/assets/base.py
+3 -4
@@ -4,9 +4,8 @@ from typing import Optional
4
5 import vllm.envs as envs
6 from vllm.connections import global_http_connection
7 -from vllm.envs import VLLM_IMAGE_FETCH_TIMEOUT
7
9 -vLLM_S3_BUCKET_URL = "https://vllm-public-assets.s3.us-west-2.amazonaws.com"
8 +VLLM_S3_BUCKET_URL = "https://vllm-public-assets.s3.us-west-2.amazonaws.com"
9
10
11 def get_cache_dir() -> Path:
@@ -32,8 +31,8 @@ def get_vllm_public_assets(filename: str,
31 if s3_prefix is not None:
32 filename = s3_prefix + "/" + filename
33 global_http_connection.download_file(
35 - f"{vLLM_S3_BUCKET_URL}/{filename}",
34 + f"{VLLM_S3_BUCKET_URL}/{filename}",
35 asset_path,
37 - timeout=VLLM_IMAGE_FETCH_TIMEOUT)
36 + timeout=envs.VLLM_IMAGE_FETCH_TIMEOUT)
37
38 return asset_path
vllm/assets/image.py
+1 -2
@@ -4,7 +4,7 @@ from typing import Literal
4 import torch
5 from PIL import Image
6
7 -from vllm.assets.base import get_vllm_public_assets
7 +from .base import get_vllm_public_assets
8
9 VLM_IMAGES_DIR = "vision_model_images"
10
@@ -15,7 +15,6 @@ class ImageAsset:
15
16 @property
17 def pil_image(self) -> Image.Image:
18 -
18 image_path = get_vllm_public_assets(filename=f"{self.name}.jpg",
19 s3_prefix=VLM_IMAGES_DIR)
20 return Image.open(image_path)
vllm/assets/video.py
+3 -6
@@ -2,13 +2,13 @@ from dataclasses import dataclass
2 from functools import lru_cache
3 from typing import List, Literal
4
5 +import cv2
6 import numpy as np
7 import numpy.typing as npt
8 from huggingface_hub import hf_hub_download
9 from PIL import Image
10
10 -from vllm.multimodal.video import (sample_frames_from_video,
11 - try_import_video_packages)
11 +from vllm.multimodal.video import sample_frames_from_video
12
13 from .base import get_cache_dir
14
@@ -19,7 +19,7 @@ def download_video_asset(filename: str) -> str:
19 Download and open an image from huggingface
20 repo: raushan-testing-hf/videos-test
21 """
22 - video_directory = get_cache_dir() / "video-eample-data"
22 + video_directory = get_cache_dir() / "video-example-data"
23 video_directory.mkdir(parents=True, exist_ok=True)
24
25 video_path = video_directory / filename
@@ -35,8 +35,6 @@ def download_video_asset(filename: str) -> str:
35
36
37 def video_to_ndarrays(path: str, num_frames: int = -1) -> npt.NDArray:
38 - cv2, _ = try_import_video_packages()
39 -
38 cap = cv2.VideoCapture(path)
39 if not cap.isOpened():
40 raise ValueError(f"Could not open video file {path}")
@@ -59,7 +57,6 @@ def video_to_ndarrays(path: str, num_frames: int = -1) -> npt.NDArray:
57
58 def video_to_pil_images_list(path: str,
59 num_frames: int = -1) -> List[Image.Image]:
62 - cv2, _ = try_import_video_packages()
60 frames = video_to_ndarrays(path, num_frames)
61 return [
62 Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
vllm/config.py
+1 -9
@@ -29,6 +29,7 @@ from vllm.transformers_utils.config import (
29 get_hf_text_config, get_pooling_config,
30 get_sentence_transformer_tokenizer_config, is_encoder_decoder,
31 try_get_generation_config, uses_mrope)
32 +from vllm.transformers_utils.s3_utils import S3Model
33 from vllm.transformers_utils.utils import is_s3
34 from vllm.utils import (GiB_bytes, LayerBlockType, cuda_device_count_stateless,
35 get_cpu_memory, print_warning_once, random_uuid,
@@ -372,15 +373,6 @@ class ModelConfig:
373
374 """
375 if is_s3(model) or is_s3(tokenizer):
375 - try:
376 - from vllm.transformers_utils.s3_utils import S3Model
377 - except ImportError as err:
378 - raise ImportError(
379 - "Please install Run:ai optional dependency "
380 - "to use the S3 capabilities. "
381 - "You can install it with: pip install vllm[runai]"
382 - ) from err
383 -
376 if is_s3(model):
377 self.s3_model = S3Model()
378 self.s3_model.pull_files(model, allow_pattern=["*config.json"])
vllm/model_executor/model_loader/loader.py
+1 -10
@@ -48,6 +48,7 @@ from vllm.model_executor.model_loader.weight_utils import (
48 runai_safetensors_weights_iterator, safetensors_weights_iterator)
49 from vllm.model_executor.utils import set_weight_attrs
50 from vllm.platforms import current_platform
51 +from vllm.transformers_utils.s3_utils import glob as s3_glob
52 from vllm.transformers_utils.utils import is_s3
53 from vllm.utils import is_pin_memory_available
54
@@ -1269,16 +1270,6 @@ class RunaiModelStreamerLoader(BaseModelLoader):
1270
1271 If the model is not local, it will be downloaded."""
1272 is_s3_path = is_s3(model_name_or_path)
1272 - if is_s3_path:
1273 - try:
1274 - from vllm.transformers_utils.s3_utils import glob as s3_glob
1275 - except ImportError as err:
1276 - raise ImportError(
1277 - "Please install Run:ai optional dependency "
1278 - "to use the S3 capabilities. "
1279 - "You can install it with: pip install vllm[runai]"
1280 - ) from err
1281 -
1273 is_local = os.path.isdir(model_name_or_path)
1274 safetensors_pattern = "*.safetensors"
1275 index_file = SAFE_WEIGHTS_INDEX_NAME
vllm/model_executor/model_loader/tensorizer.py
+14 -11
@@ -19,9 +19,7 @@ from vllm.engine.llm_engine import LLMEngine
19 from vllm.logger import init_logger
20 from vllm.model_executor.layers.vocab_parallel_embedding import (
21 VocabParallelEmbedding)
22 -from vllm.utils import FlexibleArgumentParser
23 -
24 -tensorizer_error_msg = None
22 +from vllm.utils import FlexibleArgumentParser, PlaceholderModule
23
24 try:
25 from tensorizer import (DecryptionParams, EncryptionParams,
@@ -34,8 +32,19 @@ try:
32 open_stream,
33 mode=mode,
34 ) for mode in ("rb", "wb+"))
37 -except ImportError as e:
38 - tensorizer_error_msg = str(e)
35 +except ImportError:
36 + tensorizer = PlaceholderModule("tensorizer")
37 + DecryptionParams = tensorizer.placeholder_attr("DecryptionParams")
38 + EncryptionParams = tensorizer.placeholder_attr("EncryptionParams")
39 + TensorDeserializer = tensorizer.placeholder_attr("TensorDeserializer")
40 + TensorSerializer = tensorizer.placeholder_attr("TensorSerializer")
41 + open_stream = tensorizer.placeholder_attr("stream_io.open_stream")
42 + convert_bytes = tensorizer.placeholder_attr("utils.convert_bytes")
43 + get_mem_usage = tensorizer.placeholder_attr("utils.get_mem_usage")
44 + no_init_or_tensor = tensorizer.placeholder_attr("utils.no_init_or_tensor")
45 +
46 + _read_stream = tensorizer.placeholder_attr("_read_stream")
47 + _write_stream = tensorizer.placeholder_attr("_write_stream")
48
49 __all__ = [
50 'EncryptionParams', 'DecryptionParams', 'TensorDeserializer',
@@ -267,12 +276,6 @@ class TensorizerAgent:
276 """
277
278 def __init__(self, tensorizer_config: TensorizerConfig, vllm_config):
270 - if tensorizer_error_msg is not None:
271 - raise ImportError(
272 - "Tensorizer is not installed. Please install tensorizer "
273 - "to use this feature with `pip install vllm[tensorizer]`. "
274 - "Error message: {}".format(tensorizer_error_msg))
275 -
279 self.tensorizer_config = tensorizer_config
280 self.tensorizer_args = (
281 self.tensorizer_config._construct_tensorizer_args())
vllm/model_executor/model_loader/weight_utils.py
+9 -8
@@ -25,7 +25,15 @@ from vllm.model_executor.layers.quantization import (QuantizationConfig,
25 get_quantization_config)
26 from vllm.model_executor.layers.quantization.schema import QuantParamSchema
27 from vllm.platforms import current_platform
28 -from vllm.utils import print_warning_once
28 +from vllm.utils import PlaceholderModule, print_warning_once
29 +
30 +try:
31 + from runai_model_streamer import SafetensorsStreamer
32 +except ImportError:
33 + runai_model_streamer = PlaceholderModule(
34 + "runai_model_streamer") # type: ignore[assignment]
35 + SafetensorsStreamer = runai_model_streamer.placeholder_attr(
36 + "SafetensorsStreamer")
37
38 logger = init_logger(__name__)
39
@@ -414,13 +422,6 @@ def runai_safetensors_weights_iterator(
422 hf_weights_files: List[str]
423 ) -> Generator[Tuple[str, torch.Tensor], None, None]:
424 """Iterate over the weights in the model safetensor files."""
417 - try:
418 - from runai_model_streamer import SafetensorsStreamer
419 - except ImportError as err:
420 - raise ImportError(
421 - "Please install Run:ai optional dependency."
422 - "You can install it with: pip install vllm[runai]") from err
423 -
425 enable_tqdm = not torch.distributed.is_initialized(
426 ) or torch.distributed.get_rank() == 0
427 with SafetensorsStreamer() as streamer:
vllm/multimodal/audio.py
+6 -18
@@ -1,13 +1,17 @@
1 -from typing import Any
2 -
1 import numpy as np
2 import numpy.typing as npt
3
4 from vllm.inputs.registry import InputContext
5 +from vllm.utils import PlaceholderModule
6
7 from .base import MultiModalPlugin
8 from .inputs import AudioItem, MultiModalData, MultiModalKwargs
9
10 +try:
11 + import librosa
12 +except ImportError:
13 + librosa = PlaceholderModule("librosa") # type: ignore[assignment]
14 +
15
16 class AudioPlugin(MultiModalPlugin):
17 """Plugin for audio data."""
@@ -28,26 +32,10 @@ class AudioPlugin(MultiModalPlugin):
32 "There is no default maximum multimodal tokens")
33
34
31 -def try_import_audio_packages() -> tuple[Any, Any]:
32 - try:
33 - import librosa
34 - import soundfile
35 - except ImportError as exc:
36 - raise ImportError(
37 - "Please install vllm[audio] for audio support.") from exc
38 - return librosa, soundfile
39 -
40 -
35 def resample_audio(
36 audio: npt.NDArray[np.floating],
37 *,
38 orig_sr: float,
39 target_sr: float,
40 ) -> npt.NDArray[np.floating]:
47 - try:
48 - import librosa
49 - except ImportError as exc:
50 - msg = "Please install vllm[audio] for audio support."
51 - raise ImportError(msg) from exc
52 -
41 return librosa.resample(audio, orig_sr=orig_sr, target_sr=target_sr)
vllm/multimodal/utils.py
+16 -10
@@ -13,10 +13,24 @@ import vllm.envs as envs
13 from vllm.connections import global_http_connection
14 from vllm.logger import init_logger
15 from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer
16 +from vllm.utils import PlaceholderModule
17
17 -from .audio import try_import_audio_packages
18 from .inputs import MultiModalDataDict, PlaceholderRange
19 -from .video import try_import_video_packages
19 +
20 +try:
21 + import decord
22 +except ImportError:
23 + decord = PlaceholderModule("decord") # type: ignore[assignment]
24 +
25 +try:
26 + import librosa
27 +except ImportError:
28 + librosa = PlaceholderModule("librosa") # type: ignore[assignment]
29 +
30 +try:
31 + import soundfile
32 +except ImportError:
33 + soundfile = PlaceholderModule("soundfile") # type: ignore[assignment]
34
35 logger = init_logger(__name__)
36
@@ -128,8 +142,6 @@ async def async_fetch_image(image_url: str,
142
143
144 def _load_video_from_bytes(b: bytes, num_frames: int = 32) -> npt.NDArray:
131 - _, decord = try_import_video_packages()
132 -
145 video_path = BytesIO(b)
146 vr = decord.VideoReader(video_path, num_threads=1)
147 total_frame_num = len(vr)
@@ -204,8 +216,6 @@ def fetch_audio(audio_url: str) -> Tuple[np.ndarray, Union[int, float]]:
216 """
217 Load audio from a URL.
218 """
207 - librosa, _ = try_import_audio_packages()
208 -
219 if audio_url.startswith("http"):
220 audio_bytes = global_http_connection.get_bytes(
221 audio_url,
@@ -226,8 +236,6 @@ async def async_fetch_audio(
236 """
237 Asynchronously fetch audio from a URL.
238 """
229 - librosa, _ = try_import_audio_packages()
230 -
239 if audio_url.startswith("http"):
240 audio_bytes = await global_http_connection.async_get_bytes(
241 audio_url,
@@ -286,8 +294,6 @@ def encode_audio_base64(
294 sampling_rate: int,
295 ) -> str:
296 """Encode audio as base64."""
289 - _, soundfile = try_import_audio_packages()
290 -
297 buffered = BytesIO()
298 soundfile.write(buffered, audio, sampling_rate, format="WAV")
299
vllm/multimodal/video.py
+1 -12
@@ -1,6 +1,7 @@
1 from functools import lru_cache
2 from typing import TYPE_CHECKING, Any, Dict, Optional
3
4 +import cv2
5 import numpy as np
6 import numpy.typing as npt
7
@@ -78,19 +79,7 @@ class VideoPlugin(ImagePlugin):
79 return 4096
80
81
81 -def try_import_video_packages() -> tuple[Any, Any]:
82 - try:
83 - import cv2
84 - import decord
85 - except ImportError as exc:
86 - raise ImportError(
87 - "Please install vllm[video] for video support.") from exc
88 - return cv2, decord
89 -
90 -
82 def resize_video(frames: npt.NDArray, size: tuple[int, int]) -> npt.NDArray:
92 - cv2, _ = try_import_video_packages()
93 -
83 num_frames, _, _, channels = frames.shape
84 new_height, new_width = size
85 resized_frames = np.empty((num_frames, new_height, new_width, channels),
vllm/transformers_utils/s3_utils.py
+6 -1
@@ -6,7 +6,12 @@ import tempfile
6 from pathlib import Path
7 from typing import Optional
8
9 -import boto3
9 +from vllm.utils import PlaceholderModule
10 +
11 +try:
12 + import boto3
13 +except ImportError:
14 + boto3 = PlaceholderModule("boto3") # type: ignore[assignment]
15
16
17 def _filter_allow(paths: list[str], patterns: list[str]) -> list[str]:
vllm/utils.py
+63
@@ -6,10 +6,12 @@ import datetime
6 import enum
7 import gc
8 import getpass
9 +import importlib.metadata
10 import importlib.util
11 import inspect
12 import ipaddress
13 import os
14 +import re
15 import signal
16 import socket
17 import subprocess
@@ -1550,6 +1552,67 @@ def import_from_path(module_name: str, file_path: Union[str, os.PathLike]):
1552 return module
1553
1554
1555 +@lru_cache(maxsize=None)
1556 +def get_vllm_optional_dependencies():
1557 + metadata = importlib.metadata.metadata("vllm")
1558 + requirements = metadata.get_all("Requires-Dist", [])
1559 + extras = metadata.get_all("Provides-Extra", [])
1560 +
1561 + return {
1562 + extra: [
1563 + re.split(r";|>=|<=|==", req)[0] for req in requirements
1564 + if req.endswith(f'extra == "{extra}"')
1565 + ]
1566 + for extra in extras
1567 + }
1568 +
1569 +
1570 +@dataclass(frozen=True)
1571 +class PlaceholderModule:
1572 + """
1573 + A placeholder object to use when a module does not exist.
1574 +
1575 + This enables more informative errors when trying to access attributes
1576 + of a module that does not exists.
1577 + """
1578 + name: str
1579 +
1580 + def placeholder_attr(self, attr_path: str):
1581 + return _PlaceholderModuleAttr(self, attr_path)
1582 +
1583 + def __getattr__(self, key: str):
1584 + name = self.name
1585 +
1586 + try:
1587 + importlib.import_module(self.name)
1588 + except ImportError as exc:
1589 + for extra, names in get_vllm_optional_dependencies().items():
1590 + if name in names:
1591 + msg = f"Please install vllm[{extra}] for {extra} support"
1592 + raise ImportError(msg) from exc
1593 +
1594 + raise exc
1595 +
1596 + raise AssertionError("PlaceholderModule should not be used "
1597 + "when the original module can be imported")
1598 +
1599 +
1600 +@dataclass(frozen=True)
1601 +class _PlaceholderModuleAttr:
1602 + module: PlaceholderModule
1603 + attr_path: str
1604 +
1605 + def placeholder_attr(self, attr_path: str):
1606 + return _PlaceholderModuleAttr(self.module,
1607 + f"{self.attr_path}.{attr_path}")
1608 +
1609 + def __getattr__(self, key: str):
1610 + getattr(self.module, f"{self.attr_path}.{key}")
1611 +
1612 + raise AssertionError("PlaceholderModule should not be used "
1613 + "when the original module can be imported")
1614 +
1615 +
1616 # create a library to hold the custom op
1617 vllm_lib = Library("vllm", "FRAGMENT") # noqa
1618