A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[CI/Build] Minor refactoring for vLLM assets (#7407)
Roger Wang committed
Aug 11, 2024 at 19:41 UTC
86ab567bae0698425095e28cce67e9a31a261b72
2 files changed
+35
-26
vllm/assets/base.py
+29
-1
@@ -1,11 +1,39 @@
1
+from functools import lru_cache
2
from pathlib import Path
3
+from typing import Optional
4
5
import vllm.envs as envs
6
+from vllm.connections import global_http_connection
7
+from vllm.envs import VLLM_IMAGE_FETCH_TIMEOUT
8
9
+vLLM_S3_BUCKET_URL = "https://vllm-public-assets.s3.us-west-2.amazonaws.com"
10
6
-def get_cache_dir():
11
+
12
+def get_cache_dir() -> Path:
13
"""Get the path to the cache for storing downloaded assets."""
14
path = Path(envs.VLLM_ASSETS_CACHE)
15
path.mkdir(parents=True, exist_ok=True)
16
17
return path
18
+
19
+
20
+@lru_cache
21
+def get_vllm_public_assets(filename: str,
22
+ s3_prefix: Optional[str] = None) -> Path:
23
+ """
24
+ Download an asset file from ``s3://vllm-public-assets``
25
+ and return the path to the downloaded file.
26
+ """
27
+ asset_directory = get_cache_dir() / "vllm_public_assets"
28
+ asset_directory.mkdir(parents=True, exist_ok=True)
29
+
30
+ asset_path = asset_directory / filename
31
+ if not asset_path.exists():
32
+ if s3_prefix is not None:
33
+ filename = s3_prefix + "/" + filename
34
+ global_http_connection.download_file(
35
+ f"{vLLM_S3_BUCKET_URL}/{filename}",
36
+ asset_path,
37
+ timeout=VLLM_IMAGE_FETCH_TIMEOUT)
38
+
39
+ return asset_path
vllm/assets/image.py
+6
-25
@@ -1,33 +1,11 @@
1
from dataclasses import dataclass
2
-from functools import lru_cache
2
from typing import Literal
3
4
from PIL import Image
5
7
-from vllm.connections import global_http_connection
8
-from vllm.envs import VLLM_IMAGE_FETCH_TIMEOUT
6
+from vllm.assets.base import get_vllm_public_assets
7
10
-from .base import get_cache_dir
11
-
12
-
13
-@lru_cache
14
-def get_air_example_data_2_asset(filename: str) -> Image.Image:
15
- """
16
- Download and open an image from
17
- ``s3://air-example-data-2/vllm_opensource_llava/``.
18
- """
19
- image_directory = get_cache_dir() / "air-example-data-2"
20
- image_directory.mkdir(parents=True, exist_ok=True)
21
-
22
- image_path = image_directory / filename
23
- if not image_path.exists():
24
- base_url = "https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava"
25
-
26
- global_http_connection.download_file(f"{base_url}/{filename}",
27
- image_path,
28
- timeout=VLLM_IMAGE_FETCH_TIMEOUT)
29
-
30
- return Image.open(image_path)
8
+VLM_IMAGES_DIR = "vision_model_images"
9
10
11
@dataclass(frozen=True)
@@ -36,4 +14,7 @@ class ImageAsset:
14
15
@property
16
def pil_image(self) -> Image.Image:
39
- return get_air_example_data_2_asset(f"{self.name}.jpg")
17
+
18
+ image_path = get_vllm_public_assets(filename=f"{self.name}.jpg",
19
+ s3_prefix=VLM_IMAGES_DIR)
20
+ return Image.open(image_path)