@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 86ab567bae
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[CI/Build] Minor refactoring for vLLM assets (#7407)

Roger Wang committed Aug 11, 2024 at 19:41 UTC 86ab567bae0698425095e28cce67e9a31a261b72
2 files changed +35 -26
vllm/assets/base.py
+29 -1
@@ -1,11 +1,39 @@
1 +from functools import lru_cache
2 from pathlib import Path
3 +from typing import Optional
4
5 import vllm.envs as envs
6 +from vllm.connections import global_http_connection
7 +from vllm.envs import VLLM_IMAGE_FETCH_TIMEOUT
8
9 +vLLM_S3_BUCKET_URL = "https://vllm-public-assets.s3.us-west-2.amazonaws.com"
10
6 -def get_cache_dir():
11 +
12 +def get_cache_dir() -> Path:
13 """Get the path to the cache for storing downloaded assets."""
14 path = Path(envs.VLLM_ASSETS_CACHE)
15 path.mkdir(parents=True, exist_ok=True)
16
17 return path
18 +
19 +
20 +@lru_cache
21 +def get_vllm_public_assets(filename: str,
22 + s3_prefix: Optional[str] = None) -> Path:
23 + """
24 + Download an asset file from ``s3://vllm-public-assets``
25 + and return the path to the downloaded file.
26 + """
27 + asset_directory = get_cache_dir() / "vllm_public_assets"
28 + asset_directory.mkdir(parents=True, exist_ok=True)
29 +
30 + asset_path = asset_directory / filename
31 + if not asset_path.exists():
32 + if s3_prefix is not None:
33 + filename = s3_prefix + "/" + filename
34 + global_http_connection.download_file(
35 + f"{vLLM_S3_BUCKET_URL}/{filename}",
36 + asset_path,
37 + timeout=VLLM_IMAGE_FETCH_TIMEOUT)
38 +
39 + return asset_path
vllm/assets/image.py
+6 -25
@@ -1,33 +1,11 @@
1 from dataclasses import dataclass
2 -from functools import lru_cache
2 from typing import Literal
3
4 from PIL import Image
5
7 -from vllm.connections import global_http_connection
8 -from vllm.envs import VLLM_IMAGE_FETCH_TIMEOUT
6 +from vllm.assets.base import get_vllm_public_assets
7
10 -from .base import get_cache_dir
11 -
12 -
13 -@lru_cache
14 -def get_air_example_data_2_asset(filename: str) -> Image.Image:
15 - """
16 - Download and open an image from
17 - ``s3://air-example-data-2/vllm_opensource_llava/``.
18 - """
19 - image_directory = get_cache_dir() / "air-example-data-2"
20 - image_directory.mkdir(parents=True, exist_ok=True)
21 -
22 - image_path = image_directory / filename
23 - if not image_path.exists():
24 - base_url = "https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava"
25 -
26 - global_http_connection.download_file(f"{base_url}/{filename}",
27 - image_path,
28 - timeout=VLLM_IMAGE_FETCH_TIMEOUT)
29 -
30 - return Image.open(image_path)
8 +VLM_IMAGES_DIR = "vision_model_images"
9
10
11 @dataclass(frozen=True)
@@ -36,4 +14,7 @@ class ImageAsset:
14
15 @property
16 def pil_image(self) -> Image.Image:
39 - return get_air_example_data_2_asset(f"{self.name}.jpg")
17 +
18 + image_path = get_vllm_public_assets(filename=f"{self.name}.jpg",
19 + s3_prefix=VLM_IMAGES_DIR)
20 + return Image.open(image_path)