A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[CI/Build] vLLM cache directory for images (#6444)
Cyrus Leung committed
Jul 16, 2024 at 14:12 UTC
d97011512e5a816acbdb5bd8ffbf691dd227fe27
13 files changed
+123
-140
.buildkite/download-images.sh
deleted
-14
@@ -1,14 +0,0 @@
1
-#!/bin/bash
2
-
3
-set -ex
4
-set -o pipefail
5
-
6
-(which wget && which curl) || (apt-get update && apt-get install -y wget curl)
7
-
8
-# aws s3 sync s3://air-example-data-2/vllm_opensource_llava/ images/
9
-mkdir -p images
10
-cd images
11
-wget https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava/stop_sign.jpg
12
-wget https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava/cherry_blossom.jpg
13
-
14
-cd -
.buildkite/test-pipeline.yaml
-4
@@ -12,7 +12,6 @@ steps:
12
fast_check_only: true
13
commands:
14
- pytest -v -s async_engine # Async Engine
15
- - bash ../.buildkite/download-images.sh # Inputs
15
- pytest -v -s test_inputs.py
16
- pytest -v -s multimodal
17
- pytest -v -s test_utils.py # Utils
@@ -82,7 +81,6 @@ steps:
81
working_dir: "/vllm-workspace/tests"
82
num_gpus: 2
83
commands:
85
- - bash ../.buildkite/download-images.sh
84
- VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py
85
- TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
86
- TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
@@ -155,7 +153,6 @@ steps:
153
- label: Inputs Test
154
#mirror_hardwares: [amd]
155
commands:
158
- - bash ../.buildkite/download-images.sh
156
- pytest -v -s test_inputs.py
157
- pytest -v -s multimodal
158
@@ -175,7 +172,6 @@ steps:
172
- label: Vision Language Models Test
173
mirror_hardwares: [amd]
174
commands:
178
- - bash ../.buildkite/download-images.sh
175
- pytest -v -s models -m vlm
176
177
- label: Prefix Caching Test
examples/llava_example.py
+3
-30
@@ -1,12 +1,5 @@
1
-import os
2
-import subprocess
3
-
4
-from PIL import Image
5
-
1
from vllm import LLM
7
-
8
-# The assets are located at `s3://air-example-data-2/vllm_opensource_llava/`.
9
-# You can use `.buildkite/download-images.sh` to download them
2
+from vllm.assets.image import ImageAsset
3
4
5
def run_llava():
@@ -14,7 +7,7 @@ def run_llava():
7
8
prompt = "USER: <image>\nWhat is the content of this image?\nASSISTANT:"
9
17
- image = Image.open("images/stop_sign.jpg")
10
+ image = ImageAsset("stop_sign").pil_image
11
12
outputs = llm.generate({
13
"prompt": prompt,
@@ -28,25 +21,5 @@ def run_llava():
21
print(generated_text)
22
23
31
-def main():
32
- run_llava()
33
-
34
-
24
if __name__ == "__main__":
36
- # Download from s3
37
- s3_bucket_path = "s3://air-example-data-2/vllm_opensource_llava/"
38
- local_directory = "images"
39
-
40
- # Make sure the local directory exists or create it
41
- os.makedirs(local_directory, exist_ok=True)
42
-
43
- # Use AWS CLI to sync the directory, assume anonymous access
44
- subprocess.check_call([
45
- "aws",
46
- "s3",
47
- "sync",
48
- s3_bucket_path,
49
- local_directory,
50
- "--no-sign-request",
51
- ])
52
- main()
25
+ run_llava()
examples/paligemma_example.py
+3
-30
@@ -1,12 +1,5 @@
1
-import os
2
-import subprocess
3
-
4
-from PIL import Image
5
-
1
from vllm import LLM
7
-
8
-# The assets are located at `s3://air-example-data-2/vllm_opensource_llava/`.
9
-# You can use `.buildkite/download-images.sh` to download them
2
+from vllm.assets.image import ImageAsset
3
4
5
def run_paligemma():
@@ -14,7 +7,7 @@ def run_paligemma():
7
8
prompt = "caption es"
9
17
- image = Image.open("images/stop_sign.jpg")
10
+ image = ImageAsset("stop_sign").pil_image
11
12
outputs = llm.generate({
13
"prompt": prompt,
@@ -28,25 +21,5 @@ def run_paligemma():
21
print(generated_text)
22
23
31
-def main():
32
- run_paligemma()
33
-
34
-
24
if __name__ == "__main__":
36
- # Download from s3
37
- s3_bucket_path = "s3://air-example-data-2/vllm_opensource_llava/"
38
- local_directory = "images"
39
-
40
- # Make sure the local directory exists or create it
41
- os.makedirs(local_directory, exist_ok=True)
42
-
43
- # Use AWS CLI to sync the directory, assume anonymous access
44
- subprocess.check_call([
45
- "aws",
46
- "s3",
47
- "sync",
48
- s3_bucket_path,
49
- local_directory,
50
- "--no-sign-request",
51
- ])
52
- main()
25
+ run_paligemma()
examples/phi3v_example.py
+2
-24
@@ -1,12 +1,5 @@
1
-import os
2
-import subprocess
3
-
4
-from PIL import Image
5
-
1
from vllm import LLM, SamplingParams
7
-
8
-# The assets are located at `s3://air-example-data-2/vllm_opensource_llava/`.
9
-# You can use `.buildkite/download-images.sh` to download them
2
+from vllm.assets.image import ImageAsset
3
4
5
def run_phi3v():
@@ -24,7 +17,7 @@ def run_phi3v():
17
max_num_seqs=5,
18
)
19
27
- image = Image.open("images/cherry_blossom.jpg")
20
+ image = ImageAsset("cherry_blossom").pil_image
21
22
# single-image prompt
23
prompt = "<|user|>\n<|image_1|>\nWhat is the season?<|end|>\n<|assistant|>\n" # noqa: E501
@@ -44,19 +37,4 @@ def run_phi3v():
37
38
39
if __name__ == "__main__":
47
- s3_bucket_path = "s3://air-example-data-2/vllm_opensource_llava/"
48
- local_directory = "images"
49
-
50
- # Make sure the local directory exists or create it
51
- os.makedirs(local_directory, exist_ok=True)
52
-
53
- # Use AWS CLI to sync the directory, assume anonymous access
54
- subprocess.check_call([
55
- "aws",
56
- "s3",
57
- "sync",
58
- s3_bucket_path,
59
- local_directory,
60
- "--no-sign-request",
61
- ])
40
run_phi3v()
tests/conftest.py
+2
-23
@@ -3,11 +3,7 @@ import gc
3
import os
4
import sys
5
from collections import UserList
6
-from dataclasses import dataclass
7
-from functools import cached_property
8
-from pathlib import Path
9
-from typing import (Any, Dict, List, Literal, Optional, Tuple, TypedDict,
10
- TypeVar)
6
+from typing import Any, Dict, List, Optional, Tuple, TypedDict, TypeVar
7
8
import pytest
9
import torch
@@ -18,12 +14,12 @@ from transformers import (AutoModelForCausalLM, AutoModelForVision2Seq,
14
AutoTokenizer, BatchEncoding)
15
16
from vllm import LLM, SamplingParams
17
+from vllm.assets.image import ImageAsset
18
from vllm.config import TokenizerPoolConfig
19
from vllm.distributed import (destroy_distributed_environment,
20
destroy_model_parallel)
21
from vllm.inputs import TextPrompt
22
from vllm.logger import init_logger
26
-from vllm.multimodal.utils import fetch_image
23
from vllm.sequence import SampleLogprobs
24
from vllm.utils import cuda_device_count_stateless, is_cpu
25
@@ -33,9 +29,6 @@ _TEST_DIR = os.path.dirname(__file__)
29
_TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]
30
_LONG_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "summary.txt")]
31
36
-_IMAGE_DIR = Path(_TEST_DIR) / "images"
37
-"""You can use `.buildkite/download-images.sh` to download the assets."""
38
-
32
33
def _read_prompts(filename: str) -> List[str]:
34
with open(filename, "r") as f:
@@ -43,20 +36,6 @@ def _read_prompts(filename: str) -> List[str]:
36
return prompts
37
38
46
-@dataclass(frozen=True)
47
-class ImageAsset:
48
- name: Literal["stop_sign", "cherry_blossom", "boardwalk"]
49
-
50
- @cached_property
51
- def pil_image(self) -> Image.Image:
52
- if self.name == "boardwalk":
53
- return fetch_image(
54
- "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
55
- )
56
-
57
- return Image.open(_IMAGE_DIR / f"{self.name}.jpg")
58
-
59
-
39
class _ImageAssetPrompts(TypedDict):
40
stop_sign: str
41
cherry_blossom: str
vllm/assets/__init__.py
vllm/assets/base.py
new
+11
@@ -0,0 +1,11 @@
1
+from pathlib import Path
2
+
3
+import vllm.envs as envs
4
+
5
+
6
+def get_cache_dir():
7
+ """Get the path to the cache for storing downloaded assets."""
8
+ path = Path(envs.VLLM_ASSETS_CACHE)
9
+ path.mkdir(parents=True, exist_ok=True)
10
+
11
+ return path
vllm/assets/image.py
new
+47
@@ -0,0 +1,47 @@
1
+import shutil
2
+from dataclasses import dataclass
3
+from functools import cached_property, lru_cache
4
+from typing import Literal
5
+
6
+import requests
7
+from PIL import Image
8
+
9
+from vllm.multimodal.utils import fetch_image
10
+
11
+from .base import get_cache_dir
12
+
13
+
14
+@lru_cache
15
+def get_air_example_data_2_asset(filename: str) -> Image.Image:
16
+ """
17
+ Download and open an image from
18
+ ``s3://air-example-data-2/vllm_opensource_llava/``.
19
+ """
20
+ image_directory = get_cache_dir() / "air-example-data-2"
21
+ image_directory.mkdir(parents=True, exist_ok=True)
22
+
23
+ image_path = image_directory / filename
24
+ if not image_path.exists():
25
+ base_url = "https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava"
26
+
27
+ with requests.get(f"{base_url}/{filename}", stream=True) as response:
28
+ response.raise_for_status()
29
+
30
+ with image_path.open("wb") as f:
31
+ shutil.copyfileobj(response.raw, f)
32
+
33
+ return Image.open(image_path)
34
+
35
+
36
+@dataclass(frozen=True)
37
+class ImageAsset:
38
+ name: Literal["stop_sign", "cherry_blossom", "boardwalk"]
39
+
40
+ @cached_property
41
+ def pil_image(self) -> Image.Image:
42
+ if self.name == "boardwalk":
43
+ return fetch_image(
44
+ "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
45
+ )
46
+
47
+ return get_air_example_data_2_asset(f"{self.name}.jpg")
vllm/distributed/device_communicators/custom_all_reduce_utils.py
+4
-4
@@ -189,10 +189,10 @@ def gpu_p2p_access_check(src: int, tgt: int) -> bool:
189
cuda_visible_devices = envs.CUDA_VISIBLE_DEVICES
190
if cuda_visible_devices is None:
191
cuda_visible_devices = ",".join(str(i) for i in range(num_dev))
192
- VLLM_CONFIG_ROOT = envs.VLLM_CONFIG_ROOT
193
- path = os.path.expanduser(
194
- f"{VLLM_CONFIG_ROOT}/vllm/gpu_p2p_access_cache_for_{cuda_visible_devices}.json"
195
- )
192
+
193
+ path = os.path.join(
194
+ envs.VLLM_CACHE_ROOT,
195
+ f"gpu_p2p_access_cache_for_{cuda_visible_devices}.json")
196
os.makedirs(os.path.dirname(path), exist_ok=True)
197
from vllm.distributed.parallel_state import get_world_group
198
if ((not is_distributed or get_world_group().local_rank == 0)
vllm/envs.py
+48
-6
@@ -17,7 +17,8 @@ if TYPE_CHECKING:
17
S3_ACCESS_KEY_ID: Optional[str] = None
18
S3_SECRET_ACCESS_KEY: Optional[str] = None
19
S3_ENDPOINT_URL: Optional[str] = None
20
- VLLM_CONFIG_ROOT: str = ""
20
+ VLLM_CACHE_ROOT: str = os.path.expanduser("~/.cache/vllm")
21
+ VLLM_CONFIG_ROOT: str = os.path.expanduser("~/.config/vllm")
22
VLLM_USAGE_STATS_SERVER: str = "https://stats.vllm.ai"
23
VLLM_NO_USAGE_STATS: bool = False
24
VLLM_DO_NOT_TRACK: bool = False
@@ -31,10 +32,11 @@ if TYPE_CHECKING:
32
VLLM_OPENVINO_KVCACHE_SPACE: int = 0
33
VLLM_OPENVINO_CPU_KV_CACHE_PRECISION: Optional[str] = None
34
VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS: bool = False
34
- VLLM_XLA_CACHE_PATH: str = "~/.vllm/xla_cache/"
35
+ VLLM_XLA_CACHE_PATH: str = os.path.join(VLLM_CACHE_ROOT, "xla_cache")
36
VLLM_FUSED_MOE_CHUNK_SIZE: int = 64 * 1024
37
VLLM_USE_RAY_COMPILED_DAG: bool = False
38
VLLM_WORKER_MULTIPROC_METHOD: str = "fork"
39
+ VLLM_ASSETS_CACHE: str = os.path.join(VLLM_CACHE_ROOT, "assets")
40
VLLM_IMAGE_FETCH_TIMEOUT: int = 5
41
VLLM_TARGET_DEVICE: str = "cuda"
42
MAX_JOBS: Optional[str] = None
@@ -45,6 +47,21 @@ if TYPE_CHECKING:
47
CMAKE_BUILD_TYPE: Optional[str] = None
48
VERBOSE: bool = False
49
50
+
51
+def get_default_cache_root():
52
+ return os.getenv(
53
+ "XDG_CACHE_HOME",
54
+ os.path.join(os.path.expanduser("~"), ".cache"),
55
+ )
56
+
57
+
58
+def get_default_config_root():
59
+ return os.getenv(
60
+ "XDG_CONFIG_HOME",
61
+ os.path.join(os.path.expanduser("~"), ".config"),
62
+ )
63
+
64
+
65
# The begin-* and end* here are used by the documentation generator
66
# to extract the used env vars.
67
@@ -89,15 +106,28 @@ environment_variables: Dict[str, Callable[[], Any]] = {
106
lambda: bool(int(os.getenv('VERBOSE', '0'))),
107
108
# Root directory for VLLM configuration files
109
+ # Defaults to `~/.config/vllm` unless `XDG_CONFIG_HOME` is set
110
# Note that this not only affects how vllm finds its configuration files
111
# during runtime, but also affects how vllm installs its configuration
112
# files during **installation**.
113
"VLLM_CONFIG_ROOT":
96
- lambda: os.environ.get("VLLM_CONFIG_ROOT", None) or os.getenv(
97
- "XDG_CONFIG_HOME", None) or os.path.expanduser("~/.config"),
114
+ lambda: os.path.expanduser(
115
+ os.getenv(
116
+ "VLLM_CONFIG_ROOT",
117
+ os.path.join(get_default_config_root(), "vllm"),
118
+ )),
119
120
# ================== Runtime Env Vars ==================
121
122
+ # Root directory for VLLM cache files
123
+ # Defaults to `~/.cache/vllm` unless `XDG_CACHE_HOME` is set
124
+ "VLLM_CACHE_ROOT":
125
+ lambda: os.path.expanduser(
126
+ os.getenv(
127
+ "VLLM_CACHE_ROOT",
128
+ os.path.join(get_default_cache_root(), "vllm"),
129
+ )),
130
+
131
# used in distributed environment to determine the master address
132
'VLLM_HOST_IP':
133
lambda: os.getenv('VLLM_HOST_IP', "") or os.getenv("HOST_IP", ""),
@@ -242,6 +272,14 @@ environment_variables: Dict[str, Callable[[], Any]] = {
272
"VLLM_WORKER_MULTIPROC_METHOD":
273
lambda: os.getenv("VLLM_WORKER_MULTIPROC_METHOD", "fork"),
274
275
+ # Path to the cache for storing downloaded assets
276
+ "VLLM_ASSETS_CACHE":
277
+ lambda: os.path.expanduser(
278
+ os.getenv(
279
+ "VLLM_ASSETS_CACHE",
280
+ os.path.join(get_default_cache_root(), "vllm", "assets"),
281
+ )),
282
+
283
# Timeout for fetching images when serving multimodal models
284
# Default is 5 seconds
285
"VLLM_IMAGE_FETCH_TIMEOUT":
@@ -250,7 +288,11 @@ environment_variables: Dict[str, Callable[[], Any]] = {
288
# Path to the XLA persistent cache directory.
289
# Only used for XLA devices such as TPUs.
290
"VLLM_XLA_CACHE_PATH":
253
- lambda: os.getenv("VLLM_XLA_CACHE_PATH", "~/.vllm/xla_cache/"),
291
+ lambda: os.path.expanduser(
292
+ os.getenv(
293
+ "VLLM_ASSETS_CACHE",
294
+ os.path.join(get_default_cache_root(), "vllm", "xla_cache"),
295
+ )),
296
"VLLM_FUSED_MOE_CHUNK_SIZE":
297
lambda: int(os.getenv("VLLM_FUSED_MOE_CHUNK_SIZE", "65536")),
298
@@ -262,7 +304,7 @@ environment_variables: Dict[str, Callable[[], Any]] = {
304
# end-env-vars-definition
305
306
265
-def __getattr__(name):
307
+def __getattr__(name: str):
308
# lazy evaluation of environment variables
309
if name in environment_variables:
310
return environment_variables[name]()
vllm/usage/usage_lib.py
+2
-3
@@ -19,9 +19,8 @@ import vllm.envs as envs
19
from vllm.version import __version__ as VLLM_VERSION
20
21
_config_home = envs.VLLM_CONFIG_ROOT
22
-_USAGE_STATS_JSON_PATH = os.path.join(_config_home, "vllm/usage_stats.json")
23
-_USAGE_STATS_DO_NOT_TRACK_PATH = os.path.join(_config_home,
24
- "vllm/do_not_track")
22
+_USAGE_STATS_JSON_PATH = os.path.join(_config_home, "usage_stats.json")
23
+_USAGE_STATS_DO_NOT_TRACK_PATH = os.path.join(_config_home, "do_not_track")
24
_USAGE_STATS_ENABLED = None
25
_USAGE_STATS_SERVER = envs.VLLM_USAGE_STATS_SERVER
26
vllm/worker/tpu_worker.py
+1
-2
@@ -98,8 +98,7 @@ class TPUWorker(LoraNotSupportedWorkerBase):
98
# Use persistent cache to avoid XLA recompilation.
99
# NOTE(woosuk): This does not completely eliminate the recompilation
100
# overhead because dynamo does not cache the compiled results.
101
- xr.initialize_cache(os.path.expanduser(envs.VLLM_XLA_CACHE_PATH),
102
- readonly=False)
101
+ xr.initialize_cache(envs.VLLM_XLA_CACHE_PATH, readonly=False)
102
103
def load_model(self):
104
self.model_runner.load_model()