@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d97011512e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[CI/Build] vLLM cache directory for images (#6444)

Cyrus Leung committed Jul 16, 2024 at 14:12 UTC d97011512e5a816acbdb5bd8ffbf691dd227fe27
13 files changed +123 -140
.buildkite/download-images.sh deleted
-14
@@ -1,14 +0,0 @@
1 -#!/bin/bash
2 -
3 -set -ex
4 -set -o pipefail
5 -
6 -(which wget && which curl) || (apt-get update && apt-get install -y wget curl)
7 -
8 -# aws s3 sync s3://air-example-data-2/vllm_opensource_llava/ images/
9 -mkdir -p images
10 -cd images
11 -wget https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava/stop_sign.jpg
12 -wget https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava/cherry_blossom.jpg
13 -
14 -cd -
.buildkite/test-pipeline.yaml
-4
@@ -12,7 +12,6 @@ steps:
12 fast_check_only: true
13 commands:
14 - pytest -v -s async_engine # Async Engine
15 - - bash ../.buildkite/download-images.sh # Inputs
15 - pytest -v -s test_inputs.py
16 - pytest -v -s multimodal
17 - pytest -v -s test_utils.py # Utils
@@ -82,7 +81,6 @@ steps:
81 working_dir: "/vllm-workspace/tests"
82 num_gpus: 2
83 commands:
85 - - bash ../.buildkite/download-images.sh
84 - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py
85 - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
86 - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
@@ -155,7 +153,6 @@ steps:
153 - label: Inputs Test
154 #mirror_hardwares: [amd]
155 commands:
158 - - bash ../.buildkite/download-images.sh
156 - pytest -v -s test_inputs.py
157 - pytest -v -s multimodal
158
@@ -175,7 +172,6 @@ steps:
172 - label: Vision Language Models Test
173 mirror_hardwares: [amd]
174 commands:
178 - - bash ../.buildkite/download-images.sh
175 - pytest -v -s models -m vlm
176
177 - label: Prefix Caching Test
examples/llava_example.py
+3 -30
@@ -1,12 +1,5 @@
1 -import os
2 -import subprocess
3 -
4 -from PIL import Image
5 -
1 from vllm import LLM
7 -
8 -# The assets are located at `s3://air-example-data-2/vllm_opensource_llava/`.
9 -# You can use `.buildkite/download-images.sh` to download them
2 +from vllm.assets.image import ImageAsset
3
4
5 def run_llava():
@@ -14,7 +7,7 @@ def run_llava():
7
8 prompt = "USER: <image>\nWhat is the content of this image?\nASSISTANT:"
9
17 - image = Image.open("images/stop_sign.jpg")
10 + image = ImageAsset("stop_sign").pil_image
11
12 outputs = llm.generate({
13 "prompt": prompt,
@@ -28,25 +21,5 @@ def run_llava():
21 print(generated_text)
22
23
31 -def main():
32 - run_llava()
33 -
34 -
24 if __name__ == "__main__":
36 - # Download from s3
37 - s3_bucket_path = "s3://air-example-data-2/vllm_opensource_llava/"
38 - local_directory = "images"
39 -
40 - # Make sure the local directory exists or create it
41 - os.makedirs(local_directory, exist_ok=True)
42 -
43 - # Use AWS CLI to sync the directory, assume anonymous access
44 - subprocess.check_call([
45 - "aws",
46 - "s3",
47 - "sync",
48 - s3_bucket_path,
49 - local_directory,
50 - "--no-sign-request",
51 - ])
52 - main()
25 + run_llava()
examples/paligemma_example.py
+3 -30
@@ -1,12 +1,5 @@
1 -import os
2 -import subprocess
3 -
4 -from PIL import Image
5 -
1 from vllm import LLM
7 -
8 -# The assets are located at `s3://air-example-data-2/vllm_opensource_llava/`.
9 -# You can use `.buildkite/download-images.sh` to download them
2 +from vllm.assets.image import ImageAsset
3
4
5 def run_paligemma():
@@ -14,7 +7,7 @@ def run_paligemma():
7
8 prompt = "caption es"
9
17 - image = Image.open("images/stop_sign.jpg")
10 + image = ImageAsset("stop_sign").pil_image
11
12 outputs = llm.generate({
13 "prompt": prompt,
@@ -28,25 +21,5 @@ def run_paligemma():
21 print(generated_text)
22
23
31 -def main():
32 - run_paligemma()
33 -
34 -
24 if __name__ == "__main__":
36 - # Download from s3
37 - s3_bucket_path = "s3://air-example-data-2/vllm_opensource_llava/"
38 - local_directory = "images"
39 -
40 - # Make sure the local directory exists or create it
41 - os.makedirs(local_directory, exist_ok=True)
42 -
43 - # Use AWS CLI to sync the directory, assume anonymous access
44 - subprocess.check_call([
45 - "aws",
46 - "s3",
47 - "sync",
48 - s3_bucket_path,
49 - local_directory,
50 - "--no-sign-request",
51 - ])
52 - main()
25 + run_paligemma()
examples/phi3v_example.py
+2 -24
@@ -1,12 +1,5 @@
1 -import os
2 -import subprocess
3 -
4 -from PIL import Image
5 -
1 from vllm import LLM, SamplingParams
7 -
8 -# The assets are located at `s3://air-example-data-2/vllm_opensource_llava/`.
9 -# You can use `.buildkite/download-images.sh` to download them
2 +from vllm.assets.image import ImageAsset
3
4
5 def run_phi3v():
@@ -24,7 +17,7 @@ def run_phi3v():
17 max_num_seqs=5,
18 )
19
27 - image = Image.open("images/cherry_blossom.jpg")
20 + image = ImageAsset("cherry_blossom").pil_image
21
22 # single-image prompt
23 prompt = "<|user|>\n<|image_1|>\nWhat is the season?<|end|>\n<|assistant|>\n" # noqa: E501
@@ -44,19 +37,4 @@ def run_phi3v():
37
38
39 if __name__ == "__main__":
47 - s3_bucket_path = "s3://air-example-data-2/vllm_opensource_llava/"
48 - local_directory = "images"
49 -
50 - # Make sure the local directory exists or create it
51 - os.makedirs(local_directory, exist_ok=True)
52 -
53 - # Use AWS CLI to sync the directory, assume anonymous access
54 - subprocess.check_call([
55 - "aws",
56 - "s3",
57 - "sync",
58 - s3_bucket_path,
59 - local_directory,
60 - "--no-sign-request",
61 - ])
40 run_phi3v()
tests/conftest.py
+2 -23
@@ -3,11 +3,7 @@ import gc
3 import os
4 import sys
5 from collections import UserList
6 -from dataclasses import dataclass
7 -from functools import cached_property
8 -from pathlib import Path
9 -from typing import (Any, Dict, List, Literal, Optional, Tuple, TypedDict,
10 - TypeVar)
6 +from typing import Any, Dict, List, Optional, Tuple, TypedDict, TypeVar
7
8 import pytest
9 import torch
@@ -18,12 +14,12 @@ from transformers import (AutoModelForCausalLM, AutoModelForVision2Seq,
14 AutoTokenizer, BatchEncoding)
15
16 from vllm import LLM, SamplingParams
17 +from vllm.assets.image import ImageAsset
18 from vllm.config import TokenizerPoolConfig
19 from vllm.distributed import (destroy_distributed_environment,
20 destroy_model_parallel)
21 from vllm.inputs import TextPrompt
22 from vllm.logger import init_logger
26 -from vllm.multimodal.utils import fetch_image
23 from vllm.sequence import SampleLogprobs
24 from vllm.utils import cuda_device_count_stateless, is_cpu
25
@@ -33,9 +29,6 @@ _TEST_DIR = os.path.dirname(__file__)
29 _TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]
30 _LONG_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "summary.txt")]
31
36 -_IMAGE_DIR = Path(_TEST_DIR) / "images"
37 -"""You can use `.buildkite/download-images.sh` to download the assets."""
38 -
32
33 def _read_prompts(filename: str) -> List[str]:
34 with open(filename, "r") as f:
@@ -43,20 +36,6 @@ def _read_prompts(filename: str) -> List[str]:
36 return prompts
37
38
46 -@dataclass(frozen=True)
47 -class ImageAsset:
48 - name: Literal["stop_sign", "cherry_blossom", "boardwalk"]
49 -
50 - @cached_property
51 - def pil_image(self) -> Image.Image:
52 - if self.name == "boardwalk":
53 - return fetch_image(
54 - "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
55 - )
56 -
57 - return Image.open(_IMAGE_DIR / f"{self.name}.jpg")
58 -
59 -
39 class _ImageAssetPrompts(TypedDict):
40 stop_sign: str
41 cherry_blossom: str
vllm/assets/__init__.py
vllm/assets/base.py new
+11
@@ -0,0 +1,11 @@
1 +from pathlib import Path
2 +
3 +import vllm.envs as envs
4 +
5 +
6 +def get_cache_dir():
7 + """Get the path to the cache for storing downloaded assets."""
8 + path = Path(envs.VLLM_ASSETS_CACHE)
9 + path.mkdir(parents=True, exist_ok=True)
10 +
11 + return path
vllm/assets/image.py new
+47
@@ -0,0 +1,47 @@
1 +import shutil
2 +from dataclasses import dataclass
3 +from functools import cached_property, lru_cache
4 +from typing import Literal
5 +
6 +import requests
7 +from PIL import Image
8 +
9 +from vllm.multimodal.utils import fetch_image
10 +
11 +from .base import get_cache_dir
12 +
13 +
14 +@lru_cache
15 +def get_air_example_data_2_asset(filename: str) -> Image.Image:
16 + """
17 + Download and open an image from
18 + ``s3://air-example-data-2/vllm_opensource_llava/``.
19 + """
20 + image_directory = get_cache_dir() / "air-example-data-2"
21 + image_directory.mkdir(parents=True, exist_ok=True)
22 +
23 + image_path = image_directory / filename
24 + if not image_path.exists():
25 + base_url = "https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava"
26 +
27 + with requests.get(f"{base_url}/{filename}", stream=True) as response:
28 + response.raise_for_status()
29 +
30 + with image_path.open("wb") as f:
31 + shutil.copyfileobj(response.raw, f)
32 +
33 + return Image.open(image_path)
34 +
35 +
36 +@dataclass(frozen=True)
37 +class ImageAsset:
38 + name: Literal["stop_sign", "cherry_blossom", "boardwalk"]
39 +
40 + @cached_property
41 + def pil_image(self) -> Image.Image:
42 + if self.name == "boardwalk":
43 + return fetch_image(
44 + "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
45 + )
46 +
47 + return get_air_example_data_2_asset(f"{self.name}.jpg")
vllm/distributed/device_communicators/custom_all_reduce_utils.py
+4 -4
@@ -189,10 +189,10 @@ def gpu_p2p_access_check(src: int, tgt: int) -> bool:
189 cuda_visible_devices = envs.CUDA_VISIBLE_DEVICES
190 if cuda_visible_devices is None:
191 cuda_visible_devices = ",".join(str(i) for i in range(num_dev))
192 - VLLM_CONFIG_ROOT = envs.VLLM_CONFIG_ROOT
193 - path = os.path.expanduser(
194 - f"{VLLM_CONFIG_ROOT}/vllm/gpu_p2p_access_cache_for_{cuda_visible_devices}.json"
195 - )
192 +
193 + path = os.path.join(
194 + envs.VLLM_CACHE_ROOT,
195 + f"gpu_p2p_access_cache_for_{cuda_visible_devices}.json")
196 os.makedirs(os.path.dirname(path), exist_ok=True)
197 from vllm.distributed.parallel_state import get_world_group
198 if ((not is_distributed or get_world_group().local_rank == 0)
vllm/envs.py
+48 -6
@@ -17,7 +17,8 @@ if TYPE_CHECKING:
17 S3_ACCESS_KEY_ID: Optional[str] = None
18 S3_SECRET_ACCESS_KEY: Optional[str] = None
19 S3_ENDPOINT_URL: Optional[str] = None
20 - VLLM_CONFIG_ROOT: str = ""
20 + VLLM_CACHE_ROOT: str = os.path.expanduser("~/.cache/vllm")
21 + VLLM_CONFIG_ROOT: str = os.path.expanduser("~/.config/vllm")
22 VLLM_USAGE_STATS_SERVER: str = "https://stats.vllm.ai"
23 VLLM_NO_USAGE_STATS: bool = False
24 VLLM_DO_NOT_TRACK: bool = False
@@ -31,10 +32,11 @@ if TYPE_CHECKING:
32 VLLM_OPENVINO_KVCACHE_SPACE: int = 0
33 VLLM_OPENVINO_CPU_KV_CACHE_PRECISION: Optional[str] = None
34 VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS: bool = False
34 - VLLM_XLA_CACHE_PATH: str = "~/.vllm/xla_cache/"
35 + VLLM_XLA_CACHE_PATH: str = os.path.join(VLLM_CACHE_ROOT, "xla_cache")
36 VLLM_FUSED_MOE_CHUNK_SIZE: int = 64 * 1024
37 VLLM_USE_RAY_COMPILED_DAG: bool = False
38 VLLM_WORKER_MULTIPROC_METHOD: str = "fork"
39 + VLLM_ASSETS_CACHE: str = os.path.join(VLLM_CACHE_ROOT, "assets")
40 VLLM_IMAGE_FETCH_TIMEOUT: int = 5
41 VLLM_TARGET_DEVICE: str = "cuda"
42 MAX_JOBS: Optional[str] = None
@@ -45,6 +47,21 @@ if TYPE_CHECKING:
47 CMAKE_BUILD_TYPE: Optional[str] = None
48 VERBOSE: bool = False
49
50 +
51 +def get_default_cache_root():
52 + return os.getenv(
53 + "XDG_CACHE_HOME",
54 + os.path.join(os.path.expanduser("~"), ".cache"),
55 + )
56 +
57 +
58 +def get_default_config_root():
59 + return os.getenv(
60 + "XDG_CONFIG_HOME",
61 + os.path.join(os.path.expanduser("~"), ".config"),
62 + )
63 +
64 +
65 # The begin-* and end* here are used by the documentation generator
66 # to extract the used env vars.
67
@@ -89,15 +106,28 @@ environment_variables: Dict[str, Callable[[], Any]] = {
106 lambda: bool(int(os.getenv('VERBOSE', '0'))),
107
108 # Root directory for VLLM configuration files
109 + # Defaults to `~/.config/vllm` unless `XDG_CONFIG_HOME` is set
110 # Note that this not only affects how vllm finds its configuration files
111 # during runtime, but also affects how vllm installs its configuration
112 # files during **installation**.
113 "VLLM_CONFIG_ROOT":
96 - lambda: os.environ.get("VLLM_CONFIG_ROOT", None) or os.getenv(
97 - "XDG_CONFIG_HOME", None) or os.path.expanduser("~/.config"),
114 + lambda: os.path.expanduser(
115 + os.getenv(
116 + "VLLM_CONFIG_ROOT",
117 + os.path.join(get_default_config_root(), "vllm"),
118 + )),
119
120 # ================== Runtime Env Vars ==================
121
122 + # Root directory for VLLM cache files
123 + # Defaults to `~/.cache/vllm` unless `XDG_CACHE_HOME` is set
124 + "VLLM_CACHE_ROOT":
125 + lambda: os.path.expanduser(
126 + os.getenv(
127 + "VLLM_CACHE_ROOT",
128 + os.path.join(get_default_cache_root(), "vllm"),
129 + )),
130 +
131 # used in distributed environment to determine the master address
132 'VLLM_HOST_IP':
133 lambda: os.getenv('VLLM_HOST_IP', "") or os.getenv("HOST_IP", ""),
@@ -242,6 +272,14 @@ environment_variables: Dict[str, Callable[[], Any]] = {
272 "VLLM_WORKER_MULTIPROC_METHOD":
273 lambda: os.getenv("VLLM_WORKER_MULTIPROC_METHOD", "fork"),
274
275 + # Path to the cache for storing downloaded assets
276 + "VLLM_ASSETS_CACHE":
277 + lambda: os.path.expanduser(
278 + os.getenv(
279 + "VLLM_ASSETS_CACHE",
280 + os.path.join(get_default_cache_root(), "vllm", "assets"),
281 + )),
282 +
283 # Timeout for fetching images when serving multimodal models
284 # Default is 5 seconds
285 "VLLM_IMAGE_FETCH_TIMEOUT":
@@ -250,7 +288,11 @@ environment_variables: Dict[str, Callable[[], Any]] = {
288 # Path to the XLA persistent cache directory.
289 # Only used for XLA devices such as TPUs.
290 "VLLM_XLA_CACHE_PATH":
253 - lambda: os.getenv("VLLM_XLA_CACHE_PATH", "~/.vllm/xla_cache/"),
291 + lambda: os.path.expanduser(
292 + os.getenv(
293 + "VLLM_ASSETS_CACHE",
294 + os.path.join(get_default_cache_root(), "vllm", "xla_cache"),
295 + )),
296 "VLLM_FUSED_MOE_CHUNK_SIZE":
297 lambda: int(os.getenv("VLLM_FUSED_MOE_CHUNK_SIZE", "65536")),
298
@@ -262,7 +304,7 @@ environment_variables: Dict[str, Callable[[], Any]] = {
304 # end-env-vars-definition
305
306
265 -def __getattr__(name):
307 +def __getattr__(name: str):
308 # lazy evaluation of environment variables
309 if name in environment_variables:
310 return environment_variables[name]()
vllm/usage/usage_lib.py
+2 -3
@@ -19,9 +19,8 @@ import vllm.envs as envs
19 from vllm.version import __version__ as VLLM_VERSION
20
21 _config_home = envs.VLLM_CONFIG_ROOT
22 -_USAGE_STATS_JSON_PATH = os.path.join(_config_home, "vllm/usage_stats.json")
23 -_USAGE_STATS_DO_NOT_TRACK_PATH = os.path.join(_config_home,
24 - "vllm/do_not_track")
22 +_USAGE_STATS_JSON_PATH = os.path.join(_config_home, "usage_stats.json")
23 +_USAGE_STATS_DO_NOT_TRACK_PATH = os.path.join(_config_home, "do_not_track")
24 _USAGE_STATS_ENABLED = None
25 _USAGE_STATS_SERVER = envs.VLLM_USAGE_STATS_SERVER
26
vllm/worker/tpu_worker.py
+1 -2
@@ -98,8 +98,7 @@ class TPUWorker(LoraNotSupportedWorkerBase):
98 # Use persistent cache to avoid XLA recompilation.
99 # NOTE(woosuk): This does not completely eliminate the recompilation
100 # overhead because dynamo does not cache the compiled results.
101 - xr.initialize_cache(os.path.expanduser(envs.VLLM_XLA_CACHE_PATH),
102 - readonly=False)
101 + xr.initialize_cache(envs.VLLM_XLA_CACHE_PATH, readonly=False)
102
103 def load_model(self):
104 self.model_runner.load_model()