@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 344bf7cd2d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Misc] add installation time env vars (#4574)

youkaichao committed May 3, 2024 at 15:55 UTC 344bf7cd2d66a8b13f216f61c7a6d5d70576a498
2 files changed +81 -18
setup.py
+23 -10
@@ -1,3 +1,4 @@
1 +import importlib.util
2 import io
3 import logging
4 import os
@@ -13,10 +14,23 @@ from setuptools import Extension, find_packages, setup
14 from setuptools.command.build_ext import build_ext
15 from torch.utils.cpp_extension import CUDA_HOME
16
17 +
18 +def load_module_from_path(module_name, path):
19 + spec = importlib.util.spec_from_file_location(module_name, path)
20 + module = importlib.util.module_from_spec(spec)
21 + sys.modules[module_name] = module
22 + spec.loader.exec_module(module)
23 + return module
24 +
25 +
26 ROOT_DIR = os.path.dirname(__file__)
27 logger = logging.getLogger(__name__)
18 -# Target device of vLLM, supporting [cuda (by default), rocm, neuron, cpu]
19 -VLLM_TARGET_DEVICE = os.getenv("VLLM_TARGET_DEVICE", "cuda")
28 +
29 +# cannot import envs directly because it depends on vllm,
30 +# which is not installed yet
31 +envs = load_module_from_path('envs', os.path.join(ROOT_DIR, 'vllm', 'envs.py'))
32 +
33 +VLLM_TARGET_DEVICE = envs.VLLM_TARGET_DEVICE
34
35 # vLLM only supports Linux platform
36 assert sys.platform.startswith(
@@ -60,7 +74,7 @@ class cmake_build_ext(build_ext):
74 def compute_num_jobs(self):
75 # `num_jobs` is either the value of the MAX_JOBS environment variable
76 # (if defined) or the number of CPUs available.
63 - num_jobs = os.environ.get("MAX_JOBS", None)
77 + num_jobs = envs.MAX_JOBS
78 if num_jobs is not None:
79 num_jobs = int(num_jobs)
80 logger.info("Using MAX_JOBS=%d as the number of jobs.", num_jobs)
@@ -78,7 +92,7 @@ class cmake_build_ext(build_ext):
92 # environment variable (if defined) or 1.
93 # when it is set, we reduce `num_jobs` to avoid
94 # overloading the system.
81 - nvcc_threads = os.getenv("NVCC_THREADS", None)
95 + nvcc_threads = envs.NVCC_THREADS
96 if nvcc_threads is not None:
97 nvcc_threads = int(nvcc_threads)
98 logger.info(
@@ -104,7 +118,7 @@ class cmake_build_ext(build_ext):
118 # Select the build type.
119 # Note: optimization level + debug info are set by the build type
120 default_cfg = "Debug" if self.debug else "RelWithDebInfo"
107 - cfg = os.getenv("CMAKE_BUILD_TYPE", default_cfg)
121 + cfg = envs.CMAKE_BUILD_TYPE or default_cfg
122
123 # where .so files will be written, should be the same for all extensions
124 # that use the same CMakeLists.txt.
@@ -118,7 +132,7 @@ class cmake_build_ext(build_ext):
132 '-DVLLM_TARGET_DEVICE={}'.format(VLLM_TARGET_DEVICE),
133 ]
134
121 - verbose = bool(int(os.getenv('VERBOSE', '0')))
135 + verbose = envs.VERBOSE
136 if verbose:
137 cmake_args += ['-DCMAKE_VERBOSE_MAKEFILE=ON']
138
@@ -205,8 +219,7 @@ def _is_neuron() -> bool:
219 subprocess.run(["neuron-ls"], capture_output=True, check=True)
220 except (FileNotFoundError, PermissionError, subprocess.CalledProcessError):
221 torch_neuronx_installed = False
208 - return torch_neuronx_installed or os.environ.get("VLLM_BUILD_WITH_NEURON",
209 - False)
222 + return torch_neuronx_installed or envs.VLLM_BUILD_WITH_NEURON
223
224
225 def _is_cpu() -> bool:
@@ -214,7 +227,7 @@ def _is_cpu() -> bool:
227
228
229 def _install_punica() -> bool:
217 - return bool(int(os.getenv("VLLM_INSTALL_PUNICA_KERNELS", "0")))
230 + return envs.VLLM_INSTALL_PUNICA_KERNELS
231
232
233 def get_hipcc_rocm_version():
@@ -377,7 +390,7 @@ if not _is_neuron():
390 package_data = {
391 "vllm": ["py.typed", "model_executor/layers/fused_moe/configs/*.json"]
392 }
380 -if os.environ.get("VLLM_USE_PRECOMPILED"):
393 +if envs.VLLM_USE_PRECOMPILED:
394 ext_modules = []
395 package_data["vllm"].append("*.so")
396
vllm/envs.py
+58 -8
@@ -27,6 +27,14 @@ if TYPE_CHECKING:
27 VLLM_CPU_KVCACHE_SPACE: int = 0
28 VLLM_USE_RAY_COMPILED_DAG: bool = False
29 VLLM_WORKER_MULTIPROC_METHOD: str = "spawn"
30 + VLLM_TARGET_DEVICE: str = "cuda"
31 + MAX_JOBS: Optional[str] = None
32 + NVCC_THREADS: Optional[str] = None
33 + VLLM_BUILD_WITH_NEURON: bool = False
34 + VLLM_USE_PRECOMPILED: bool = False
35 + VLLM_INSTALL_PUNICA_KERNELS: bool = False
36 + CMAKE_BUILD_TYPE: Optional[str] = None
37 + VERBOSE: bool = False
38
39 # The begin-* and end* here are used by the documentation generator
40 # to extract the used env vars.
@@ -34,6 +42,56 @@ if TYPE_CHECKING:
42 # begin-env-vars-definition
43
44 environment_variables: Dict[str, Callable[[], Any]] = {
45 +
46 + # ================== Installation Time Env Vars ==================
47 +
48 + # Target device of vLLM, supporting [cuda (by default), rocm, neuron, cpu]
49 + "VLLM_TARGET_DEVICE":
50 + lambda: os.getenv("VLLM_TARGET_DEVICE", "cuda"),
51 +
52 + # Maximum number of compilation jobs to run in parallel.
53 + # By default this is the number of CPUs
54 + "MAX_JOBS":
55 + lambda: os.getenv("MAX_JOBS", None),
56 +
57 + # Number of threads to use for nvcc
58 + # By default this is 1.
59 + # If set, `MAX_JOBS` will be reduced to avoid oversubscribing the CPU.
60 + "NVCC_THREADS":
61 + lambda: os.getenv("NVCC_THREADS", None),
62 +
63 + # If set, vllm will build with Neuron support
64 + "VLLM_BUILD_WITH_NEURON":
65 + lambda: bool(os.environ.get("VLLM_BUILD_WITH_NEURON", False)),
66 +
67 + # If set, vllm will use precompiled binaries (*.so)
68 + "VLLM_USE_PRECOMPILED":
69 + lambda: bool(os.environ.get("VLLM_USE_PRECOMPILED")),
70 +
71 + # If set, vllm will install Punica kernels
72 + "VLLM_INSTALL_PUNICA_KERNELS":
73 + lambda: bool(int(os.getenv("VLLM_INSTALL_PUNICA_KERNELS", "0"))),
74 +
75 + # CMake build type
76 + # If not set, defaults to "Debug" or "RelWithDebInfo"
77 + # Available options: "Debug", "Release", "RelWithDebInfo"
78 + "CMAKE_BUILD_TYPE":
79 + lambda: os.getenv("CMAKE_BUILD_TYPE"),
80 +
81 + # If set, vllm will print verbose logs during installation
82 + "VERBOSE":
83 + lambda: bool(int(os.getenv('VERBOSE', '0'))),
84 +
85 + # Root directory for VLLM configuration files
86 + # Note that this not only affects how vllm finds its configuration files
87 + # during runtime, but also affects how vllm installs its configuration
88 + # files during **installation**.
89 + "VLLM_CONFIG_ROOT":
90 + lambda: os.environ.get("VLLM_CONFIG_ROOT", None) or os.getenv(
91 + "XDG_CONFIG_HOME", None) or os.path.expanduser("~/.config"),
92 +
93 + # ================== Runtime Env Vars ==================
94 +
95 # used in distributed environment to determine the master address
96 'VLLM_HOST_IP':
97 lambda: os.getenv('VLLM_HOST_IP', "") or os.getenv("HOST_IP", ""),
@@ -93,14 +151,6 @@ environment_variables: Dict[str, Callable[[], Any]] = {
151 "S3_ENDPOINT_URL":
152 lambda: os.environ.get("S3_ENDPOINT_URL", None),
153
96 - # Root directory for VLLM configuration files
97 - # Note that this not only affects how vllm finds its configuration files
98 - # during runtime, but also affects how vllm installs its configuration
99 - # files during **installation**.
100 - "VLLM_CONFIG_ROOT":
101 - lambda: os.environ.get("VLLM_CONFIG_ROOT", None) or os.getenv(
102 - "XDG_CONFIG_HOME", None) or os.path.expanduser("~/.config"),
103 -
154 # Usage stats collection
155 "VLLM_USAGE_STATS_SERVER":
156 lambda: os.environ.get("VLLM_USAGE_STATS_SERVER", "https://stats.vllm.ai"),