A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Update setup.py (#1006)
Woosuk Kwon committed
Sep 10, 2023 at 23:42 UTC
d6770d1f23b642289a2f1462f2851a7be9d3cc83
1 file changed
+35
-14
setup.py
+35
-14
@@ -22,7 +22,7 @@ NVCC_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]
22
23
if CUDA_HOME is None:
24
raise RuntimeError(
25
- f"Cannot find CUDA_HOME. CUDA must be available to build the package.")
25
+ "Cannot find CUDA_HOME. CUDA must be available to build the package.")
26
27
28
def get_nvcc_cuda_version(cuda_dir: str) -> Version:
@@ -54,7 +54,8 @@ if nvcc_cuda_version < Version("11.0"):
54
raise RuntimeError("CUDA 11.0 or higher is required to build the package.")
55
if 86 in compute_capabilities and nvcc_cuda_version < Version("11.1"):
56
raise RuntimeError(
57
- "CUDA 11.1 or higher is required for GPUs with compute capability 8.6.")
57
+ "CUDA 11.1 or higher is required for GPUs with compute capability 8.6."
58
+ )
59
if 89 in compute_capabilities and nvcc_cuda_version < Version("11.8"):
60
# CUDA 11.8 is required to generate the code targeting compute capability 8.9.
61
# However, GPUs with compute capability 8.9 can also run the code generated by
@@ -65,7 +66,8 @@ if 89 in compute_capabilities and nvcc_cuda_version < Version("11.8"):
66
compute_capabilities.add(80)
67
if 90 in compute_capabilities and nvcc_cuda_version < Version("11.8"):
68
raise RuntimeError(
68
- "CUDA 11.8 or higher is required for GPUs with compute capability 9.0.")
69
+ "CUDA 11.8 or higher is required for GPUs with compute capability 9.0."
70
+ )
71
72
# If no GPU is available, add all supported compute capabilities.
73
if not compute_capabilities:
@@ -78,7 +80,9 @@ if not compute_capabilities:
80
81
# Add target compute capabilities to NVCC flags.
82
for capability in compute_capabilities:
81
- NVCC_FLAGS += ["-gencode", f"arch=compute_{capability},code=sm_{capability}"]
83
+ NVCC_FLAGS += [
84
+ "-gencode", f"arch=compute_{capability},code=sm_{capability}"
85
+ ]
86
87
# Use NVCC threads to parallelize the build.
88
if nvcc_cuda_version >= Version("11.2"):
@@ -91,7 +95,10 @@ ext_modules = []
95
cache_extension = CUDAExtension(
96
name="vllm.cache_ops",
97
sources=["csrc/cache.cpp", "csrc/cache_kernels.cu"],
94
- extra_compile_args={"cxx": CXX_FLAGS, "nvcc": NVCC_FLAGS},
98
+ extra_compile_args={
99
+ "cxx": CXX_FLAGS,
100
+ "nvcc": NVCC_FLAGS,
101
+ },
102
)
103
ext_modules.append(cache_extension)
104
@@ -99,7 +106,10 @@ ext_modules.append(cache_extension)
106
attention_extension = CUDAExtension(
107
name="vllm.attention_ops",
108
sources=["csrc/attention.cpp", "csrc/attention/attention_kernels.cu"],
102
- extra_compile_args={"cxx": CXX_FLAGS, "nvcc": NVCC_FLAGS},
109
+ extra_compile_args={
110
+ "cxx": CXX_FLAGS,
111
+ "nvcc": NVCC_FLAGS,
112
+ },
113
)
114
ext_modules.append(attention_extension)
115
@@ -107,7 +117,10 @@ ext_modules.append(attention_extension)
117
positional_encoding_extension = CUDAExtension(
118
name="vllm.pos_encoding_ops",
119
sources=["csrc/pos_encoding.cpp", "csrc/pos_encoding_kernels.cu"],
110
- extra_compile_args={"cxx": CXX_FLAGS, "nvcc": NVCC_FLAGS},
120
+ extra_compile_args={
121
+ "cxx": CXX_FLAGS,
122
+ "nvcc": NVCC_FLAGS,
123
+ },
124
)
125
ext_modules.append(positional_encoding_extension)
126
@@ -115,7 +128,10 @@ ext_modules.append(positional_encoding_extension)
128
layernorm_extension = CUDAExtension(
129
name="vllm.layernorm_ops",
130
sources=["csrc/layernorm.cpp", "csrc/layernorm_kernels.cu"],
118
- extra_compile_args={"cxx": CXX_FLAGS, "nvcc": NVCC_FLAGS},
131
+ extra_compile_args={
132
+ "cxx": CXX_FLAGS,
133
+ "nvcc": NVCC_FLAGS,
134
+ },
135
)
136
ext_modules.append(layernorm_extension)
137
@@ -123,7 +139,10 @@ ext_modules.append(layernorm_extension)
139
activation_extension = CUDAExtension(
140
name="vllm.activation_ops",
141
sources=["csrc/activation.cpp", "csrc/activation_kernels.cu"],
126
- extra_compile_args={"cxx": CXX_FLAGS, "nvcc": NVCC_FLAGS},
142
+ extra_compile_args={
143
+ "cxx": CXX_FLAGS,
144
+ "nvcc": NVCC_FLAGS,
145
+ },
146
)
147
ext_modules.append(activation_extension)
148
@@ -138,8 +157,8 @@ def find_version(filepath: str):
157
Adapted from https://github.com/ray-project/ray/blob/0b190ee1160eeca9796bc091e07eaebf4c85b511/python/setup.py
158
"""
159
with open(filepath) as fp:
141
- version_match = re.search(
142
- r"^__version__ = ['\"]([^'\"]*)['\"]", fp.read(), re.M)
160
+ version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]",
161
+ fp.read(), re.M)
162
if version_match:
163
return version_match.group(1)
164
raise RuntimeError("Unable to find version string.")
@@ -162,7 +181,8 @@ setuptools.setup(
181
version=find_version(get_path("vllm", "__init__.py")),
182
author="vLLM Team",
183
license="Apache 2.0",
165
- description="A high-throughput and memory-efficient inference and serving engine for LLMs",
184
+ description=("A high-throughput and memory-efficient inference and "
185
+ "serving engine for LLMs"),
186
long_description=read_readme(),
187
long_description_content_type="text/markdown",
188
url="https://github.com/vllm-project/vllm",
@@ -174,11 +194,12 @@ setuptools.setup(
194
"Programming Language :: Python :: 3.8",
195
"Programming Language :: Python :: 3.9",
196
"Programming Language :: Python :: 3.10",
197
+ "Programming Language :: Python :: 3.11",
198
"License :: OSI Approved :: Apache Software License",
199
"Topic :: Scientific/Engineering :: Artificial Intelligence",
200
],
180
- packages=setuptools.find_packages(
181
- exclude=("assets", "benchmarks", "csrc", "docs", "examples", "tests")),
201
+ packages=setuptools.find_packages(exclude=("benchmarks", "csrc", "docs",
202
+ "examples", "tests")),
203
python_requires=">=3.8",
204
install_requires=get_requirements(),
205
ext_modules=ext_modules,