@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d6770d1f23
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Update setup.py (#1006)

Woosuk Kwon committed Sep 10, 2023 at 23:42 UTC d6770d1f23b642289a2f1462f2851a7be9d3cc83
1 file changed +35 -14
setup.py
+35 -14
@@ -22,7 +22,7 @@ NVCC_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]
22
23 if CUDA_HOME is None:
24 raise RuntimeError(
25 - f"Cannot find CUDA_HOME. CUDA must be available to build the package.")
25 + "Cannot find CUDA_HOME. CUDA must be available to build the package.")
26
27
28 def get_nvcc_cuda_version(cuda_dir: str) -> Version:
@@ -54,7 +54,8 @@ if nvcc_cuda_version < Version("11.0"):
54 raise RuntimeError("CUDA 11.0 or higher is required to build the package.")
55 if 86 in compute_capabilities and nvcc_cuda_version < Version("11.1"):
56 raise RuntimeError(
57 - "CUDA 11.1 or higher is required for GPUs with compute capability 8.6.")
57 + "CUDA 11.1 or higher is required for GPUs with compute capability 8.6."
58 + )
59 if 89 in compute_capabilities and nvcc_cuda_version < Version("11.8"):
60 # CUDA 11.8 is required to generate the code targeting compute capability 8.9.
61 # However, GPUs with compute capability 8.9 can also run the code generated by
@@ -65,7 +66,8 @@ if 89 in compute_capabilities and nvcc_cuda_version < Version("11.8"):
66 compute_capabilities.add(80)
67 if 90 in compute_capabilities and nvcc_cuda_version < Version("11.8"):
68 raise RuntimeError(
68 - "CUDA 11.8 or higher is required for GPUs with compute capability 9.0.")
69 + "CUDA 11.8 or higher is required for GPUs with compute capability 9.0."
70 + )
71
72 # If no GPU is available, add all supported compute capabilities.
73 if not compute_capabilities:
@@ -78,7 +80,9 @@ if not compute_capabilities:
80
81 # Add target compute capabilities to NVCC flags.
82 for capability in compute_capabilities:
81 - NVCC_FLAGS += ["-gencode", f"arch=compute_{capability},code=sm_{capability}"]
83 + NVCC_FLAGS += [
84 + "-gencode", f"arch=compute_{capability},code=sm_{capability}"
85 + ]
86
87 # Use NVCC threads to parallelize the build.
88 if nvcc_cuda_version >= Version("11.2"):
@@ -91,7 +95,10 @@ ext_modules = []
95 cache_extension = CUDAExtension(
96 name="vllm.cache_ops",
97 sources=["csrc/cache.cpp", "csrc/cache_kernels.cu"],
94 - extra_compile_args={"cxx": CXX_FLAGS, "nvcc": NVCC_FLAGS},
98 + extra_compile_args={
99 + "cxx": CXX_FLAGS,
100 + "nvcc": NVCC_FLAGS,
101 + },
102 )
103 ext_modules.append(cache_extension)
104
@@ -99,7 +106,10 @@ ext_modules.append(cache_extension)
106 attention_extension = CUDAExtension(
107 name="vllm.attention_ops",
108 sources=["csrc/attention.cpp", "csrc/attention/attention_kernels.cu"],
102 - extra_compile_args={"cxx": CXX_FLAGS, "nvcc": NVCC_FLAGS},
109 + extra_compile_args={
110 + "cxx": CXX_FLAGS,
111 + "nvcc": NVCC_FLAGS,
112 + },
113 )
114 ext_modules.append(attention_extension)
115
@@ -107,7 +117,10 @@ ext_modules.append(attention_extension)
117 positional_encoding_extension = CUDAExtension(
118 name="vllm.pos_encoding_ops",
119 sources=["csrc/pos_encoding.cpp", "csrc/pos_encoding_kernels.cu"],
110 - extra_compile_args={"cxx": CXX_FLAGS, "nvcc": NVCC_FLAGS},
120 + extra_compile_args={
121 + "cxx": CXX_FLAGS,
122 + "nvcc": NVCC_FLAGS,
123 + },
124 )
125 ext_modules.append(positional_encoding_extension)
126
@@ -115,7 +128,10 @@ ext_modules.append(positional_encoding_extension)
128 layernorm_extension = CUDAExtension(
129 name="vllm.layernorm_ops",
130 sources=["csrc/layernorm.cpp", "csrc/layernorm_kernels.cu"],
118 - extra_compile_args={"cxx": CXX_FLAGS, "nvcc": NVCC_FLAGS},
131 + extra_compile_args={
132 + "cxx": CXX_FLAGS,
133 + "nvcc": NVCC_FLAGS,
134 + },
135 )
136 ext_modules.append(layernorm_extension)
137
@@ -123,7 +139,10 @@ ext_modules.append(layernorm_extension)
139 activation_extension = CUDAExtension(
140 name="vllm.activation_ops",
141 sources=["csrc/activation.cpp", "csrc/activation_kernels.cu"],
126 - extra_compile_args={"cxx": CXX_FLAGS, "nvcc": NVCC_FLAGS},
142 + extra_compile_args={
143 + "cxx": CXX_FLAGS,
144 + "nvcc": NVCC_FLAGS,
145 + },
146 )
147 ext_modules.append(activation_extension)
148
@@ -138,8 +157,8 @@ def find_version(filepath: str):
157 Adapted from https://github.com/ray-project/ray/blob/0b190ee1160eeca9796bc091e07eaebf4c85b511/python/setup.py
158 """
159 with open(filepath) as fp:
141 - version_match = re.search(
142 - r"^__version__ = ['\"]([^'\"]*)['\"]", fp.read(), re.M)
160 + version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]",
161 + fp.read(), re.M)
162 if version_match:
163 return version_match.group(1)
164 raise RuntimeError("Unable to find version string.")
@@ -162,7 +181,8 @@ setuptools.setup(
181 version=find_version(get_path("vllm", "__init__.py")),
182 author="vLLM Team",
183 license="Apache 2.0",
165 - description="A high-throughput and memory-efficient inference and serving engine for LLMs",
184 + description=("A high-throughput and memory-efficient inference and "
185 + "serving engine for LLMs"),
186 long_description=read_readme(),
187 long_description_content_type="text/markdown",
188 url="https://github.com/vllm-project/vllm",
@@ -174,11 +194,12 @@ setuptools.setup(
194 "Programming Language :: Python :: 3.8",
195 "Programming Language :: Python :: 3.9",
196 "Programming Language :: Python :: 3.10",
197 + "Programming Language :: Python :: 3.11",
198 "License :: OSI Approved :: Apache Software License",
199 "Topic :: Scientific/Engineering :: Artificial Intelligence",
200 ],
180 - packages=setuptools.find_packages(
181 - exclude=("assets", "benchmarks", "csrc", "docs", "examples", "tests")),
201 + packages=setuptools.find_packages(exclude=("benchmarks", "csrc", "docs",
202 + "examples", "tests")),
203 python_requires=">=3.8",
204 install_requires=get_requirements(),
205 ext_modules=ext_modules,