A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Add compute capability 8.9 to default targets (#829)
Woosuk Kwon committed
Aug 23, 2023 at 07:28 UTC
a41c20435eab2abf3584144a056deed9ebe9f18e
1 file changed
+10
-1
setup.py
+10
-1
@@ -22,7 +22,7 @@ NVCC_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]
22
23
if CUDA_HOME is None:
24
raise RuntimeError(
25
- f"Cannot find CUDA_HOME. CUDA must be available in order to build the package.")
25
+ f"Cannot find CUDA_HOME. CUDA must be available to build the package.")
26
27
28
def get_nvcc_cuda_version(cuda_dir: str) -> Version:
@@ -55,6 +55,14 @@ if nvcc_cuda_version < Version("11.0"):
55
if 86 in compute_capabilities and nvcc_cuda_version < Version("11.1"):
56
raise RuntimeError(
57
"CUDA 11.1 or higher is required for GPUs with compute capability 8.6.")
58
+if 89 in compute_capabilities and nvcc_cuda_version < Version("11.8"):
59
+ # CUDA 11.8 is required to generate the code targeting compute capability 8.9.
60
+ # However, GPUs with compute capability 8.9 can also run the code generated by
61
+ # the previous versions of CUDA 11 and targeting compute capability 8.0.
62
+ # Therefore, if CUDA 11.8 is not available, we target compute capability 8.0
63
+ # instead of 8.9.
64
+ compute_capabilities.remove(89)
65
+ compute_capabilities.add(80)
66
if 90 in compute_capabilities and nvcc_cuda_version < Version("11.8"):
67
raise RuntimeError(
68
"CUDA 11.8 or higher is required for GPUs with compute capability 9.0.")
@@ -65,6 +73,7 @@ if not compute_capabilities:
73
if nvcc_cuda_version >= Version("11.1"):
74
compute_capabilities.add(86)
75
if nvcc_cuda_version >= Version("11.8"):
76
+ compute_capabilities.add(89)
77
compute_capabilities.add(90)
78
79
# Add target compute capabilities to NVCC flags.