@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a41c20435e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Add compute capability 8.9 to default targets (#829)

Woosuk Kwon committed Aug 23, 2023 at 07:28 UTC a41c20435eab2abf3584144a056deed9ebe9f18e
1 file changed +10 -1
setup.py
+10 -1
@@ -22,7 +22,7 @@ NVCC_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]
22
23 if CUDA_HOME is None:
24 raise RuntimeError(
25 - f"Cannot find CUDA_HOME. CUDA must be available in order to build the package.")
25 + f"Cannot find CUDA_HOME. CUDA must be available to build the package.")
26
27
28 def get_nvcc_cuda_version(cuda_dir: str) -> Version:
@@ -55,6 +55,14 @@ if nvcc_cuda_version < Version("11.0"):
55 if 86 in compute_capabilities and nvcc_cuda_version < Version("11.1"):
56 raise RuntimeError(
57 "CUDA 11.1 or higher is required for GPUs with compute capability 8.6.")
58 +if 89 in compute_capabilities and nvcc_cuda_version < Version("11.8"):
59 + # CUDA 11.8 is required to generate the code targeting compute capability 8.9.
60 + # However, GPUs with compute capability 8.9 can also run the code generated by
61 + # the previous versions of CUDA 11 and targeting compute capability 8.0.
62 + # Therefore, if CUDA 11.8 is not available, we target compute capability 8.0
63 + # instead of 8.9.
64 + compute_capabilities.remove(89)
65 + compute_capabilities.add(80)
66 if 90 in compute_capabilities and nvcc_cuda_version < Version("11.8"):
67 raise RuntimeError(
68 "CUDA 11.8 or higher is required for GPUs with compute capability 9.0.")
@@ -65,6 +73,7 @@ if not compute_capabilities:
73 if nvcc_cuda_version >= Version("11.1"):
74 compute_capabilities.add(86)
75 if nvcc_cuda_version >= Version("11.8"):
76 + compute_capabilities.add(89)
77 compute_capabilities.add(90)
78
79 # Add target compute capabilities to NVCC flags.