A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
set default coompute capability according to cuda version (#773)
Xudong Zhang committed
Aug 22, 2023 at 07:05 UTC
65fc1c3127f4df8a99a0a5db4a205dbfcc673713
1 file changed
+12
-6
setup.py
+12
-6
@@ -47,12 +47,6 @@ for i in range(device_count):
47
raise RuntimeError(
48
"GPUs with compute capability less than 7.0 are not supported.")
49
compute_capabilities.add(major * 10 + minor)
50
-# If no GPU is available, add all supported compute capabilities.
51
-if not compute_capabilities:
52
- compute_capabilities = {70, 75, 80, 86, 90}
53
-# Add target compute capabilities to NVCC flags.
54
-for capability in compute_capabilities:
55
- NVCC_FLAGS += ["-gencode", f"arch=compute_{capability},code=sm_{capability}"]
50
51
# Validate the NVCC CUDA version.
52
nvcc_cuda_version = get_nvcc_cuda_version(CUDA_HOME)
@@ -65,6 +59,18 @@ if 90 in compute_capabilities and nvcc_cuda_version < Version("11.8"):
59
raise RuntimeError(
60
"CUDA 11.8 or higher is required for GPUs with compute capability 9.0.")
61
62
+# If no GPU is available, add all supported compute capabilities.
63
+if not compute_capabilities:
64
+ compute_capabilities = {70, 75, 80}
65
+ if nvcc_cuda_version >= Version("11.1"):
66
+ compute_capabilities.add(86)
67
+ if nvcc_cuda_version >= Version("11.8"):
68
+ compute_capabilities.add(90)
69
+
70
+# Add target compute capabilities to NVCC flags.
71
+for capability in compute_capabilities:
72
+ NVCC_FLAGS += ["-gencode", f"arch=compute_{capability},code=sm_{capability}"]
73
+
74
# Use NVCC threads to parallelize the build.
75
if nvcc_cuda_version >= Version("11.2"):
76
num_threads = min(os.cpu_count(), 8)