@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 65fc1c3127
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

set default coompute capability according to cuda version (#773)

Xudong Zhang committed Aug 22, 2023 at 07:05 UTC 65fc1c3127f4df8a99a0a5db4a205dbfcc673713
1 file changed +12 -6
setup.py
+12 -6
@@ -47,12 +47,6 @@ for i in range(device_count):
47 raise RuntimeError(
48 "GPUs with compute capability less than 7.0 are not supported.")
49 compute_capabilities.add(major * 10 + minor)
50 -# If no GPU is available, add all supported compute capabilities.
51 -if not compute_capabilities:
52 - compute_capabilities = {70, 75, 80, 86, 90}
53 -# Add target compute capabilities to NVCC flags.
54 -for capability in compute_capabilities:
55 - NVCC_FLAGS += ["-gencode", f"arch=compute_{capability},code=sm_{capability}"]
50
51 # Validate the NVCC CUDA version.
52 nvcc_cuda_version = get_nvcc_cuda_version(CUDA_HOME)
@@ -65,6 +59,18 @@ if 90 in compute_capabilities and nvcc_cuda_version < Version("11.8"):
59 raise RuntimeError(
60 "CUDA 11.8 or higher is required for GPUs with compute capability 9.0.")
61
62 +# If no GPU is available, add all supported compute capabilities.
63 +if not compute_capabilities:
64 + compute_capabilities = {70, 75, 80}
65 + if nvcc_cuda_version >= Version("11.1"):
66 + compute_capabilities.add(86)
67 + if nvcc_cuda_version >= Version("11.8"):
68 + compute_capabilities.add(90)
69 +
70 +# Add target compute capabilities to NVCC flags.
71 +for capability in compute_capabilities:
72 + NVCC_FLAGS += ["-gencode", f"arch=compute_{capability},code=sm_{capability}"]
73 +
74 # Use NVCC threads to parallelize the build.
75 if nvcc_cuda_version >= Version("11.2"):
76 num_threads = min(os.cpu_count(), 8)