@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a425bd9a9a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

[Setup] Enable `TORCH_CUDA_ARCH_LIST` for selecting target GPUs (#1074)

Woosuk Kwon committed Sep 26, 2023 at 10:21 UTC a425bd9a9af6b32e0e93b2787d6682a1a5133983
1 file changed +73 -38
setup.py
+73 -38
@@ -3,6 +3,7 @@ import os
3 import re
4 import subprocess
5 from typing import List, Set
6 +import warnings
7
8 from packaging.version import parse, Version
9 import setuptools
@@ -11,6 +12,9 @@ from torch.utils.cpp_extension import BuildExtension, CUDAExtension, CUDA_HOME
12
13 ROOT_DIR = os.path.dirname(__file__)
14
15 +# Supported NVIDIA GPU architectures.
16 +SUPPORTED_ARCHS = ["7.0", "7.5", "8.0", "8.6", "8.9", "9.0"]
17 +
18 # Compiler flags.
19 CXX_FLAGS = ["-g", "-O2", "-std=c++17"]
20 # TODO(woosuk): Should we use -O3?
@@ -38,51 +42,82 @@ def get_nvcc_cuda_version(cuda_dir: str) -> Version:
42 return nvcc_cuda_version
43
44
41 -# Collect the compute capabilities of all available GPUs.
42 -device_count = torch.cuda.device_count()
43 -compute_capabilities: Set[int] = set()
44 -for i in range(device_count):
45 - major, minor = torch.cuda.get_device_capability(i)
46 - if major < 7:
47 - raise RuntimeError(
48 - "GPUs with compute capability less than 7.0 are not supported.")
49 - compute_capabilities.add(major * 10 + minor)
45 +def get_torch_arch_list() -> Set[str]:
46 + # TORCH_CUDA_ARCH_LIST can have one or more architectures,
47 + # e.g. "8.0" or "7.5,8.0,8.6+PTX". Here, the "8.6+PTX" option asks the
48 + # compiler to additionally include PTX code that can be runtime-compiled
49 + # and executed on the 8.6 or newer architectures. While the PTX code will
50 + # not give the best performance on the newer architectures, it provides
51 + # forward compatibility.
52 + valid_arch_strs = SUPPORTED_ARCHS + [s + "+PTX" for s in SUPPORTED_ARCHS]
53 + arch_list = os.environ.get("TORCH_CUDA_ARCH_LIST", None)
54 + if arch_list is None:
55 + return set()
56 +
57 + # List are separated by ; or space.
58 + arch_list = arch_list.replace(" ", ";").split(";")
59 + for arch in arch_list:
60 + if arch not in valid_arch_strs:
61 + raise ValueError(
62 + f"Unsupported CUDA arch ({arch}). "
63 + f"Valid CUDA arch strings are: {valid_arch_strs}.")
64 + return set(arch_list)
65 +
66 +
67 +# First, check the TORCH_CUDA_ARCH_LIST environment variable.
68 +compute_capabilities = get_torch_arch_list()
69 +if not compute_capabilities:
70 + # If TORCH_CUDA_ARCH_LIST is not defined or empty, target all available
71 + # GPUs on the current machine.
72 + device_count = torch.cuda.device_count()
73 + for i in range(device_count):
74 + major, minor = torch.cuda.get_device_capability(i)
75 + if major < 7:
76 + raise RuntimeError(
77 + "GPUs with compute capability below 7.0 are not supported.")
78 + compute_capabilities.add(f"{major}.{minor}")
79
51 -# Validate the NVCC CUDA version.
80 nvcc_cuda_version = get_nvcc_cuda_version(CUDA_HOME)
81 +if not compute_capabilities:
82 + # If no GPU is specified nor available, add all supported architectures
83 + # based on the NVCC CUDA version.
84 + compute_capabilities = set(SUPPORTED_ARCHS)
85 + if nvcc_cuda_version < Version("11.1"):
86 + compute_capabilities.remove("8.6")
87 + if nvcc_cuda_version < Version("11.8"):
88 + compute_capabilities.remove("8.9")
89 + compute_capabilities.remove("9.0")
90 +
91 +# Validate the NVCC CUDA version.
92 if nvcc_cuda_version < Version("11.0"):
93 raise RuntimeError("CUDA 11.0 or higher is required to build the package.")
55 -if 86 in compute_capabilities and nvcc_cuda_version < Version("11.1"):
56 - raise RuntimeError(
57 - "CUDA 11.1 or higher is required for GPUs with compute capability 8.6."
58 - )
59 -if 89 in compute_capabilities and nvcc_cuda_version < Version("11.8"):
60 - # CUDA 11.8 is required to generate the code targeting compute capability 8.9.
61 - # However, GPUs with compute capability 8.9 can also run the code generated by
62 - # the previous versions of CUDA 11 and targeting compute capability 8.0.
63 - # Therefore, if CUDA 11.8 is not available, we target compute capability 8.0
64 - # instead of 8.9.
65 - compute_capabilities.remove(89)
66 - compute_capabilities.add(80)
67 -if 90 in compute_capabilities and nvcc_cuda_version < Version("11.8"):
68 - raise RuntimeError(
69 - "CUDA 11.8 or higher is required for GPUs with compute capability 9.0."
70 - )
71 -
72 -# If no GPU is available, add all supported compute capabilities.
73 -if not compute_capabilities:
74 - compute_capabilities = {70, 75, 80}
75 - if nvcc_cuda_version >= Version("11.1"):
76 - compute_capabilities.add(86)
77 - if nvcc_cuda_version >= Version("11.8"):
78 - compute_capabilities.add(89)
79 - compute_capabilities.add(90)
94 +if nvcc_cuda_version < Version("11.1"):
95 + if any(cc.startswith("8.6") for cc in compute_capabilities):
96 + raise RuntimeError(
97 + "CUDA 11.1 or higher is required for compute capability 8.6.")
98 +if nvcc_cuda_version < Version("11.8"):
99 + if any(cc.startswith("8.9") for cc in compute_capabilities):
100 + # CUDA 11.8 is required to generate the code targeting compute capability 8.9.
101 + # However, GPUs with compute capability 8.9 can also run the code generated by
102 + # the previous versions of CUDA 11 and targeting compute capability 8.0.
103 + # Therefore, if CUDA 11.8 is not available, we target compute capability 8.0
104 + # instead of 8.9.
105 + warnings.warn(
106 + "CUDA 11.8 or higher is required for compute capability 8.9. "
107 + "Targeting compute capability 8.0 instead.")
108 + compute_capabilities = set(cc for cc in compute_capabilities
109 + if not cc.startswith("8.9"))
110 + compute_capabilities.add("8.0+PTX")
111 + if any(cc.startswith("9.0") for cc in compute_capabilities):
112 + raise RuntimeError(
113 + "CUDA 11.8 or higher is required for compute capability 9.0.")
114
115 # Add target compute capabilities to NVCC flags.
116 for capability in compute_capabilities:
83 - NVCC_FLAGS += [
84 - "-gencode", f"arch=compute_{capability},code=sm_{capability}"
85 - ]
117 + num = capability[0] + capability[2]
118 + NVCC_FLAGS += ["-gencode", f"arch=compute_{num},code=sm_{num}"]
119 + if capability.endswith("+PTX"):
120 + NVCC_FLAGS += ["-gencode", f"arch=compute_{num},code=compute_{num}"]
121
122 # Use NVCC threads to parallelize the build.
123 if nvcc_cuda_version >= Version("11.2"):