@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d0740dff1b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Fix error message on `TORCH_CUDA_ARCH_LIST` (#1239)

Co-authored-by: Yunfeng Bai <yunfeng.bai@scale.com>

Woosuk Kwon committed Oct 14, 2023 at 14:47 UTC d0740dff1b3ab06860316d98c937a3d4e143f25f
1 file changed +25 -12
setup.py
+25 -12
@@ -13,7 +13,7 @@ from torch.utils.cpp_extension import BuildExtension, CUDAExtension, CUDA_HOME
13 ROOT_DIR = os.path.dirname(__file__)
14
15 # Supported NVIDIA GPU architectures.
16 -SUPPORTED_ARCHS = ["7.0", "7.5", "8.0", "8.6", "8.9", "9.0"]
16 +SUPPORTED_ARCHS = {"7.0", "7.5", "8.0", "8.6", "8.9", "9.0"}
17
18 # Compiler flags.
19 CXX_FLAGS = ["-g", "-O2", "-std=c++17"]
@@ -49,19 +49,32 @@ def get_torch_arch_list() -> Set[str]:
49 # and executed on the 8.6 or newer architectures. While the PTX code will
50 # not give the best performance on the newer architectures, it provides
51 # forward compatibility.
52 - valid_arch_strs = SUPPORTED_ARCHS + [s + "+PTX" for s in SUPPORTED_ARCHS]
53 - arch_list = os.environ.get("TORCH_CUDA_ARCH_LIST", None)
54 - if arch_list is None:
52 + env_arch_list = os.environ.get("TORCH_CUDA_ARCH_LIST", None)
53 + if env_arch_list is None:
54 return set()
55
56 # List are separated by ; or space.
58 - arch_list = arch_list.replace(" ", ";").split(";")
59 - for arch in arch_list:
60 - if arch not in valid_arch_strs:
61 - raise ValueError(
62 - f"Unsupported CUDA arch ({arch}). "
63 - f"Valid CUDA arch strings are: {valid_arch_strs}.")
64 - return set(arch_list)
57 + torch_arch_list = set(env_arch_list.replace(" ", ";").split(";"))
58 + if not torch_arch_list:
59 + return set()
60 +
61 + # Filter out the invalid architectures and print a warning.
62 + valid_archs = SUPPORTED_ARCHS.union({s + "+PTX" for s in SUPPORTED_ARCHS})
63 + arch_list = torch_arch_list.intersection(valid_archs)
64 + # If none of the specified architectures are valid, raise an error.
65 + if not arch_list:
66 + raise RuntimeError(
67 + "None of the CUDA architectures in `TORCH_CUDA_ARCH_LIST` env "
68 + f"variable ({env_arch_list}) is supported. "
69 + f"Supported CUDA architectures are: {valid_archs}.")
70 + invalid_arch_list = torch_arch_list - valid_archs
71 + if invalid_arch_list:
72 + warnings.warn(
73 + f"Unsupported CUDA architectures ({invalid_arch_list}) are "
74 + "excluded from the `TORCH_CUDA_ARCH_LIST` env variable "
75 + f"({env_arch_list}). Supported CUDA architectures are: "
76 + f"{valid_archs}.")
77 + return arch_list
78
79
80 # First, check the TORCH_CUDA_ARCH_LIST environment variable.
@@ -81,7 +94,7 @@ nvcc_cuda_version = get_nvcc_cuda_version(CUDA_HOME)
94 if not compute_capabilities:
95 # If no GPU is specified nor available, add all supported architectures
96 # based on the NVCC CUDA version.
84 - compute_capabilities = set(SUPPORTED_ARCHS)
97 + compute_capabilities = SUPPORTED_ARCHS.copy()
98 if nvcc_cuda_version < Version("11.1"):
99 compute_capabilities.remove("8.6")
100 if nvcc_cuda_version < Version("11.8"):