A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
[ROCm] Fixup arch checks for ROCM (#2627)
Douglas Lehr committed
Feb 5, 2024 at 16:59 UTC
2ccee3def6d8532afaf6fcd351b228ec1dfd6013
2 files changed
+56
-37
Dockerfile.rocm
-3
@@ -10,9 +10,6 @@ RUN echo "Base image is $BASE_IMAGE"
10
# BASE_IMAGE for ROCm_5.7: "rocm/pytorch:rocm5.7_ubuntu22.04_py3.10_pytorch_2.0.1"
11
# BASE_IMAGE for ROCm_6.0: "rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1"
12
13
-# this does not always work for all rocm versions
14
-RUN LLVM_GFX_ARCH=$(/opt/rocm/llvm/bin/amdgpu-offload-arch) && \
15
- echo "LLVM_GFX_ARCH is $LLVM_GFX_ARCH"
13
14
ARG FA_GFX_ARCHS="gfx90a;gfx942"
15
RUN echo "FA_GFX_ARCHS is $FA_GFX_ARCHS"
setup.py
+56
-34
@@ -19,7 +19,7 @@ MAIN_CUDA_VERSION = "12.1"
19
20
# Supported NVIDIA GPU architectures.
21
NVIDIA_SUPPORTED_ARCHS = {"7.0", "7.5", "8.0", "8.6", "8.9", "9.0"}
22
-ROCM_SUPPORTED_ARCHS = {"gfx90a", "gfx908", "gfx906", "gfx1030", "gfx1100"}
22
+ROCM_SUPPORTED_ARCHS = {"gfx90a", "gfx942"}
23
# SUPPORTED_ARCHS = NVIDIA_SUPPORTED_ARCHS.union(ROCM_SUPPORTED_ARCHS)
24
25
@@ -63,22 +63,6 @@ CXX_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]
63
NVCC_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]
64
65
66
-def get_amdgpu_offload_arch():
67
- command = "/opt/rocm/llvm/bin/amdgpu-offload-arch"
68
- try:
69
- output = subprocess.check_output([command])
70
- return output.decode('utf-8').strip()
71
- except subprocess.CalledProcessError as e:
72
- error_message = f"Error: {e}"
73
- raise RuntimeError(error_message) from e
74
- except FileNotFoundError as e:
75
- # If the command is not found, print an error message
76
- error_message = f"The command {command} was not found."
77
- raise RuntimeError(error_message) from e
78
-
79
- return None
80
-
81
-
66
def get_hipcc_rocm_version():
67
# Run the hipcc --version command
68
result = subprocess.run(['hipcc', '--version'],
@@ -138,6 +122,50 @@ def get_nvcc_cuda_version(cuda_dir: str) -> Version:
122
return nvcc_cuda_version
123
124
125
+def get_pytorch_rocm_arch() -> Set[str]:
126
+ """Get the cross section of Pytorch,and vllm supported gfx arches
127
+
128
+ ROCM can get the supported gfx architectures in one of two ways
129
+ Either through the PYTORCH_ROCM_ARCH env var, or output from
130
+ rocm_agent_enumerator.
131
+
132
+ In either case we can generate a list of supported arch's and
133
+ cross reference with VLLM's own ROCM_SUPPORTED_ARCHs.
134
+ """
135
+ env_arch_list = os.environ.get("PYTORCH_ROCM_ARCH", None)
136
+
137
+ # If we don't have PYTORCH_ROCM_ARCH specified pull the list from rocm_agent_enumerator
138
+ if env_arch_list is None:
139
+ command = "rocm_agent_enumerator"
140
+ env_arch_list = subprocess.check_output([command]).decode('utf-8')\
141
+ .strip().replace("\n", ";")
142
+ arch_source_str = "rocm_agent_enumerator"
143
+ else:
144
+ arch_source_str = "PYTORCH_ROCM_ARCH env variable"
145
+
146
+ # List are separated by ; or space.
147
+ pytorch_rocm_arch = set(env_arch_list.replace(" ", ";").split(";"))
148
+
149
+ # Filter out the invalid architectures and print a warning.
150
+ arch_list = pytorch_rocm_arch.intersection(ROCM_SUPPORTED_ARCHS)
151
+
152
+ # If none of the specified architectures are valid, raise an error.
153
+ if not arch_list:
154
+ raise RuntimeError(
155
+ f"None of the ROCM architectures in {arch_source_str} "
156
+ f"({env_arch_list}) is supported. "
157
+ f"Supported ROCM architectures are: {ROCM_SUPPORTED_ARCHS}.")
158
+ invalid_arch_list = pytorch_rocm_arch - ROCM_SUPPORTED_ARCHS
159
+ if invalid_arch_list:
160
+ warnings.warn(
161
+ f"Unsupported ROCM architectures ({invalid_arch_list}) are "
162
+ f"excluded from the {arch_source_str} output "
163
+ f"({env_arch_list}). Supported ROCM architectures are: "
164
+ f"{ROCM_SUPPORTED_ARCHS}.",
165
+ stacklevel=2)
166
+ return arch_list
167
+
168
+
169
def get_torch_arch_list() -> Set[str]:
170
# TORCH_CUDA_ARCH_LIST can have one or more architectures,
171
# e.g. "8.0" or "7.5,8.0,8.6+PTX". Here, the "8.6+PTX" option asks the
@@ -162,22 +190,27 @@ def get_torch_arch_list() -> Set[str]:
190
# If none of the specified architectures are valid, raise an error.
191
if not arch_list:
192
raise RuntimeError(
165
- "None of the CUDA/ROCM architectures in `TORCH_CUDA_ARCH_LIST` env "
193
+ "None of the CUDA architectures in `TORCH_CUDA_ARCH_LIST` env "
194
f"variable ({env_arch_list}) is supported. "
167
- f"Supported CUDA/ROCM architectures are: {valid_archs}.")
195
+ f"Supported CUDA architectures are: {valid_archs}.")
196
invalid_arch_list = torch_arch_list - valid_archs
197
if invalid_arch_list:
198
warnings.warn(
171
- f"Unsupported CUDA/ROCM architectures ({invalid_arch_list}) are "
199
+ f"Unsupported CUDA architectures ({invalid_arch_list}) are "
200
"excluded from the `TORCH_CUDA_ARCH_LIST` env variable "
173
- f"({env_arch_list}). Supported CUDA/ROCM architectures are: "
201
+ f"({env_arch_list}). Supported CUDA architectures are: "
202
f"{valid_archs}.",
203
stacklevel=2)
204
return arch_list
205
206
179
-# First, check the TORCH_CUDA_ARCH_LIST environment variable.
180
-compute_capabilities = get_torch_arch_list()
207
+if _is_hip():
208
+ rocm_arches = get_pytorch_rocm_arch()
209
+ NVCC_FLAGS += ["--offload-arch=" + arch for arch in rocm_arches]
210
+else:
211
+ # First, check the TORCH_CUDA_ARCH_LIST environment variable.
212
+ compute_capabilities = get_torch_arch_list()
213
+
214
if _is_cuda() and not compute_capabilities:
215
# If TORCH_CUDA_ARCH_LIST is not defined or empty, target all available
216
# GPUs on the current machine.
@@ -286,17 +319,6 @@ if _is_cuda():
319
"nvcc": NVCC_FLAGS_PUNICA,
320
},
321
))
289
-elif _is_hip():
290
- amd_archs = os.getenv("GPU_ARCHS")
291
- if amd_archs is None:
292
- amd_archs = get_amdgpu_offload_arch()
293
- for arch in amd_archs.split(";"):
294
- if arch not in ROCM_SUPPORTED_ARCHS:
295
- raise RuntimeError(
296
- f"Only the following arch is supported: {ROCM_SUPPORTED_ARCHS}"
297
- f"amdgpu_arch_found: {arch}")
298
- NVCC_FLAGS += [f"--offload-arch={arch}"]
299
-
322
elif _is_neuron():
323
neuronxcc_version = get_neuronxcc_version()
324