@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2ccee3def6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[ROCm] Fixup arch checks for ROCM (#2627)

Douglas Lehr committed Feb 5, 2024 at 16:59 UTC 2ccee3def6d8532afaf6fcd351b228ec1dfd6013
2 files changed +56 -37
Dockerfile.rocm
-3
@@ -10,9 +10,6 @@ RUN echo "Base image is $BASE_IMAGE"
10 # BASE_IMAGE for ROCm_5.7: "rocm/pytorch:rocm5.7_ubuntu22.04_py3.10_pytorch_2.0.1"
11 # BASE_IMAGE for ROCm_6.0: "rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1"
12
13 -# this does not always work for all rocm versions
14 -RUN LLVM_GFX_ARCH=$(/opt/rocm/llvm/bin/amdgpu-offload-arch) && \
15 - echo "LLVM_GFX_ARCH is $LLVM_GFX_ARCH"
13
14 ARG FA_GFX_ARCHS="gfx90a;gfx942"
15 RUN echo "FA_GFX_ARCHS is $FA_GFX_ARCHS"
setup.py
+56 -34
@@ -19,7 +19,7 @@ MAIN_CUDA_VERSION = "12.1"
19
20 # Supported NVIDIA GPU architectures.
21 NVIDIA_SUPPORTED_ARCHS = {"7.0", "7.5", "8.0", "8.6", "8.9", "9.0"}
22 -ROCM_SUPPORTED_ARCHS = {"gfx90a", "gfx908", "gfx906", "gfx1030", "gfx1100"}
22 +ROCM_SUPPORTED_ARCHS = {"gfx90a", "gfx942"}
23 # SUPPORTED_ARCHS = NVIDIA_SUPPORTED_ARCHS.union(ROCM_SUPPORTED_ARCHS)
24
25
@@ -63,22 +63,6 @@ CXX_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]
63 NVCC_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]
64
65
66 -def get_amdgpu_offload_arch():
67 - command = "/opt/rocm/llvm/bin/amdgpu-offload-arch"
68 - try:
69 - output = subprocess.check_output([command])
70 - return output.decode('utf-8').strip()
71 - except subprocess.CalledProcessError as e:
72 - error_message = f"Error: {e}"
73 - raise RuntimeError(error_message) from e
74 - except FileNotFoundError as e:
75 - # If the command is not found, print an error message
76 - error_message = f"The command {command} was not found."
77 - raise RuntimeError(error_message) from e
78 -
79 - return None
80 -
81 -
66 def get_hipcc_rocm_version():
67 # Run the hipcc --version command
68 result = subprocess.run(['hipcc', '--version'],
@@ -138,6 +122,50 @@ def get_nvcc_cuda_version(cuda_dir: str) -> Version:
122 return nvcc_cuda_version
123
124
125 +def get_pytorch_rocm_arch() -> Set[str]:
126 + """Get the cross section of Pytorch,and vllm supported gfx arches
127 +
128 + ROCM can get the supported gfx architectures in one of two ways
129 + Either through the PYTORCH_ROCM_ARCH env var, or output from
130 + rocm_agent_enumerator.
131 +
132 + In either case we can generate a list of supported arch's and
133 + cross reference with VLLM's own ROCM_SUPPORTED_ARCHs.
134 + """
135 + env_arch_list = os.environ.get("PYTORCH_ROCM_ARCH", None)
136 +
137 + # If we don't have PYTORCH_ROCM_ARCH specified pull the list from rocm_agent_enumerator
138 + if env_arch_list is None:
139 + command = "rocm_agent_enumerator"
140 + env_arch_list = subprocess.check_output([command]).decode('utf-8')\
141 + .strip().replace("\n", ";")
142 + arch_source_str = "rocm_agent_enumerator"
143 + else:
144 + arch_source_str = "PYTORCH_ROCM_ARCH env variable"
145 +
146 + # List are separated by ; or space.
147 + pytorch_rocm_arch = set(env_arch_list.replace(" ", ";").split(";"))
148 +
149 + # Filter out the invalid architectures and print a warning.
150 + arch_list = pytorch_rocm_arch.intersection(ROCM_SUPPORTED_ARCHS)
151 +
152 + # If none of the specified architectures are valid, raise an error.
153 + if not arch_list:
154 + raise RuntimeError(
155 + f"None of the ROCM architectures in {arch_source_str} "
156 + f"({env_arch_list}) is supported. "
157 + f"Supported ROCM architectures are: {ROCM_SUPPORTED_ARCHS}.")
158 + invalid_arch_list = pytorch_rocm_arch - ROCM_SUPPORTED_ARCHS
159 + if invalid_arch_list:
160 + warnings.warn(
161 + f"Unsupported ROCM architectures ({invalid_arch_list}) are "
162 + f"excluded from the {arch_source_str} output "
163 + f"({env_arch_list}). Supported ROCM architectures are: "
164 + f"{ROCM_SUPPORTED_ARCHS}.",
165 + stacklevel=2)
166 + return arch_list
167 +
168 +
169 def get_torch_arch_list() -> Set[str]:
170 # TORCH_CUDA_ARCH_LIST can have one or more architectures,
171 # e.g. "8.0" or "7.5,8.0,8.6+PTX". Here, the "8.6+PTX" option asks the
@@ -162,22 +190,27 @@ def get_torch_arch_list() -> Set[str]:
190 # If none of the specified architectures are valid, raise an error.
191 if not arch_list:
192 raise RuntimeError(
165 - "None of the CUDA/ROCM architectures in `TORCH_CUDA_ARCH_LIST` env "
193 + "None of the CUDA architectures in `TORCH_CUDA_ARCH_LIST` env "
194 f"variable ({env_arch_list}) is supported. "
167 - f"Supported CUDA/ROCM architectures are: {valid_archs}.")
195 + f"Supported CUDA architectures are: {valid_archs}.")
196 invalid_arch_list = torch_arch_list - valid_archs
197 if invalid_arch_list:
198 warnings.warn(
171 - f"Unsupported CUDA/ROCM architectures ({invalid_arch_list}) are "
199 + f"Unsupported CUDA architectures ({invalid_arch_list}) are "
200 "excluded from the `TORCH_CUDA_ARCH_LIST` env variable "
173 - f"({env_arch_list}). Supported CUDA/ROCM architectures are: "
201 + f"({env_arch_list}). Supported CUDA architectures are: "
202 f"{valid_archs}.",
203 stacklevel=2)
204 return arch_list
205
206
179 -# First, check the TORCH_CUDA_ARCH_LIST environment variable.
180 -compute_capabilities = get_torch_arch_list()
207 +if _is_hip():
208 + rocm_arches = get_pytorch_rocm_arch()
209 + NVCC_FLAGS += ["--offload-arch=" + arch for arch in rocm_arches]
210 +else:
211 + # First, check the TORCH_CUDA_ARCH_LIST environment variable.
212 + compute_capabilities = get_torch_arch_list()
213 +
214 if _is_cuda() and not compute_capabilities:
215 # If TORCH_CUDA_ARCH_LIST is not defined or empty, target all available
216 # GPUs on the current machine.
@@ -286,17 +319,6 @@ if _is_cuda():
319 "nvcc": NVCC_FLAGS_PUNICA,
320 },
321 ))
289 -elif _is_hip():
290 - amd_archs = os.getenv("GPU_ARCHS")
291 - if amd_archs is None:
292 - amd_archs = get_amdgpu_offload_arch()
293 - for arch in amd_archs.split(";"):
294 - if arch not in ROCM_SUPPORTED_ARCHS:
295 - raise RuntimeError(
296 - f"Only the following arch is supported: {ROCM_SUPPORTED_ARCHS}"
297 - f"amdgpu_arch_found: {arch}")
298 - NVCC_FLAGS += [f"--offload-arch={arch}"]
299 -
322 elif _is_neuron():
323 neuronxcc_version = get_neuronxcc_version()
324