@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1b7c791d60
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[ROCm] Fixes for GPTQ on ROCm (#2180)

kliuae committed Dec 19, 2023 at 02:41 UTC 1b7c791d60629453030de1600e756a8ba555455e
4 files changed +23 -16
csrc/quantization/gptq/q_gemm.cu
+10
@@ -28,6 +28,7 @@ namespace gptq {
28 #define DIVIDE(x, size) (((x) + (size) - 1) / (size))
29
30 #if defined(USE_ROCM)
31 +#include <hipblas/hipblas.h>
32 __host__ __forceinline__ hipblasStatus_t __compat_hipblasHgemm(hipblasHandle_t handle,
33 hipblasOperation_t transA,
34 hipblasOperation_t transB,
@@ -520,12 +521,21 @@ __global__ void gemm_half_q_half_alt_kernel(
521 zeros_tmp[tmp_k] = zero;
522 }
523 for (int m = 0; m < b_end; m++) {
524 +#ifndef USE_ROCM
525 res2 = {};
526 +#else
527 + res2.x = __half_as_ushort(__float2half(0));
528 + res2.y = __half_as_ushort(__float2half(0));
529 +#endif
530 res2 = __hfma2(__hfma2(deq2[(tmp >> 0) & 0xff][off], scales_tmp[0], zeros_tmp[0]), blockvec[m][k + 0], res2);
531 res2 = __hfma2(__hfma2(deq2[(tmp >> 8) & 0xff][off], scales_tmp[1], zeros_tmp[1]), blockvec[m][k + 1], res2);
532 res2 = __hfma2(__hfma2(deq2[(tmp >> 16) & 0xff][off], scales_tmp[2], zeros_tmp[2]), blockvec[m][k + 2], res2);
533 res2 = __hfma2(__hfma2(deq2[(tmp >> 24) & 0xff][off], scales_tmp[3], zeros_tmp[3]), blockvec[m][k + 3], res2);
534 +#ifndef USE_ROCM
535 res[m] = __hadd(res[m], __hadd(res2.x, res2.y));
536 +#else
537 + res[m] = __hadd(res[m], __hadd(__ushort_as_half(res2.x), __ushort_as_half(res2.y)));
538 +#endif
539 }
540 i += width;
541 k += 4;
docs/source/getting_started/amd-installation.rst
+1
@@ -116,6 +116,7 @@ Alternatively, if you plan to install vLLM-ROCm on a local machine or start from
116
117 - `ROCm <https://rocm.docs.amd.com/en/latest/deploy/linux/index.html>`_
118 - `Pytorch <https://pytorch.org/>`_
119 +- `hipBLAS <https://rocm.docs.amd.com/projects/hipBLAS/en/latest/install.html>`_
120
121 1. Install `flash attention for ROCm <https://github.com/ROCmSoftwarePlatform/flash-attention/tree/flash_attention_for_rocm>`_
122
setup.py
+1 -1
@@ -219,13 +219,13 @@ vllm_extension_sources = [
219 "csrc/activation_kernels.cu",
220 "csrc/layernorm_kernels.cu",
221 "csrc/quantization/squeezellm/quant_cuda_kernel.cu",
222 + "csrc/quantization/gptq/q_gemm.cu",
223 "csrc/cuda_utils_kernels.cu",
224 "csrc/pybind.cpp",
225 ]
226
227 if _is_cuda():
228 vllm_extension_sources.append("csrc/quantization/awq/gemm_kernels.cu")
228 - vllm_extension_sources.append("csrc/quantization/gptq/q_gemm.cu")
229
230 vllm_extension = CUDAExtension(
231 name="vllm._C",
vllm/config.py
+11 -15
@@ -112,24 +112,20 @@ class ModelConfig:
112 supported_load_format = [
113 "auto", "pt", "safetensors", "npcache", "dummy"
114 ]
115 - rocm_not_supported_load_format = ["safetensors"]
115 + rocm_not_supported_load_format = []
116 if load_format not in supported_load_format:
117 raise ValueError(
118 f"Unknown load format: {self.load_format}. Must be one of "
119 "'auto', 'pt', 'safetensors', 'npcache', or 'dummy'.")
120 - if is_hip():
121 - if load_format in ["safetensors"]:
122 - rocm_supported_load_format = [
123 - f for f in supported_load_format
124 - if (f not in rocm_not_supported_load_format)
125 - ]
126 - raise ValueError(
127 - f"load format \'{load_format}\' is not supported in ROCm. "
128 - f"Supported load format are "
129 - f"{rocm_supported_load_format}")
130 - # Force ROCm to load from pt weights if nothing specific is set
131 - if load_format == "auto":
132 - load_format = "pt"
120 + if is_hip() and load_format in rocm_not_supported_load_format:
121 + rocm_supported_load_format = [
122 + f for f in supported_load_format
123 + if (f not in rocm_not_supported_load_format)
124 + ]
125 + raise ValueError(
126 + f"load format \'{load_format}\' is not supported in ROCm. "
127 + f"Supported load format are "
128 + f"{rocm_supported_load_format}")
129
130 # TODO: Remove this check once HF updates the pt weights of Mixtral.
131 architectures = getattr(self.hf_config, "architectures", [])
@@ -149,7 +145,7 @@ class ModelConfig:
145
146 def _verify_quantization(self) -> None:
147 supported_quantization = ["awq", "gptq", "squeezellm"]
152 - rocm_not_supported_quantization = ["awq", "gptq"]
148 + rocm_not_supported_quantization = ["awq"]
149 if self.quantization is not None:
150 self.quantization = self.quantization.lower()
151