A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
[ROCm] Fixes for GPTQ on ROCm (#2180)
kliuae committed
Dec 19, 2023 at 02:41 UTC
1b7c791d60629453030de1600e756a8ba555455e
4 files changed
+23
-16
csrc/quantization/gptq/q_gemm.cu
+10
@@ -28,6 +28,7 @@ namespace gptq {
28
#define DIVIDE(x, size) (((x) + (size) - 1) / (size))
29
30
#if defined(USE_ROCM)
31
+#include <hipblas/hipblas.h>
32
__host__ __forceinline__ hipblasStatus_t __compat_hipblasHgemm(hipblasHandle_t handle,
33
hipblasOperation_t transA,
34
hipblasOperation_t transB,
@@ -520,12 +521,21 @@ __global__ void gemm_half_q_half_alt_kernel(
521
zeros_tmp[tmp_k] = zero;
522
}
523
for (int m = 0; m < b_end; m++) {
524
+#ifndef USE_ROCM
525
res2 = {};
526
+#else
527
+ res2.x = __half_as_ushort(__float2half(0));
528
+ res2.y = __half_as_ushort(__float2half(0));
529
+#endif
530
res2 = __hfma2(__hfma2(deq2[(tmp >> 0) & 0xff][off], scales_tmp[0], zeros_tmp[0]), blockvec[m][k + 0], res2);
531
res2 = __hfma2(__hfma2(deq2[(tmp >> 8) & 0xff][off], scales_tmp[1], zeros_tmp[1]), blockvec[m][k + 1], res2);
532
res2 = __hfma2(__hfma2(deq2[(tmp >> 16) & 0xff][off], scales_tmp[2], zeros_tmp[2]), blockvec[m][k + 2], res2);
533
res2 = __hfma2(__hfma2(deq2[(tmp >> 24) & 0xff][off], scales_tmp[3], zeros_tmp[3]), blockvec[m][k + 3], res2);
534
+#ifndef USE_ROCM
535
res[m] = __hadd(res[m], __hadd(res2.x, res2.y));
536
+#else
537
+ res[m] = __hadd(res[m], __hadd(__ushort_as_half(res2.x), __ushort_as_half(res2.y)));
538
+#endif
539
}
540
i += width;
541
k += 4;
docs/source/getting_started/amd-installation.rst
+1
@@ -116,6 +116,7 @@ Alternatively, if you plan to install vLLM-ROCm on a local machine or start from
116
117
- `ROCm <https://rocm.docs.amd.com/en/latest/deploy/linux/index.html>`_
118
- `Pytorch <https://pytorch.org/>`_
119
+- `hipBLAS <https://rocm.docs.amd.com/projects/hipBLAS/en/latest/install.html>`_
120
121
1. Install `flash attention for ROCm <https://github.com/ROCmSoftwarePlatform/flash-attention/tree/flash_attention_for_rocm>`_
122
setup.py
+1
-1
@@ -219,13 +219,13 @@ vllm_extension_sources = [
219
"csrc/activation_kernels.cu",
220
"csrc/layernorm_kernels.cu",
221
"csrc/quantization/squeezellm/quant_cuda_kernel.cu",
222
+ "csrc/quantization/gptq/q_gemm.cu",
223
"csrc/cuda_utils_kernels.cu",
224
"csrc/pybind.cpp",
225
]
226
227
if _is_cuda():
228
vllm_extension_sources.append("csrc/quantization/awq/gemm_kernels.cu")
228
- vllm_extension_sources.append("csrc/quantization/gptq/q_gemm.cu")
229
230
vllm_extension = CUDAExtension(
231
name="vllm._C",
vllm/config.py
+11
-15
@@ -112,24 +112,20 @@ class ModelConfig:
112
supported_load_format = [
113
"auto", "pt", "safetensors", "npcache", "dummy"
114
]
115
- rocm_not_supported_load_format = ["safetensors"]
115
+ rocm_not_supported_load_format = []
116
if load_format not in supported_load_format:
117
raise ValueError(
118
f"Unknown load format: {self.load_format}. Must be one of "
119
"'auto', 'pt', 'safetensors', 'npcache', or 'dummy'.")
120
- if is_hip():
121
- if load_format in ["safetensors"]:
122
- rocm_supported_load_format = [
123
- f for f in supported_load_format
124
- if (f not in rocm_not_supported_load_format)
125
- ]
126
- raise ValueError(
127
- f"load format \'{load_format}\' is not supported in ROCm. "
128
- f"Supported load format are "
129
- f"{rocm_supported_load_format}")
130
- # Force ROCm to load from pt weights if nothing specific is set
131
- if load_format == "auto":
132
- load_format = "pt"
120
+ if is_hip() and load_format in rocm_not_supported_load_format:
121
+ rocm_supported_load_format = [
122
+ f for f in supported_load_format
123
+ if (f not in rocm_not_supported_load_format)
124
+ ]
125
+ raise ValueError(
126
+ f"load format \'{load_format}\' is not supported in ROCm. "
127
+ f"Supported load format are "
128
+ f"{rocm_supported_load_format}")
129
130
# TODO: Remove this check once HF updates the pt weights of Mixtral.
131
architectures = getattr(self.hf_config, "architectures", [])
@@ -149,7 +145,7 @@ class ModelConfig:
145
146
def _verify_quantization(self) -> None:
147
supported_quantization = ["awq", "gptq", "squeezellm"]
152
- rocm_not_supported_quantization = ["awq", "gptq"]
148
+ rocm_not_supported_quantization = ["awq"]
149
if self.quantization is not None:
150
self.quantization = self.quantization.lower()
151