A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Add minimum capability requirement for AWQ (#1064)
Woosuk Kwon committed
Sep 18, 2023 at 12:02 UTC
2b1c116b5acdf3b738e310f98617875132214c37
5 files changed
+47
-2
csrc/quantization/awq/dequantize.cuh
+8
@@ -11,9 +11,14 @@ Modified from NVIDIA FasterTransformer: https://github.com/NVIDIA/FasterTransfor
11
12
#pragma once
13
14
+namespace vllm {
15
+namespace awq {
16
17
__device__ uint4 dequantize_s4_to_fp16x2(uint32_t const& source)
18
{
19
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
20
+ assert(false);
21
+#else
22
uint4 result;
23
24
uint32_t* h = reinterpret_cast<uint32_t*>(&result);
@@ -75,5 +80,8 @@ __device__ uint4 dequantize_s4_to_fp16x2(uint32_t const& source)
80
asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(h[3]) : "r"(h[3]), "r"(ONE_SIXTEENTH), "r"(NEG_64));
81
82
return result;
83
+#endif
84
}
85
86
+} // namespace awq
87
+} // namespace vllm
csrc/quantization/awq/gemm_kernels.cu
+16
-2
@@ -16,6 +16,9 @@ Adapted from https://github.com/mit-han-lab/llm-awq
16
17
#include <cuda_fp16.h>
18
19
+namespace vllm {
20
+namespace awq {
21
+
22
// Pack two half values.
23
static inline __device__ __host__ unsigned
24
__pack_half2(const half x, const half y) {
@@ -26,6 +29,9 @@ __pack_half2(const half x, const half y) {
29
30
__global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16n128k32(int G, int split_k_iters, half* __restrict__ A, int* __restrict__ B, half* __restrict__ scaling_factors, int* __restrict__ zeros, int M, int IC, int OC, half* __restrict__ C)
31
{
32
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
33
+ assert(false);
34
+#else
35
static constexpr uint32_t ZERO = 0x0;
36
float C_warp[32];
37
__shared__ half A_shared[16 * (32 + 8)];
@@ -214,11 +220,15 @@ __global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16n128k32(int G, i
220
}
221
}
222
}
223
+#endif
224
}
225
226
227
__global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16n64k32(int G, int split_k_iters, half* __restrict__ A, int* __restrict__ B, half* __restrict__ scaling_factors, int* __restrict__ zeros, int M, int IC, int OC, half* __restrict__ C)
228
{
229
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
230
+ assert(false);
231
+#else
232
static constexpr uint32_t ZERO = 0x0;
233
float C_warp[32];
234
__shared__ half A_shared[16 * (32 + 8)];
@@ -412,8 +422,12 @@ __global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16n64k32(int G, in
422
}
423
}
424
}
425
+#endif
426
}
427
428
+} // namespace awq
429
+} // namespace vllm
430
+
431
// in_feats: M, IC [float16]
432
// kernel: IC, OC // 8 [int32] -> cast to IC, OC [uint4b]
433
// scaling_factors: IC // G, OC [float16]
@@ -459,7 +473,7 @@ torch::Tensor awq_gemm(
473
// threadIdx.x: 32
474
// threadIdx.y: i_factors[2] * j_factors[2]
475
dim3 threads_per_block(32, 2);
462
- gemm_forward_4bit_cuda_m16n128k32<<<num_blocks, threads_per_block>>>(
476
+ vllm::awq::gemm_forward_4bit_cuda_m16n128k32<<<num_blocks, threads_per_block>>>(
477
group_size, split_k_iters, in_feats, kernel, scaling_factors, zeros, num_in_feats, num_in_channels, num_out_channels, out_feats);
478
}
479
else if (num_out_channels % 64 == 0)
@@ -470,7 +484,7 @@ torch::Tensor awq_gemm(
484
// threadIdx.x: 32
485
// threadIdx.y: i_factors[2] * j_factors[2]
486
dim3 threads_per_block(32, 2);
473
- gemm_forward_4bit_cuda_m16n64k32<<<num_blocks, threads_per_block>>>(
487
+ vllm::awq::gemm_forward_4bit_cuda_m16n64k32<<<num_blocks, threads_per_block>>>(
488
group_size, split_k_iters, in_feats, kernel, scaling_factors, zeros, num_in_feats, num_in_channels, num_out_channels, out_feats);
489
}
490
return _out_feats.sum(0);
vllm/model_executor/model_loader.py
+8
@@ -68,6 +68,14 @@ def get_model(model_config: ModelConfig) -> nn.Module:
68
quant_config = get_quant_config(model_config.quantization,
69
model_config.model,
70
model_config.download_dir)
71
+ capability = torch.cuda.get_device_capability()
72
+ capability = capability[0] * 10 + capability[1]
73
+ if capability < quant_config.get_min_capability():
74
+ raise ValueError(
75
+ f"The quantization method {model_config.quantization} is not "
76
+ "supported for the current GPU. "
77
+ f"Minimum capability: {quant_config.get_min_capability()}. "
78
+ f"Current capability: {capability}.")
79
supported_dtypes = quant_config.get_supported_act_dtypes()
80
if model_config.dtype not in supported_dtypes:
81
raise ValueError(
vllm/model_executor/quantization_utils/awq.py
+5
@@ -40,6 +40,11 @@ class AWQConfig(QuantizationConfig):
40
def get_supported_act_dtypes(cls) -> List[torch.dtype]:
41
return [torch.half]
42
43
+ @classmethod
44
+ def get_min_capability(cls) -> int:
45
+ # The AWQ kernel only supports Ampere or newer GPUs.
46
+ return 80
47
+
48
@classmethod
49
def get_config_filenames(cls) -> List[str]:
50
return [
vllm/model_executor/quantization_utils/base.py
+10
@@ -15,6 +15,16 @@ class QuantizationConfig:
15
"""List of supported activation dtypes."""
16
raise NotImplementedError
17
18
+ @classmethod
19
+ def get_min_capability(cls) -> int:
20
+ """Minimum GPU capability to support the quantization method.
21
+
22
+ E.g., 70 for Volta, 75 for Turing, 80 for Ampere.
23
+ This requirement is due to the custom CUDA kernels used by the
24
+ quantization method.
25
+ """
26
+ raise NotImplementedError
27
+
28
@classmethod
29
def get_config_filenames(cls) -> List[str]:
30
"""List of filenames to search for in the model directory."""