@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2b1c116b5a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Add minimum capability requirement for AWQ (#1064)

Woosuk Kwon committed Sep 18, 2023 at 12:02 UTC 2b1c116b5acdf3b738e310f98617875132214c37
5 files changed +47 -2
csrc/quantization/awq/dequantize.cuh
+8
@@ -11,9 +11,14 @@ Modified from NVIDIA FasterTransformer: https://github.com/NVIDIA/FasterTransfor
11
12 #pragma once
13
14 +namespace vllm {
15 +namespace awq {
16
17 __device__ uint4 dequantize_s4_to_fp16x2(uint32_t const& source)
18 {
19 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
20 + assert(false);
21 +#else
22 uint4 result;
23
24 uint32_t* h = reinterpret_cast<uint32_t*>(&result);
@@ -75,5 +80,8 @@ __device__ uint4 dequantize_s4_to_fp16x2(uint32_t const& source)
80 asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(h[3]) : "r"(h[3]), "r"(ONE_SIXTEENTH), "r"(NEG_64));
81
82 return result;
83 +#endif
84 }
85
86 +} // namespace awq
87 +} // namespace vllm
csrc/quantization/awq/gemm_kernels.cu
+16 -2
@@ -16,6 +16,9 @@ Adapted from https://github.com/mit-han-lab/llm-awq
16
17 #include <cuda_fp16.h>
18
19 +namespace vllm {
20 +namespace awq {
21 +
22 // Pack two half values.
23 static inline __device__ __host__ unsigned
24 __pack_half2(const half x, const half y) {
@@ -26,6 +29,9 @@ __pack_half2(const half x, const half y) {
29
30 __global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16n128k32(int G, int split_k_iters, half* __restrict__ A, int* __restrict__ B, half* __restrict__ scaling_factors, int* __restrict__ zeros, int M, int IC, int OC, half* __restrict__ C)
31 {
32 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
33 + assert(false);
34 +#else
35 static constexpr uint32_t ZERO = 0x0;
36 float C_warp[32];
37 __shared__ half A_shared[16 * (32 + 8)];
@@ -214,11 +220,15 @@ __global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16n128k32(int G, i
220 }
221 }
222 }
223 +#endif
224 }
225
226
227 __global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16n64k32(int G, int split_k_iters, half* __restrict__ A, int* __restrict__ B, half* __restrict__ scaling_factors, int* __restrict__ zeros, int M, int IC, int OC, half* __restrict__ C)
228 {
229 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
230 + assert(false);
231 +#else
232 static constexpr uint32_t ZERO = 0x0;
233 float C_warp[32];
234 __shared__ half A_shared[16 * (32 + 8)];
@@ -412,8 +422,12 @@ __global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16n64k32(int G, in
422 }
423 }
424 }
425 +#endif
426 }
427
428 +} // namespace awq
429 +} // namespace vllm
430 +
431 // in_feats: M, IC [float16]
432 // kernel: IC, OC // 8 [int32] -> cast to IC, OC [uint4b]
433 // scaling_factors: IC // G, OC [float16]
@@ -459,7 +473,7 @@ torch::Tensor awq_gemm(
473 // threadIdx.x: 32
474 // threadIdx.y: i_factors[2] * j_factors[2]
475 dim3 threads_per_block(32, 2);
462 - gemm_forward_4bit_cuda_m16n128k32<<<num_blocks, threads_per_block>>>(
476 + vllm::awq::gemm_forward_4bit_cuda_m16n128k32<<<num_blocks, threads_per_block>>>(
477 group_size, split_k_iters, in_feats, kernel, scaling_factors, zeros, num_in_feats, num_in_channels, num_out_channels, out_feats);
478 }
479 else if (num_out_channels % 64 == 0)
@@ -470,7 +484,7 @@ torch::Tensor awq_gemm(
484 // threadIdx.x: 32
485 // threadIdx.y: i_factors[2] * j_factors[2]
486 dim3 threads_per_block(32, 2);
473 - gemm_forward_4bit_cuda_m16n64k32<<<num_blocks, threads_per_block>>>(
487 + vllm::awq::gemm_forward_4bit_cuda_m16n64k32<<<num_blocks, threads_per_block>>>(
488 group_size, split_k_iters, in_feats, kernel, scaling_factors, zeros, num_in_feats, num_in_channels, num_out_channels, out_feats);
489 }
490 return _out_feats.sum(0);
vllm/model_executor/model_loader.py
+8
@@ -68,6 +68,14 @@ def get_model(model_config: ModelConfig) -> nn.Module:
68 quant_config = get_quant_config(model_config.quantization,
69 model_config.model,
70 model_config.download_dir)
71 + capability = torch.cuda.get_device_capability()
72 + capability = capability[0] * 10 + capability[1]
73 + if capability < quant_config.get_min_capability():
74 + raise ValueError(
75 + f"The quantization method {model_config.quantization} is not "
76 + "supported for the current GPU. "
77 + f"Minimum capability: {quant_config.get_min_capability()}. "
78 + f"Current capability: {capability}.")
79 supported_dtypes = quant_config.get_supported_act_dtypes()
80 if model_config.dtype not in supported_dtypes:
81 raise ValueError(
vllm/model_executor/quantization_utils/awq.py
+5
@@ -40,6 +40,11 @@ class AWQConfig(QuantizationConfig):
40 def get_supported_act_dtypes(cls) -> List[torch.dtype]:
41 return [torch.half]
42
43 + @classmethod
44 + def get_min_capability(cls) -> int:
45 + # The AWQ kernel only supports Ampere or newer GPUs.
46 + return 80
47 +
48 @classmethod
49 def get_config_filenames(cls) -> List[str]:
50 return [
vllm/model_executor/quantization_utils/base.py
+10
@@ -15,6 +15,16 @@ class QuantizationConfig:
15 """List of supported activation dtypes."""
16 raise NotImplementedError
17
18 + @classmethod
19 + def get_min_capability(cls) -> int:
20 + """Minimum GPU capability to support the quantization method.
21 +
22 + E.g., 70 for Volta, 75 for Turing, 80 for Ampere.
23 + This requirement is due to the custom CUDA kernels used by the
24 + quantization method.
25 + """
26 + raise NotImplementedError
27 +
28 @classmethod
29 def get_config_filenames(cls) -> List[str]:
30 """List of filenames to search for in the model directory."""