@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / beb89f68b4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

AWQ: Up to 2.66x higher throughput (#2566)

Casper committed Jan 27, 2024 at 08:53 UTC beb89f68b448a43ac112b48e3834f80a2df626cb
4 files changed +127 -1
csrc/ops.h
+8
@@ -70,6 +70,14 @@ torch::Tensor awq_gemm(
70 torch::Tensor _scaling_factors,
71 torch::Tensor _zeros,
72 int split_k_iters);
73 +
74 +torch::Tensor awq_dequantize(
75 + torch::Tensor _kernel,
76 + torch::Tensor _scaling_factors,
77 + torch::Tensor _zeros,
78 + int split_k_iters,
79 + int thx,
80 + int thy);
81 #endif
82
83 void squeezellm_gemm(
csrc/pybind.cpp
+1
@@ -51,6 +51,7 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
51 #ifndef USE_ROCM
52 // Quantization ops
53 ops.def("awq_gemm", &awq_gemm, "Quantized GEMM for AWQ");
54 + ops.def("awq_dequantize", &awq_dequantize, "Dequantization for AWQ");
55 #endif
56 ops.def("gptq_gemm", &gptq_gemm, "Quantized GEMM for GPTQ");
57 ops.def("gptq_shuffle", &gptq_shuffle, "Post processing for GPTQ");
csrc/quantization/awq/gemm_kernels.cu
+108
@@ -493,9 +493,117 @@ __global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16n64k32(int G, in
493 #endif
494 }
495
496 +__global__ void __launch_bounds__(64) dequantize_weights(
497 + int* __restrict__ B,
498 + half* __restrict__ scaling_factors,
499 + int* __restrict__ zeros,
500 + half* __restrict__ C,
501 + int G
502 +)
503 +{
504 + int j_factors1 = 4;
505 + int row_stride2 = 4;
506 + int split_k_iters = 1;
507 + static constexpr uint32_t ZERO = 0x0;
508 + half B_shared[32 * (128 + 8)];
509 +
510 + half* B_shared_ptr2 = B_shared;
511 +
512 + half B_shared_warp[32];
513 + int OC = 512;
514 +
515 + int N = blockDim.x * gridDim.x; // 2
516 + int col = (blockIdx.x * blockDim.x + threadIdx.x);
517 + int row = blockIdx.y * blockDim.y + threadIdx.y;
518 + int index1 = 8 * col + 8 * row * N;
519 + half* C_ptr2 = C + index1;
520 +
521 + int index2 = col + row * N;
522 + int* B_ptr2 = B + index2;
523 +
524 + int index3 = col + (int)(row / G) * N;
525 + int* zeros_ptr2 = zeros + index3;
526 + int index4 = 8 * col + (int)(row / G) * N * 8;
527 + half* scaling_factors_ptr2 = scaling_factors + index4;
528 +
529 +
530 + uint32_t zeros_loaded = *(uint32_t*)(zeros_ptr2);
531 + uint4 B_loaded_zero = dequantize_s4_to_fp16x2(zeros_loaded);
532 + uint4 B_loaded_scale = *(uint4*)(scaling_factors_ptr2);
533 +int j=0;
534 +
535 + uint32_t B_loaded = *(uint32_t*)(B_ptr2 + j);
536 + uint4 B_loaded_fp16 = dequantize_s4_to_fp16x2(B_loaded);
537 + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.x) : "r"(B_loaded_fp16.x), "r"(B_loaded_zero.x));
538 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.x) : "r"(B_loaded_fp16.x), "r"(B_loaded_scale.x), "r"(ZERO));
539 + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.y) : "r"(B_loaded_fp16.y), "r"(B_loaded_zero.y));
540 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.y) : "r"(B_loaded_fp16.y), "r"(B_loaded_scale.y), "r"(ZERO));
541 + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.z) : "r"(B_loaded_fp16.z), "r"(B_loaded_zero.z));
542 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.z) : "r"(B_loaded_fp16.z), "r"(B_loaded_scale.z), "r"(ZERO));
543 + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.w) : "r"(B_loaded_fp16.w), "r"(B_loaded_zero.w));
544 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.w) : "r"(B_loaded_fp16.w), "r"(B_loaded_scale.w), "r"(ZERO));
545 +
546 + *(uint4*)(B_shared_ptr2 + j) = B_loaded_fp16;
547 +
548 + for (int i=0; i<8; ++i) {
549 + *(C_ptr2 + i) = B_shared[i];
550 + }
551 +}
552 +
553 } // namespace awq
554 } // namespace vllm
555
556 +torch::Tensor awq_dequantize(
557 + torch::Tensor _kernel,
558 + torch::Tensor _scaling_factors,
559 + torch::Tensor _zeros,
560 + int split_k_iters,
561 + int thx,
562 + int thy)
563 +{
564 + int in_c = _kernel.size(0);
565 + int qout_c = _kernel.size(1);
566 + int out_c = qout_c * 8;
567 + int G = in_c / _scaling_factors.size(0);
568 +
569 + int x_thread = thx;
570 + int y_thread = thy;
571 +
572 + int x_blocks = 1;
573 + int y_blocks = 1;
574 + if (thx==0) {
575 + x_thread = qout_c;
576 + }
577 + if (thy==0) {
578 + y_thread = in_c;
579 + }
580 + if (thx==0 && thy==0) {
581 + x_thread = 8;
582 + y_thread = 8;
583 + x_blocks = (int)(qout_c / 8);
584 + y_blocks = (int)(in_c / 8);
585 + }
586 +
587 + const at::cuda::OptionalCUDAGuard device_guard(device_of(_scaling_factors));
588 +
589 + auto options = torch::TensorOptions().dtype(_scaling_factors.dtype()).device(_scaling_factors.device());
590 + at::Tensor _de_kernel = torch::empty({in_c, out_c}, options);
591 +
592 + auto kernel = reinterpret_cast<int*>(_kernel.data_ptr<int>());
593 + auto de_kernel = reinterpret_cast<half*>(_de_kernel.data_ptr<at::Half>());
594 + auto scaling_factors = reinterpret_cast<half*>(_scaling_factors.data_ptr<at::Half>());
595 + auto zeros = reinterpret_cast<int*>(_zeros.data_ptr<int>());
596 +
597 + dim3 num_blocks(x_blocks, y_blocks);
598 + dim3 threads_per_block(x_thread, y_thread);
599 +
600 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
601 + vllm::awq::dequantize_weights<<<num_blocks, threads_per_block, 0, stream>>>(
602 + kernel, scaling_factors, zeros, de_kernel, G);
603 +
604 + return _de_kernel;
605 +}
606 +
607 // in_feats: M, IC [float16]
608 // kernel: IC, OC // 8 [int32] -> cast to IC, OC [uint4b]
609 // scaling_factors: IC // G, OC [float16]
vllm/model_executor/layers/quantization/awq.py
+10 -1
@@ -153,7 +153,16 @@ class AWQLinearMethod(LinearMethodBase):
153 pack_factor = self.quant_config.pack_factor
154 out_shape = (x.shape[:-1] + (qweight.shape[-1] * pack_factor, ))
155 reshaped_x = x.reshape(-1, x.shape[-1])
156 - out = ops.awq_gemm(reshaped_x, qweight, scales, qzeros, pack_factor)
156 +
157 + # num_tokens >= threshold
158 + FP16_MATMUL_HEURISTIC_CONDITION = x.shape[:-1].numel() >= 256
159 +
160 + if FP16_MATMUL_HEURISTIC_CONDITION:
161 + out = ops.awq_dequantize(qweight, scales, qzeros, 0, 0, 0)
162 + out = torch.matmul(reshaped_x, out)
163 + else:
164 + out = ops.awq_gemm(reshaped_x, qweight, scales, qzeros,
165 + pack_factor)
166 if bias is not None:
167 out = out + bias
168 return out.reshape(out_shape)