@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 29678cd213
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Minor fix on AWQ kernel launch (#1356)

Woosuk Kwon committed Oct 15, 2023 at 21:53 UTC 29678cd21322f041518541e62992557d293c8597
1 file changed +4 -3
csrc/quantization/awq/gemm_kernels.cu
+4 -3
@@ -534,6 +534,7 @@ torch::Tensor awq_gemm(
534 if (num_out_channels % group_size != 0)
535 throw std::invalid_argument("OC is not multiple of Group size");
536
537 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
538 if (num_out_channels % 128 == 0)
539 {
540 int j_factors1 = num_out_channels / 128 / 1;
@@ -541,18 +542,18 @@ torch::Tensor awq_gemm(
542 // threadIdx.x: 32
543 // threadIdx.y: i_factors[2] * j_factors[2]
544 dim3 threads_per_block(32, 2);
544 - vllm::awq::gemm_forward_4bit_cuda_m16n128k32<<<num_blocks, threads_per_block>>>(
545 + vllm::awq::gemm_forward_4bit_cuda_m16n128k32<<<num_blocks, threads_per_block, 0, stream>>>(
546 group_size, split_k_iters, in_feats, kernel, scaling_factors, zeros, num_in_feats, num_in_channels, num_out_channels, out_feats);
547 }
548 else if (num_out_channels % 64 == 0)
549 {
549 - int j_factors1 = num_out_channels / 64 / 1;
550 + int j_factors1 = num_out_channels / 64 / 1;
551 dim3 num_blocks(1 * (num_out_feats + 16 - 1) / 16 * j_factors1 * split_k_iters);
552
553 // threadIdx.x: 32
554 // threadIdx.y: i_factors[2] * j_factors[2]
555 dim3 threads_per_block(32, 2);
555 - vllm::awq::gemm_forward_4bit_cuda_m16n64k32<<<num_blocks, threads_per_block>>>(
556 + vllm::awq::gemm_forward_4bit_cuda_m16n64k32<<<num_blocks, threads_per_block, 0, stream>>>(
557 group_size, split_k_iters, in_feats, kernel, scaling_factors, zeros, num_in_feats, num_in_channels, num_out_channels, out_feats);
558 }
559 return _out_feats.sum(0);