A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Minor fix on AWQ kernel launch (#1356)
Woosuk Kwon committed
Oct 15, 2023 at 21:53 UTC
29678cd21322f041518541e62992557d293c8597
1 file changed
+4
-3
csrc/quantization/awq/gemm_kernels.cu
+4
-3
@@ -534,6 +534,7 @@ torch::Tensor awq_gemm(
534
if (num_out_channels % group_size != 0)
535
throw std::invalid_argument("OC is not multiple of Group size");
536
537
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
538
if (num_out_channels % 128 == 0)
539
{
540
int j_factors1 = num_out_channels / 128 / 1;
@@ -541,18 +542,18 @@ torch::Tensor awq_gemm(
542
// threadIdx.x: 32
543
// threadIdx.y: i_factors[2] * j_factors[2]
544
dim3 threads_per_block(32, 2);
544
- vllm::awq::gemm_forward_4bit_cuda_m16n128k32<<<num_blocks, threads_per_block>>>(
545
+ vllm::awq::gemm_forward_4bit_cuda_m16n128k32<<<num_blocks, threads_per_block, 0, stream>>>(
546
group_size, split_k_iters, in_feats, kernel, scaling_factors, zeros, num_in_feats, num_in_channels, num_out_channels, out_feats);
547
}
548
else if (num_out_channels % 64 == 0)
549
{
549
- int j_factors1 = num_out_channels / 64 / 1;
550
+ int j_factors1 = num_out_channels / 64 / 1;
551
dim3 num_blocks(1 * (num_out_feats + 16 - 1) / 16 * j_factors1 * split_k_iters);
552
553
// threadIdx.x: 32
554
// threadIdx.y: i_factors[2] * j_factors[2]
555
dim3 threads_per_block(32, 2);
555
- vllm::awq::gemm_forward_4bit_cuda_m16n64k32<<<num_blocks, threads_per_block>>>(
556
+ vllm::awq::gemm_forward_4bit_cuda_m16n64k32<<<num_blocks, threads_per_block, 0, stream>>>(
557
group_size, split_k_iters, in_feats, kernel, scaling_factors, zeros, num_in_feats, num_in_channels, num_out_channels, out_feats);
558
}
559
return _out_feats.sum(0);