A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Fix overflow in awq kernel (#1295)
Co-authored-by: 楚天翔 <tianxiang.ctx@alibaba-inc.com>
CHU Tianxiang committed
Oct 11, 2023 at 15:19 UTC
980dd4a2c4ca965e4b10483258a006b812a1991f
1 file changed
+2
-2
csrc/quantization/awq/gemm_kernels.cu
+2
-2
@@ -90,7 +90,7 @@ __global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16n128k32(int G, i
90
+ (((int)threadIdx.x) % (128 / 8)) * 8;
91
92
half* C_ptr = C
93
- + blockIdx_z * M * OC // blockIdz.x -> split_k dim
93
+ + static_cast<long long>(blockIdx_z) * M * OC // blockIdz.x -> split_k dim
94
+ (((int)blockIdx_y) % j_factors1) * 128
95
+ ((int)threadIdx.y) * 64
96
+ (((int)threadIdx.x) % 4) * 2;
@@ -323,7 +323,7 @@ __global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16n64k32(int G, in
323
+ (((int)threadIdx.x) % (64 / 8)) * 8;
324
325
half* C_ptr = C
326
- + blockIdx_z * M * OC // blockIdz.x -> split_k dim
326
+ + static_cast<long long>(blockIdx_z) * M * OC // blockIdz.x -> split_k dim
327
+ (((int)blockIdx_y) % j_factors1) * 64
328
+ ((int)threadIdx.y) * 32
329
+ (((int)threadIdx.x) % 4) * 2;