A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Kernel] Fix awq error when n is not divisable by 128 (#13227)
Jinzhen Lin committed
Feb 14, 2025 at 12:07 UTC
8c32b08a86bd7dd1a3c59cb9257eaa80fd329276
1 file changed
+1
-1
csrc/quantization/awq/gemm_kernels.cu
+1
-1
@@ -334,7 +334,7 @@ __global__ void __launch_bounds__(64)
334
}
335
336
// TODO: Shang: Hoist loop invariance.
337
- for (int ax1_0_1 = 0; ax1_0_1 < 4; ++ax1_0_1) {
337
+ for (int ax1_0_1 = 0; ax1_0_1 < (N / 32); ++ax1_0_1) {
338
for (int local_id = 0; local_id < 8; ++local_id) {
339
int row_offset = (((int)blockIdx_y) / j_factors1) * 16 +
340
((int)threadIdx.x) / 4 + (local_id % 4) / 2 * 8;