@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8c32b08a86
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[Kernel] Fix awq error when n is not divisable by 128 (#13227)

Jinzhen Lin committed Feb 14, 2025 at 12:07 UTC 8c32b08a86bd7dd1a3c59cb9257eaa80fd329276
1 file changed +1 -1
csrc/quantization/awq/gemm_kernels.cu
+1 -1
@@ -334,7 +334,7 @@ __global__ void __launch_bounds__(64)
334 }
335
336 // TODO: Shang: Hoist loop invariance.
337 - for (int ax1_0_1 = 0; ax1_0_1 < 4; ++ax1_0_1) {
337 + for (int ax1_0_1 = 0; ax1_0_1 < (N / 32); ++ax1_0_1) {
338 for (int local_id = 0; local_id < 8; ++local_id) {
339 int row_offset = (((int)blockIdx_y) / j_factors1) * 16 +
340 ((int)threadIdx.x) / 4 + (local_id % 4) / 2 * 8;