@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ec4aaad812
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Kernel][Triton][AMD] Remove tl.atomic_add from awq_gemm_kernel, 2-5x speedup MI300, minor improvement for MI250 (#8646)

rasmith committed Sep 21, 2024 at 04:20 UTC ec4aaad8124baadc7954e30c612ca9444b22d7e7
1 file changed +7 -6
vllm/model_executor/layers/quantization/awq_triton.py
+7 -6
@@ -209,12 +209,9 @@ def awq_gemm_kernel(a_ptr, b_ptr, c_ptr, zeros_ptr, scales_ptr, M, N, K,
209 c = accumulator.to(c_ptr.type.element_ty)
210 offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
211 offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
212 - c_ptrs = c_ptr + N * offs_cm[:, None] + offs_cn[None, :]
212 + c_ptrs = c_ptr + pid_z * N * M + N * offs_cm[:, None] + offs_cn[None, :]
213 c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N)
214 - if SPLIT_K == 1:
215 - tl.store(c_ptrs, c, mask=c_mask)
216 - else:
217 - tl.atomic_add(c_ptrs, c, mask=c_mask)
214 + tl.store(c_ptrs, c, mask=c_mask)
215
216
217 # qweights - [K , M // 8], int32
@@ -295,7 +292,9 @@ def awq_gemm_triton(input: torch.Tensor,
292 split_k_iters,
293 )
294
298 - result = torch.zeros((M, N), dtype=scales.dtype, device=input.device)
295 + result = torch.zeros((split_k_iters, M, N),
296 + dtype=scales.dtype,
297 + device=input.device)
298
299 # A = input, B = qweight, C = result
300 # A = M x K, B = K x N, C = M x N
@@ -313,4 +312,6 @@ def awq_gemm_triton(input: torch.Tensor,
312 BLOCK_SIZE_K=block_size_k,
313 SPLIT_K=split_k_iters)
314
315 + result = result.sum(0)
316 +
317 return result