@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9db52eab3d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Kernel] [Triton] Memory optimization for awq_gemm and awq_dequantize, 2x throughput (#8248)

rasmith committed Sep 6, 2024 at 17:26 UTC 9db52eab3dc0b7b2cf30fa4399d569131e90c2d4
1 file changed +23 -11
vllm/model_executor/layers/quantization/awq_triton.py
+23 -11
@@ -22,7 +22,7 @@ def awq_dequantize_kernel(
22
23 # Compute offsets and masks for qweight_ptr.
24 offsets_y = pid_y * BLOCK_SIZE_Y + tl.arange(0, BLOCK_SIZE_Y)
25 - offsets_x = pid_x * BLOCK_SIZE_X + tl.arange(0, BLOCK_SIZE_X * 8) // 8
25 + offsets_x = pid_x * BLOCK_SIZE_X + tl.arange(0, BLOCK_SIZE_X)
26 offsets = num_cols * offsets_y[:, None] + offsets_x[None, :]
27
28 masks_y = offsets_y < num_rows
@@ -43,6 +43,9 @@ def awq_dequantize_kernel(
43
44 # Load the weights.
45 iweights = tl.load(qweight_ptr + offsets, masks)
46 + iweights = tl.interleave(iweights, iweights)
47 + iweights = tl.interleave(iweights, iweights)
48 + iweights = tl.interleave(iweights, iweights)
49
50 # Create reverse AWQ order as tensor: [0, 4, 1, 5, 2, 6, 3, 7]
51 # that will map given indices to the correct order.
@@ -59,9 +62,8 @@ def awq_dequantize_kernel(
62 iweights = (iweights >> shifts) & 0xF
63
64 # Compute zero offsets and masks.
62 - zero_offsets_y = (pid_y * BLOCK_SIZE_Y // group_size +
63 - tl.arange(0, BLOCK_SIZE_Y) // group_size)
64 - zero_offsets_x = pid_x * BLOCK_SIZE_X + tl.arange(0, BLOCK_SIZE_X * 8) // 8
65 + zero_offsets_y = pid_y * BLOCK_SIZE_Y // group_size + tl.arange(0, 1)
66 + zero_offsets_x = pid_x * BLOCK_SIZE_X + tl.arange(0, BLOCK_SIZE_X)
67 zero_offsets = num_cols * zero_offsets_y[:, None] + zero_offsets_x[None, :]
68
69 zero_masks_y = zero_offsets_y < num_rows // group_size
@@ -70,13 +72,16 @@ def awq_dequantize_kernel(
72
73 # Load the zeros.
74 zeros = tl.load(zeros_ptr + zero_offsets, zero_masks)
75 + zeros = tl.interleave(zeros, zeros)
76 + zeros = tl.interleave(zeros, zeros)
77 + zeros = tl.interleave(zeros, zeros)
78 + zeros = tl.broadcast_to(zeros, (BLOCK_SIZE_Y, BLOCK_SIZE_X * 8))
79
80 # Unpack and reorder: shift out the correct 4-bit value and mask.
81 zeros = (zeros >> shifts) & 0xF
82
83 # Compute scale offsets and masks.
78 - scale_offsets_y = (pid_y * BLOCK_SIZE_Y // group_size +
79 - tl.arange(0, BLOCK_SIZE_Y) // group_size)
84 + scale_offsets_y = pid_y * BLOCK_SIZE_Y // group_size + tl.arange(0, 1)
85 scale_offsets_x = (pid_x * BLOCK_SIZE_X * 8 +
86 tl.arange(0, BLOCK_SIZE_X * 8))
87 scale_offsets = (num_cols * 8 * scale_offsets_y[:, None] +
@@ -87,6 +92,7 @@ def awq_dequantize_kernel(
92
93 # Load the scales.
94 scales = tl.load(scales_ptr + scale_offsets, scale_masks)
95 + scales = tl.broadcast_to(scales, (BLOCK_SIZE_Y, BLOCK_SIZE_X * 8))
96
97 # Dequantize.
98 iweights = (iweights - zeros) * scales
@@ -137,12 +143,10 @@ def awq_gemm_kernel(a_ptr, b_ptr, c_ptr, zeros_ptr, scales_ptr, M, N, K,
143 offsets_am = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
144 masks_am = offsets_am < M
145
140 - offsets_bn = (pid_n * (BLOCK_SIZE_N // 8) +
141 - tl.arange(0, BLOCK_SIZE_N) // 8)
146 + offsets_bn = pid_n * (BLOCK_SIZE_N // 8) + tl.arange(0, BLOCK_SIZE_N // 8)
147 masks_bn = offsets_bn < N // 8
148
144 - offsets_zn = (pid_n * (BLOCK_SIZE_N // 8) +
145 - tl.arange(0, BLOCK_SIZE_N) // 8)
149 + offsets_zn = pid_n * (BLOCK_SIZE_N // 8) + tl.arange(0, BLOCK_SIZE_N // 8)
150 masks_zn = offsets_zn < N // 8
151
152 offsets_sn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
@@ -165,22 +169,30 @@ def awq_gemm_kernel(a_ptr, b_ptr, c_ptr, zeros_ptr, scales_ptr, M, N, K,
169
170 masks_b = masks_k[:, None] & masks_bn[None, :]
171 b = tl.load(b_ptrs, mask=masks_b)
172 + b = tl.interleave(b, b)
173 + b = tl.interleave(b, b)
174 + b = tl.interleave(b, b)
175
176 # Dequantize b.
177 offsets_szk = (
178 (BLOCK_SIZE_K * SPLIT_K * k + pid_z * BLOCK_SIZE_K) // group_size +
172 - tl.arange(0, BLOCK_SIZE_K) // group_size)
179 + tl.arange(0, 1))
180 offsets_z = (N // 8) * offsets_szk[:, None] + offsets_zn[None, :]
181 masks_zk = offsets_szk < K // group_size
182 masks_z = masks_zk[:, None] & masks_zn[None, :]
183 zeros_ptrs = zeros_ptr + offsets_z
184 zeros = tl.load(zeros_ptrs, mask=masks_z)
185 + zeros = tl.interleave(zeros, zeros)
186 + zeros = tl.interleave(zeros, zeros)
187 + zeros = tl.interleave(zeros, zeros)
188 + zeros = tl.broadcast_to(zeros, (BLOCK_SIZE_K, BLOCK_SIZE_N))
189
190 offsets_s = N * offsets_szk[:, None] + offsets_sn[None, :]
191 masks_sk = offsets_szk < K // group_size
192 masks_s = masks_sk[:, None] & masks_sn[None, :]
193 scales_ptrs = scales_ptr + offsets_s
194 scales = tl.load(scales_ptrs, mask=masks_s)
195 + scales = tl.broadcast_to(scales, (BLOCK_SIZE_K, BLOCK_SIZE_N))
196
197 b = (b >> shifts) & 0xF
198 zeros = (zeros >> shifts) & 0xF