@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 563836496a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Refactor 2 awq gemm kernels into m16nXk32 (#2723)

Co-authored-by: Chunan Zeng <chunanzeng@Chunans-Air.attlocal.net>

Rex committed Feb 12, 2024 at 11:02 UTC 563836496abc0914c212b693130f80be25926564
2 files changed +73 -295
csrc/quantization/awq/gemm_kernels.cu
+72 -294
@@ -27,72 +27,85 @@ __pack_half2(const half x, const half y) {
27 return (v1 << 16) | v0;
28 }
29
30 -__global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16n128k32(int G, int split_k_iters, half* __restrict__ A, int* __restrict__ B, half* __restrict__ scaling_factors, int* __restrict__ zeros, int M, int IC, int OC, half* __restrict__ C)
30 +template<int N>
31 +__global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16nXk32(
32 + int G,
33 + int split_k_iters,
34 + half* __restrict__ A,
35 + int* __restrict__ B,
36 + half* __restrict__ scaling_factors,
37 + int* __restrict__ zeros,
38 + int M,
39 + int IC,
40 + int OC,
41 + half* __restrict__ C)
42 {
43 + // Only support matrix n = 64 or 128
44 + assert(N == 64 || N == 128);
45 #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 750
46 assert(false);
47 #else
48 static constexpr uint32_t ZERO = 0x0;
49 float C_warp[32];
50 __shared__ half A_shared[16 * (32 + 8)];
38 - __shared__ half B_shared[32 * (128 + 8)];
39 -
40 - __shared__ half scaling_factors_shared[128];
41 - __shared__ half zeros_shared[128];
51 + __shared__ half B_shared[32 * (N + 8)];
52
43 - int j_factors1 = ((OC + 128 - 1) / 128);
53 + __shared__ half scaling_factors_shared[N];
54 + __shared__ half zeros_shared[N];
55 +
56 + int j_factors1 = ((OC + N - 1) / N);
57 int blockIdx_x = 0;
58 int blockIdx_y = blockIdx.x % ((M + 16 - 1) / 16 * j_factors1);
59 int blockIdx_z = blockIdx.x / ((M + 16 - 1) / 16 * j_factors1);
60
61 half A_shared_warp[8];
49 - half B_shared_warp[32];
50 - for (int j_0_4_init = 0; j_0_4_init < 4; ++j_0_4_init) {
62 + half B_shared_warp[N / 4];
63 + for (int j_0_4_init = 0; j_0_4_init < N / 32; ++j_0_4_init) {
64 for (int i = 0; i < 8; ++i) {
65 C_warp[(j_0_4_init * 8) + i] = 0.0;
66 }
67 }
68
69 static constexpr int row_stride_warp = 32 * 8 / 32;
57 - static constexpr int row_stride = 2 * 32 * 8 / 128;
58 - bool ld_zero_flag = (threadIdx.y * 32 + threadIdx.x) * 8 < 128;
70 + static constexpr int row_stride = 2 * 32 * 8 / N;
71 + bool ld_zero_flag = (threadIdx.y * 32 + threadIdx.x) * 8 < N;
72 // TODO: Haotian: blockIdx_y / j_factors1 in A loading to support bsz > 16
73 bool ld_A_flag = (blockIdx_y / j_factors1 * 16 + threadIdx.y * row_stride_warp + threadIdx.x * 8 / 32) < M; // threadIdx.y is warp_id
74 // bool wb_C_flag = (threadIdx.x / 4) < M;
75
63 - half* A_ptr = A
76 + half* A_ptr = A
77 + (((int)blockIdx_y) / j_factors1 * 16 + (((int)threadIdx.y) * row_stride_warp) + ((int)threadIdx.x) / (32 / 8)) * IC
78 + (((int)threadIdx.x) % (32 / 8)) * 8;
66 -
79 +
80 int* B_ptr = B
68 - + ((int)threadIdx.y) * (OC / 8) * 2
69 - + (((int)threadIdx.x) / (128 / 8)) * (OC / 8)
70 - + (((int)blockIdx_y) % j_factors1) * (128 / 8)
71 - + (((int)threadIdx.x) % (128 / 8)) * 1;
81 + + ((int)threadIdx.y) * (OC / 8) * (256 / N)
82 + + (((int)threadIdx.x) / (N / 8)) * (OC / 8)
83 + + (((int)blockIdx_y) % j_factors1) * (N / 8)
84 + + (((int)threadIdx.x) % (N / 8)) * 1;
85 // Why * 1 in the above line?
73 -
74 - half* A_shared_ptr = A_shared
75 - + ((int)threadIdx.y) * row_stride_warp * (32 + 8)
86 +
87 + half* A_shared_ptr = A_shared
88 + + ((int)threadIdx.y) * row_stride_warp * (32 + 8)
89 + (((int)threadIdx.x) / (32 / 8)) * (32 + 8)
90 + (((int)threadIdx.x) % (32 / 8) ) * 8;
91
92 half* B_shared_ptr = B_shared
80 - + ((int)threadIdx.y) * (row_stride / 2) * (128 + 8)
81 - + (((int)threadIdx.x) / (128 / 8)) * (128 + 8)
82 - + (((int)threadIdx.x) % (128 / 8)) * 8;
83 -
93 + + ((int)threadIdx.y) * (row_stride / 2) * (N + 8)
94 + + (((int)threadIdx.x) / (N / 8)) * (N + 8)
95 + + (((int)threadIdx.x) % (N / 8)) * 8;
96 +
97 int* zeros_ptr = zeros
85 - + (((int)blockIdx_y) % j_factors1) * (128 / 8)
86 - + ((int)threadIdx.x) % (128 / 8);
87 -
98 + + (((int)blockIdx_y) % j_factors1) * (N / 8)
99 + + ((int)threadIdx.x) % (N / 8);
100 +
101 half* scaling_factors_ptr = scaling_factors
89 - + (((int)blockIdx_y) % j_factors1) * (128)
90 - + (((int)threadIdx.x) % (128 / 8)) * 8;
102 + + (((int)blockIdx_y) % j_factors1) * N
103 + + (((int)threadIdx.x) % (N / 8)) * 8;
104
92 - half* C_ptr = C
105 + half* C_ptr = C
106 + static_cast<long long>(blockIdx_z) * M * OC // blockIdz.x -> split_k dim
94 - + (((int)blockIdx_y) % j_factors1) * 128
95 - + ((int)threadIdx.y) * 64
107 + + (((int)blockIdx_y) % j_factors1) * N
108 + + ((int)threadIdx.y) * (N / 2)
109 + (((int)threadIdx.x) % 4) * 2;
110
111 // preload s.f. and zeros
@@ -123,13 +136,13 @@ __global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16n128k32(int G, i
136 // uint4 B_loaded_scale = make_uint4(0, 0, 0, 0);
137 int* B_ptr_local = B_ptr + k_0_0 * 32 * (OC / 8);
138
126 - for (int ax0_ax1_fused_0 = 0; ax0_ax1_fused_0 < 8; ++ax0_ax1_fused_0) {
139 + for (int ax0_ax1_fused_0 = 0; ax0_ax1_fused_0 < N / 16; ++ax0_ax1_fused_0) {
140
141 // B: 32 x 136 (128+8) float16
142 // each warp: 32 x 4
143 // each thr: read 32 bit -> convert to 8xFP16 (a UINT4) -> scale and minus zero -> WB UINT4
144 // *(uint4*)(B_shared + ((((ax0_ax1_fused_0 * 544) + (((int)threadIdx.y) * 272)) + ((((int)threadIdx.x) >> 4) * 136)) + ((((int)threadIdx.x) & 15) * 8))) = *(uint4*)(B + ((((((k_0_0 * 163840) + (ax0_ax1_fused_0 * 20480)) + (((int)threadIdx.y) * 10240)) + ((((int)threadIdx.x) >> 4) * 5120)) + (((int)blockIdx_y) * 128)) + ((((int)threadIdx.x) & 15) * 8)));
132 - // row stride in shared memory: (NWARPS * 32 * 8 / cta_N)
145 + // row stride in shared memory: (NWARPS * 32 * 8 / cta_N)
146 uint32_t B_loaded = *(uint32_t*)(B_ptr_local + ax0_ax1_fused_0 * row_stride * (OC / 8));
147 uint4 B_loaded_fp16 = dequantize_s4_to_fp16x2(B_loaded);
148 //uint4 B_loaded_zero = *(uint4*)(zeros_shared + (threadIdx.x % (cta_N / 8)) * 8);
@@ -152,7 +165,7 @@ __global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16n128k32(int G, i
165 */
166
167 // write back
155 - *(uint4*)(B_shared_ptr + ax0_ax1_fused_0 * row_stride * (128 + 8)) = B_loaded_fp16;
168 + *(uint4*)(B_shared_ptr + ax0_ax1_fused_0 * row_stride * (N + 8)) = B_loaded_fp16;
169 }
170 __syncthreads();
171
@@ -174,13 +187,13 @@ __global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16n128k32(int G, i
187 );
188 }
189
177 - for (int ax1_0 = 0; ax1_0 < 4; ++ax1_0) {
190 + for (int ax1_0 = 0; ax1_0 < N / 32; ++ax1_0) {
191 {
192 unsigned int addr;
193 __asm__ __volatile__(
194 "{ .reg .u64 addr; cvta.to.shared.u64 addr, %1; cvt.u32.u64 %0, addr; }\n"
195 : "=r"(addr)
183 - : "l"((void *)((&(B_shared[(((k_0_1 * 2176) + (((int)threadIdx.y) * 64)) + (ax1_0 * 16))])) + (((((int)threadIdx.x) & 15) * 136) + ((((int)threadIdx.x) >> 4) * 8))))
196 + : "l"((void *)((&(B_shared[(((k_0_1 * (N * 16 + 128)) + (((int)threadIdx.y) * (N / 2))) + (ax1_0 * 16))])) + (((((int)threadIdx.x) & 15) * (N + 8)) + ((((int)threadIdx.x) >> 4) * 8))))
197 );
198 __asm__ __volatile__(
199 "ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16"
@@ -190,7 +203,7 @@ __global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16n128k32(int G, i
203 );
204 }
205 }
193 - for (int j_0_4 = 0; j_0_4 < 4; ++j_0_4) {
206 + for (int j_0_4 = 0; j_0_4 < N / 32; ++j_0_4) {
207 #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ == 750
208 {
209 __asm__ __volatile__(
@@ -258,241 +271,6 @@ __global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16n128k32(int G, i
271 #endif
272 }
273
261 -
262 -__global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16n64k32(int G, int split_k_iters, half* __restrict__ A, int* __restrict__ B, half* __restrict__ scaling_factors, int* __restrict__ zeros, int M, int IC, int OC, half* __restrict__ C)
263 -{
264 -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 750
265 - assert(false);
266 -#else
267 - static constexpr uint32_t ZERO = 0x0;
268 - float C_warp[32];
269 - __shared__ half A_shared[16 * (32 + 8)];
270 - __shared__ half B_shared[32 * (64 + 8)];
271 -
272 - __shared__ half scaling_factors_shared[64];
273 - __shared__ half zeros_shared[64];
274 -
275 - int j_factors1 = ((OC + 64 - 1) / 64);
276 -
277 - int blockIdx_x = 0;
278 - int blockIdx_y = blockIdx.x % ((M + 16 - 1) / 16 * j_factors1);
279 - int blockIdx_z = blockIdx.x / ((M + 16 - 1) / 16 * j_factors1);
280 -
281 - half A_shared_warp[8];
282 - half B_shared_warp[16];
283 - for (int j_0_4_init = 0; j_0_4_init < 2; ++j_0_4_init) {
284 - for (int i = 0; i < 8; ++i) {
285 - C_warp[(j_0_4_init * 8) + i] = 0.0;
286 - }
287 - }
288 -
289 - static constexpr int row_stride_warp = 32 * 8 / 32;
290 - static constexpr int row_stride = 2 * 32 * 8 / 64;
291 - bool ld_zero_flag = (threadIdx.y * 32 + threadIdx.x) * 8 < 64;
292 - // TODO: Haotian: blockIdx_y / j_factors1 in A loading to support bsz > 16
293 - bool ld_A_flag = (blockIdx_y / j_factors1 * 16 + threadIdx.y * row_stride_warp + threadIdx.x * 8 / 32) < M; // threadIdx.y is warp_id
294 - // bool wb_C_flag = (threadIdx.x / 4) < M;
295 -
296 - half* A_ptr = A
297 - + (((int)blockIdx_y) / j_factors1 * 16 + (((int)threadIdx.y) * row_stride_warp) + ((int)threadIdx.x) / (32 / 8)) * IC
298 - + (((int)threadIdx.x) % (32 / 8)) * 8;
299 -
300 - int* B_ptr = B
301 - + ((int)threadIdx.y) * (OC / 8) * 4
302 - + (((int)threadIdx.x) / (64 / 8)) * (OC / 8)
303 - + (((int)blockIdx_y) % j_factors1) * (64 / 8)
304 - + (((int)threadIdx.x) % (64 / 8)) * 1;
305 -// Why * 1 in the above line?
306 -
307 - half* A_shared_ptr = A_shared
308 - + ((int)threadIdx.y) * row_stride_warp * (32 + 8)
309 - + (((int)threadIdx.x) / (32 / 8)) * (32 + 8)
310 - + (((int)threadIdx.x) % (32 / 8) ) * 8;
311 -
312 - half* B_shared_ptr = B_shared
313 - + ((int)threadIdx.y) * (row_stride / 2) * (64 + 8)
314 - + (((int)threadIdx.x) / (64 / 8)) * (64 + 8)
315 - + (((int)threadIdx.x) % (64 / 8)) * 8;
316 -
317 - int* zeros_ptr = zeros
318 - + (((int)blockIdx_y) % j_factors1) * (64 / 8)
319 - + ((int)threadIdx.x) % (64 / 8);
320 -
321 - half* scaling_factors_ptr = scaling_factors
322 - + (((int)blockIdx_y) % j_factors1) * (64)
323 - + (((int)threadIdx.x) % (64 / 8)) * 8;
324 -
325 - half* C_ptr = C
326 - + static_cast<long long>(blockIdx_z) * M * OC // blockIdz.x -> split_k dim
327 - + (((int)blockIdx_y) % j_factors1) * 64
328 - + ((int)threadIdx.y) * 32
329 - + (((int)threadIdx.x) % 4) * 2;
330 -
331 - // preload s.f. and zeros
332 - int k_bound = (IC / 32 + split_k_iters - 1) / split_k_iters;
333 - if ((k_bound - 1) * split_k_iters * 32 + blockIdx_z * 32 >= IC) k_bound -= 1;
334 - for (int _k_0_0 = 0; _k_0_0 < k_bound; ++_k_0_0) {
335 - int k_0_0 = _k_0_0 * split_k_iters + blockIdx_z;
336 - __syncthreads();
337 - // TODO: Haotian: blockIdx_y / j_factors1 in A loading to support bsz > 16
338 - if (ld_A_flag)
339 - {
340 - *(uint4*)(A_shared_ptr) = *(uint4*)(A_ptr + (k_0_0 * 32));
341 - }
342 - else
343 - {
344 - *(uint4*)(A_shared_ptr) = make_uint4(0, 0, 0, 0);
345 - }
346 -
347 - // for (int ax0_ax1_fused_0 = 0; ax0_ax1_fused_0 < 2; ++ax0_ax1_fused_0) {
348 - uint32_t zeros_loaded = *(uint32_t*)(zeros_ptr + k_0_0 * 32 / G * (OC / 8));
349 - uint4 B_loaded_zero = dequantize_s4_to_fp16x2(zeros_loaded);
350 - uint4 B_loaded_scale = *(uint4*)(scaling_factors_ptr + k_0_0 * 32 / G * (OC));
351 - /*
352 - if (blockIdx_z == 0 && blockIdx_y == 0 && k_0_0 == 0 && threadIdx.x == 0 && threadIdx.y == 0){
353 - printf("%x %x %x %x %x %x %x %x\n", B_loaded_scale.x, B_loaded_scale.y, B_loaded_scale.z, B_loaded_scale.w, B_loaded_zero.x, B_loaded_zero.y, B_loaded_zero.z, B_loaded_zero.w);
354 - }
355 - */
356 - // uint4 B_loaded_scale = make_uint4(0, 0, 0, 0);
357 - int* B_ptr_local = B_ptr + k_0_0 * 32 * (OC / 8);
358 -
359 - for (int ax0_ax1_fused_0 = 0; ax0_ax1_fused_0 < 4; ++ax0_ax1_fused_0) {
360 -
361 - // B: 32 x 136 (128+8) float16
362 - // each warp: 32 x 4
363 - // each thr: read 32 bit -> convert to 8xFP16 (a UINT4) -> scale and minus zero -> WB UINT4
364 - // *(uint4*)(B_shared + ((((ax0_ax1_fused_0 * 544) + (((int)threadIdx.y) * 272)) + ((((int)threadIdx.x) >> 4) * 136)) + ((((int)threadIdx.x) & 15) * 8))) = *(uint4*)(B + ((((((k_0_0 * 163840) + (ax0_ax1_fused_0 * 20480)) + (((int)threadIdx.y) * 10240)) + ((((int)threadIdx.x) >> 4) * 5120)) + (((int)blockIdx_y) * 128)) + ((((int)threadIdx.x) & 15) * 8)));
365 - // row stride in shared memory: (NWARPS * 32 * 8 / cta_N)
366 - uint32_t B_loaded = *(uint32_t*)(B_ptr_local + ax0_ax1_fused_0 * row_stride * (OC / 8));
367 - uint4 B_loaded_fp16 = dequantize_s4_to_fp16x2(B_loaded);
368 - //uint4 B_loaded_zero = *(uint4*)(zeros_shared + (threadIdx.x % (cta_N / 8)) * 8);
369 -
370 - // uint4 B_loaded_scale = *(uint4*)(scaling_factors_shared + (threadIdx.x % (cta_N / 8)) * 8);
371 - // - zero and * scale
372 - // TODO (Haotian): can save 4 assembly instructions if sormulate as deq = q * scale - zero * scale.
373 - asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.x) : "r"(B_loaded_fp16.x), "r"(B_loaded_zero.x));
374 - asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.x) : "r"(B_loaded_fp16.x), "r"(B_loaded_scale.x), "r"(ZERO));
375 - asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.y) : "r"(B_loaded_fp16.y), "r"(B_loaded_zero.y));
376 - asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.y) : "r"(B_loaded_fp16.y), "r"(B_loaded_scale.y), "r"(ZERO));
377 - asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.z) : "r"(B_loaded_fp16.z), "r"(B_loaded_zero.z));
378 - asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.z) : "r"(B_loaded_fp16.z), "r"(B_loaded_scale.z), "r"(ZERO));
379 - asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.w) : "r"(B_loaded_fp16.w), "r"(B_loaded_zero.w));
380 - asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.w) : "r"(B_loaded_fp16.w), "r"(B_loaded_scale.w), "r"(ZERO));
381 - /*
382 - if (ax0_ax1_fused_0 == 0 && blockIdx_z == 0 && blockIdx_y == 0 && k_0_0 == 0 && threadIdx.x == 17 && threadIdx.y == 0){
383 - printf("[x] %X %X %X %X\n", B_loaded_fp16.x, B_loaded_fp16.y, B_loaded_fp16.z, B_loaded_fp16.w);
384 - }
385 - */
386 -
387 - // write back
388 - *(uint4*)(B_shared_ptr + ax0_ax1_fused_0 * row_stride * (64 + 8)) = B_loaded_fp16;
389 - }
390 - __syncthreads();
391 -
392 - for (int k_0_1 = 0; k_0_1 < 2; ++k_0_1)
393 - {
394 - {
395 - unsigned int addr;
396 - __asm__ __volatile__(
397 - "{ .reg .u64 addr; cvta.to.shared.u64 addr, %1; cvt.u32.u64 %0, addr; }\n"
398 - : "=r"(addr)
399 - : "l"((void *)((&(A_shared[(k_0_1 * 16)])) + (((((int)threadIdx.x) & 15) * 40) + ((((int)threadIdx.x) >> 4) * 8))))
400 - );
401 - __asm__ __volatile__(
402 - "ldmatrix.sync.aligned.m8n8.x4.shared.b16"
403 - "{%0, %1, %2, %3}, [%4];\n"
404 - : "=r"(((unsigned *)(A_shared_warp + 0))[0]), "=r"(((unsigned *)(A_shared_warp + 0))[1]), "=r"(((unsigned *)(A_shared_warp + 0))[2]), "=r"(((unsigned *)(A_shared_warp + 0))[3])
405 - : "r"(addr)
406 - );
407 - }
408 -
409 -
410 - for (int ax1_0 = 0; ax1_0 < 2; ++ax1_0)
411 - {
412 - {
413 - unsigned int addr;
414 - __asm__ __volatile__(
415 - "{ .reg .u64 addr; cvta.to.shared.u64 addr, %1; cvt.u32.u64 %0, addr; }\n"
416 - : "=r"(addr)
417 - : "l"((void *)((&(B_shared[(((k_0_1 * 1152) + (((int)threadIdx.y) * 32)) + (ax1_0 * 16))])) + (((((int)threadIdx.x) & 15) * 72) + ((((int)threadIdx.x) >> 4) * 8))))
418 - );
419 - __asm__ __volatile__(
420 - "ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16"
421 - "{%0, %1, %2, %3}, [%4];\n"
422 - : "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[0]), "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[1]), "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[2]), "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[3])
423 - : "r"(addr)
424 - );
425 - }
426 - }
427 -
428 - for (int j_0_4 = 0; j_0_4 < 2; ++j_0_4)
429 - {
430 -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ == 750
431 - {
432 - __asm__ __volatile__(
433 - "mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32"
434 - "{%0, %1, %2, %3}, {%4, %5}, {%6}, {%7, %8, %9, %10};\n"
435 - : "=f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[3])
436 - : "r"(((unsigned *)(A_shared_warp + 0))[0]), "r"(((unsigned *)(A_shared_warp + 0))[1]), "r"(((unsigned *)(B_shared_warp + (j_0_4 * 8)))[0]), "f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "f"(((float *)(C_warp + (j_0_4 * 8)))[3]));
437 - }
438 -
439 - {
440 - __asm__ __volatile__(
441 - "mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32"
442 - "{%0, %1, %2, %3}, {%4, %5}, {%6}, {%7, %8, %9, %10};\n"
443 - : "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3])
444 - : "r"(((unsigned *)(A_shared_warp + 0))[0]), "r"(((unsigned *)(A_shared_warp + 0))[1]), "r"(((unsigned *)(B_shared_warp + ((j_0_4 * 8) + 4)))[0]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3]));
445 - }
446 -
447 - {
448 - __asm__ __volatile__(
449 - "mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32"
450 - "{%0, %1, %2, %3}, {%4, %5}, {%6}, {%7, %8, %9, %10};\n"
451 - : "=f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[3])
452 - : "r"(((unsigned *)(A_shared_warp + 0))[2]), "r"(((unsigned *)(A_shared_warp + 0))[3]), "r"(((unsigned *)(B_shared_warp + (j_0_4 * 8)))[1]), "f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "f"(((float *)(C_warp + (j_0_4 * 8)))[3]));
453 - }
454 -
455 - {
456 - __asm__ __volatile__(
457 - "mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32"
458 - "{%0, %1, %2, %3}, {%4, %5}, {%6}, {%7, %8, %9, %10};\n"
459 - : "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3])
460 - : "r"(((unsigned *)(A_shared_warp + 0))[2]), "r"(((unsigned *)(A_shared_warp + 0))[3]), "r"(((unsigned *)(B_shared_warp + ((j_0_4 * 8) + 4)))[1]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3]));
461 - }
462 -#else
463 - {
464 - __asm__ __volatile__(
465 - "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32"
466 - "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, %13};\n"
467 - : "=f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[3])
468 - : "r"(((unsigned *)(A_shared_warp + 0))[0]), "r"(((unsigned *)(A_shared_warp + 0))[1]), "r"(((unsigned *)(A_shared_warp + 0))[2]), "r"(((unsigned *)(A_shared_warp + 0))[3]), "r"(((unsigned *)(B_shared_warp + (j_0_4 * 8)))[0]), "r"(((unsigned *)(B_shared_warp + (j_0_4 * 8)))[1]), "f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "f"(((float *)(C_warp + (j_0_4 * 8)))[3]));
469 - }
470 -
471 - {
472 - __asm__ __volatile__(
473 - "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32"
474 - "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, %13};\n"
475 - : "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3])
476 - : "r"(((unsigned *)(A_shared_warp + 0))[0]), "r"(((unsigned *)(A_shared_warp + 0))[1]), "r"(((unsigned *)(A_shared_warp + 0))[2]), "r"(((unsigned *)(A_shared_warp + 0))[3]), "r"(((unsigned *)(B_shared_warp + ((j_0_4 * 8) + 4)))[0]), "r"(((unsigned *)(B_shared_warp + ((j_0_4 * 8) + 4)))[1]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3]));
477 - }
478 -#endif
479 - }
480 - }
481 - }
482 -
483 -// TODO: Shang: Hoist loop invariance.
484 - for (int ax1_0_1 = 0; ax1_0_1 < 2; ++ax1_0_1) {
485 - for (int local_id = 0; local_id < 8; ++local_id) {
486 - int row_offset = (((int)blockIdx_y) / j_factors1) * 16 + ((int)threadIdx.x) / 4 + (local_id % 4) / 2 * 8;
487 - if (row_offset < M)
488 - {
489 - *(C_ptr + ax1_0_1 * 16 + row_offset * OC + (local_id / 4) * 8 + local_id % 2) = __float2half(C_warp[(ax1_0_1 * 8) + local_id]);
490 - }
491 - }
492 - }
493 -#endif
494 -}
495 -
274 __global__ void __launch_bounds__(64) dequantize_weights(
275 int* __restrict__ B,
276 half* __restrict__ scaling_factors,
@@ -526,26 +304,24 @@ __global__ void __launch_bounds__(64) dequantize_weights(
304 int index4 = 8 * col + (int)(row / G) * N * 8;
305 half* scaling_factors_ptr2 = scaling_factors + index4;
306
307 + uint32_t zeros_loaded = *(uint32_t*)(zeros_ptr2);
308 + uint4 B_loaded_zero = dequantize_s4_to_fp16x2(zeros_loaded);
309 + uint4 B_loaded_scale = *(uint4*)(scaling_factors_ptr2);
310
530 - uint32_t zeros_loaded = *(uint32_t*)(zeros_ptr2);
531 - uint4 B_loaded_zero = dequantize_s4_to_fp16x2(zeros_loaded);
532 - uint4 B_loaded_scale = *(uint4*)(scaling_factors_ptr2);
533 -int j=0;
311 + uint32_t B_loaded = *(uint32_t*)B_ptr2;
312 + uint4 B_loaded_fp16 = dequantize_s4_to_fp16x2(B_loaded);
313 + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.x) : "r"(B_loaded_fp16.x), "r"(B_loaded_zero.x));
314 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.x) : "r"(B_loaded_fp16.x), "r"(B_loaded_scale.x), "r"(ZERO));
315 + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.y) : "r"(B_loaded_fp16.y), "r"(B_loaded_zero.y));
316 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.y) : "r"(B_loaded_fp16.y), "r"(B_loaded_scale.y), "r"(ZERO));
317 + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.z) : "r"(B_loaded_fp16.z), "r"(B_loaded_zero.z));
318 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.z) : "r"(B_loaded_fp16.z), "r"(B_loaded_scale.z), "r"(ZERO));
319 + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.w) : "r"(B_loaded_fp16.w), "r"(B_loaded_zero.w));
320 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.w) : "r"(B_loaded_fp16.w), "r"(B_loaded_scale.w), "r"(ZERO));
321
535 - uint32_t B_loaded = *(uint32_t*)(B_ptr2 + j);
536 - uint4 B_loaded_fp16 = dequantize_s4_to_fp16x2(B_loaded);
537 - asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.x) : "r"(B_loaded_fp16.x), "r"(B_loaded_zero.x));
538 - asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.x) : "r"(B_loaded_fp16.x), "r"(B_loaded_scale.x), "r"(ZERO));
539 - asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.y) : "r"(B_loaded_fp16.y), "r"(B_loaded_zero.y));
540 - asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.y) : "r"(B_loaded_fp16.y), "r"(B_loaded_scale.y), "r"(ZERO));
541 - asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.z) : "r"(B_loaded_fp16.z), "r"(B_loaded_zero.z));
542 - asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.z) : "r"(B_loaded_fp16.z), "r"(B_loaded_scale.z), "r"(ZERO));
543 - asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.w) : "r"(B_loaded_fp16.w), "r"(B_loaded_zero.w));
544 - asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.w) : "r"(B_loaded_fp16.w), "r"(B_loaded_scale.w), "r"(ZERO));
545 -
546 - *(uint4*)(B_shared_ptr2 + j) = B_loaded_fp16;
322 + *(uint4*)B_shared_ptr2 = B_loaded_fp16;
323
548 - for (int i=0; i<8; ++i) {
324 + for (int i = 0; i < 8; ++i) {
325 *(C_ptr2 + i) = B_shared[i];
326 }
327 }
@@ -650,19 +426,21 @@ torch::Tensor awq_gemm(
426 // threadIdx.x: 32
427 // threadIdx.y: i_factors[2] * j_factors[2]
428 dim3 threads_per_block(32, 2);
653 - vllm::awq::gemm_forward_4bit_cuda_m16n128k32<<<num_blocks, threads_per_block, 0, stream>>>(
654 - group_size, split_k_iters, in_feats, kernel, scaling_factors, zeros, num_in_feats, num_in_channels, num_out_channels, out_feats);
429 + vllm::awq::gemm_forward_4bit_cuda_m16nXk32<128><<<num_blocks, threads_per_block, 0, stream>>>(
430 + group_size, split_k_iters, in_feats, kernel, scaling_factors, zeros, num_in_feats, num_in_channels,
431 + num_out_channels, out_feats);
432 }
433 else if (num_out_channels % 64 == 0)
434 {
435 int j_factors1 = num_out_channels / 64 / 1;
436 dim3 num_blocks(1 * (num_out_feats + 16 - 1) / 16 * j_factors1 * split_k_iters);
660 -
437 +
438 // threadIdx.x: 32
439 // threadIdx.y: i_factors[2] * j_factors[2]
440 dim3 threads_per_block(32, 2);
664 - vllm::awq::gemm_forward_4bit_cuda_m16n64k32<<<num_blocks, threads_per_block, 0, stream>>>(
665 - group_size, split_k_iters, in_feats, kernel, scaling_factors, zeros, num_in_feats, num_in_channels, num_out_channels, out_feats);
441 + vllm::awq::gemm_forward_4bit_cuda_m16nXk32<64><<<num_blocks, threads_per_block, 0, stream>>>(
442 + group_size, split_k_iters, in_feats, kernel, scaling_factors, zeros, num_in_feats, num_in_channels,
443 + num_out_channels, out_feats);
444 }
445 return _out_feats.sum(0);
446 }
vllm/model_executor/layers/quantization/awq.py
+1 -1
@@ -145,8 +145,8 @@ class AWQLinearMethod(LinearMethodBase):
145 x: torch.Tensor,
146 bias: Optional[torch.Tensor] = None) -> torch.Tensor:
147 qweight = weights["qweight"]
148 - qzeros = weights["qzeros"]
148 scales = weights["scales"]
149 + qzeros = weights["qzeros"]
150 pack_factor = self.quant_config.pack_factor
151 out_shape = (x.shape[:-1] + (qweight.shape[-1] * pack_factor, ))
152 reshaped_x = x.reshape(-1, x.shape[-1])