A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Refactor 2 awq gemm kernels into m16nXk32 (#2723)
Co-authored-by: Chunan Zeng <chunanzeng@Chunans-Air.attlocal.net>
Rex committed
Feb 12, 2024 at 11:02 UTC
563836496abc0914c212b693130f80be25926564
2 files changed
+73
-295
csrc/quantization/awq/gemm_kernels.cu
+72
-294
@@ -27,72 +27,85 @@ __pack_half2(const half x, const half y) {
27
return (v1 << 16) | v0;
28
}
29
30
-__global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16n128k32(int G, int split_k_iters, half* __restrict__ A, int* __restrict__ B, half* __restrict__ scaling_factors, int* __restrict__ zeros, int M, int IC, int OC, half* __restrict__ C)
30
+template<int N>
31
+__global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16nXk32(
32
+ int G,
33
+ int split_k_iters,
34
+ half* __restrict__ A,
35
+ int* __restrict__ B,
36
+ half* __restrict__ scaling_factors,
37
+ int* __restrict__ zeros,
38
+ int M,
39
+ int IC,
40
+ int OC,
41
+ half* __restrict__ C)
42
{
43
+ // Only support matrix n = 64 or 128
44
+ assert(N == 64 || N == 128);
45
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 750
46
assert(false);
47
#else
48
static constexpr uint32_t ZERO = 0x0;
49
float C_warp[32];
50
__shared__ half A_shared[16 * (32 + 8)];
38
- __shared__ half B_shared[32 * (128 + 8)];
39
-
40
- __shared__ half scaling_factors_shared[128];
41
- __shared__ half zeros_shared[128];
51
+ __shared__ half B_shared[32 * (N + 8)];
52
43
- int j_factors1 = ((OC + 128 - 1) / 128);
53
+ __shared__ half scaling_factors_shared[N];
54
+ __shared__ half zeros_shared[N];
55
+
56
+ int j_factors1 = ((OC + N - 1) / N);
57
int blockIdx_x = 0;
58
int blockIdx_y = blockIdx.x % ((M + 16 - 1) / 16 * j_factors1);
59
int blockIdx_z = blockIdx.x / ((M + 16 - 1) / 16 * j_factors1);
60
61
half A_shared_warp[8];
49
- half B_shared_warp[32];
50
- for (int j_0_4_init = 0; j_0_4_init < 4; ++j_0_4_init) {
62
+ half B_shared_warp[N / 4];
63
+ for (int j_0_4_init = 0; j_0_4_init < N / 32; ++j_0_4_init) {
64
for (int i = 0; i < 8; ++i) {
65
C_warp[(j_0_4_init * 8) + i] = 0.0;
66
}
67
}
68
69
static constexpr int row_stride_warp = 32 * 8 / 32;
57
- static constexpr int row_stride = 2 * 32 * 8 / 128;
58
- bool ld_zero_flag = (threadIdx.y * 32 + threadIdx.x) * 8 < 128;
70
+ static constexpr int row_stride = 2 * 32 * 8 / N;
71
+ bool ld_zero_flag = (threadIdx.y * 32 + threadIdx.x) * 8 < N;
72
// TODO: Haotian: blockIdx_y / j_factors1 in A loading to support bsz > 16
73
bool ld_A_flag = (blockIdx_y / j_factors1 * 16 + threadIdx.y * row_stride_warp + threadIdx.x * 8 / 32) < M; // threadIdx.y is warp_id
74
// bool wb_C_flag = (threadIdx.x / 4) < M;
75
63
- half* A_ptr = A
76
+ half* A_ptr = A
77
+ (((int)blockIdx_y) / j_factors1 * 16 + (((int)threadIdx.y) * row_stride_warp) + ((int)threadIdx.x) / (32 / 8)) * IC
78
+ (((int)threadIdx.x) % (32 / 8)) * 8;
66
-
79
+
80
int* B_ptr = B
68
- + ((int)threadIdx.y) * (OC / 8) * 2
69
- + (((int)threadIdx.x) / (128 / 8)) * (OC / 8)
70
- + (((int)blockIdx_y) % j_factors1) * (128 / 8)
71
- + (((int)threadIdx.x) % (128 / 8)) * 1;
81
+ + ((int)threadIdx.y) * (OC / 8) * (256 / N)
82
+ + (((int)threadIdx.x) / (N / 8)) * (OC / 8)
83
+ + (((int)blockIdx_y) % j_factors1) * (N / 8)
84
+ + (((int)threadIdx.x) % (N / 8)) * 1;
85
// Why * 1 in the above line?
73
-
74
- half* A_shared_ptr = A_shared
75
- + ((int)threadIdx.y) * row_stride_warp * (32 + 8)
86
+
87
+ half* A_shared_ptr = A_shared
88
+ + ((int)threadIdx.y) * row_stride_warp * (32 + 8)
89
+ (((int)threadIdx.x) / (32 / 8)) * (32 + 8)
90
+ (((int)threadIdx.x) % (32 / 8) ) * 8;
91
92
half* B_shared_ptr = B_shared
80
- + ((int)threadIdx.y) * (row_stride / 2) * (128 + 8)
81
- + (((int)threadIdx.x) / (128 / 8)) * (128 + 8)
82
- + (((int)threadIdx.x) % (128 / 8)) * 8;
83
-
93
+ + ((int)threadIdx.y) * (row_stride / 2) * (N + 8)
94
+ + (((int)threadIdx.x) / (N / 8)) * (N + 8)
95
+ + (((int)threadIdx.x) % (N / 8)) * 8;
96
+
97
int* zeros_ptr = zeros
85
- + (((int)blockIdx_y) % j_factors1) * (128 / 8)
86
- + ((int)threadIdx.x) % (128 / 8);
87
-
98
+ + (((int)blockIdx_y) % j_factors1) * (N / 8)
99
+ + ((int)threadIdx.x) % (N / 8);
100
+
101
half* scaling_factors_ptr = scaling_factors
89
- + (((int)blockIdx_y) % j_factors1) * (128)
90
- + (((int)threadIdx.x) % (128 / 8)) * 8;
102
+ + (((int)blockIdx_y) % j_factors1) * N
103
+ + (((int)threadIdx.x) % (N / 8)) * 8;
104
92
- half* C_ptr = C
105
+ half* C_ptr = C
106
+ static_cast<long long>(blockIdx_z) * M * OC // blockIdz.x -> split_k dim
94
- + (((int)blockIdx_y) % j_factors1) * 128
95
- + ((int)threadIdx.y) * 64
107
+ + (((int)blockIdx_y) % j_factors1) * N
108
+ + ((int)threadIdx.y) * (N / 2)
109
+ (((int)threadIdx.x) % 4) * 2;
110
111
// preload s.f. and zeros
@@ -123,13 +136,13 @@ __global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16n128k32(int G, i
136
// uint4 B_loaded_scale = make_uint4(0, 0, 0, 0);
137
int* B_ptr_local = B_ptr + k_0_0 * 32 * (OC / 8);
138
126
- for (int ax0_ax1_fused_0 = 0; ax0_ax1_fused_0 < 8; ++ax0_ax1_fused_0) {
139
+ for (int ax0_ax1_fused_0 = 0; ax0_ax1_fused_0 < N / 16; ++ax0_ax1_fused_0) {
140
141
// B: 32 x 136 (128+8) float16
142
// each warp: 32 x 4
143
// each thr: read 32 bit -> convert to 8xFP16 (a UINT4) -> scale and minus zero -> WB UINT4
144
// *(uint4*)(B_shared + ((((ax0_ax1_fused_0 * 544) + (((int)threadIdx.y) * 272)) + ((((int)threadIdx.x) >> 4) * 136)) + ((((int)threadIdx.x) & 15) * 8))) = *(uint4*)(B + ((((((k_0_0 * 163840) + (ax0_ax1_fused_0 * 20480)) + (((int)threadIdx.y) * 10240)) + ((((int)threadIdx.x) >> 4) * 5120)) + (((int)blockIdx_y) * 128)) + ((((int)threadIdx.x) & 15) * 8)));
132
- // row stride in shared memory: (NWARPS * 32 * 8 / cta_N)
145
+ // row stride in shared memory: (NWARPS * 32 * 8 / cta_N)
146
uint32_t B_loaded = *(uint32_t*)(B_ptr_local + ax0_ax1_fused_0 * row_stride * (OC / 8));
147
uint4 B_loaded_fp16 = dequantize_s4_to_fp16x2(B_loaded);
148
//uint4 B_loaded_zero = *(uint4*)(zeros_shared + (threadIdx.x % (cta_N / 8)) * 8);
@@ -152,7 +165,7 @@ __global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16n128k32(int G, i
165
*/
166
167
// write back
155
- *(uint4*)(B_shared_ptr + ax0_ax1_fused_0 * row_stride * (128 + 8)) = B_loaded_fp16;
168
+ *(uint4*)(B_shared_ptr + ax0_ax1_fused_0 * row_stride * (N + 8)) = B_loaded_fp16;
169
}
170
__syncthreads();
171
@@ -174,13 +187,13 @@ __global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16n128k32(int G, i
187
);
188
}
189
177
- for (int ax1_0 = 0; ax1_0 < 4; ++ax1_0) {
190
+ for (int ax1_0 = 0; ax1_0 < N / 32; ++ax1_0) {
191
{
192
unsigned int addr;
193
__asm__ __volatile__(
194
"{ .reg .u64 addr; cvta.to.shared.u64 addr, %1; cvt.u32.u64 %0, addr; }\n"
195
: "=r"(addr)
183
- : "l"((void *)((&(B_shared[(((k_0_1 * 2176) + (((int)threadIdx.y) * 64)) + (ax1_0 * 16))])) + (((((int)threadIdx.x) & 15) * 136) + ((((int)threadIdx.x) >> 4) * 8))))
196
+ : "l"((void *)((&(B_shared[(((k_0_1 * (N * 16 + 128)) + (((int)threadIdx.y) * (N / 2))) + (ax1_0 * 16))])) + (((((int)threadIdx.x) & 15) * (N + 8)) + ((((int)threadIdx.x) >> 4) * 8))))
197
);
198
__asm__ __volatile__(
199
"ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16"
@@ -190,7 +203,7 @@ __global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16n128k32(int G, i
203
);
204
}
205
}
193
- for (int j_0_4 = 0; j_0_4 < 4; ++j_0_4) {
206
+ for (int j_0_4 = 0; j_0_4 < N / 32; ++j_0_4) {
207
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ == 750
208
{
209
__asm__ __volatile__(
@@ -258,241 +271,6 @@ __global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16n128k32(int G, i
271
#endif
272
}
273
261
-
262
-__global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16n64k32(int G, int split_k_iters, half* __restrict__ A, int* __restrict__ B, half* __restrict__ scaling_factors, int* __restrict__ zeros, int M, int IC, int OC, half* __restrict__ C)
263
-{
264
-#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 750
265
- assert(false);
266
-#else
267
- static constexpr uint32_t ZERO = 0x0;
268
- float C_warp[32];
269
- __shared__ half A_shared[16 * (32 + 8)];
270
- __shared__ half B_shared[32 * (64 + 8)];
271
-
272
- __shared__ half scaling_factors_shared[64];
273
- __shared__ half zeros_shared[64];
274
-
275
- int j_factors1 = ((OC + 64 - 1) / 64);
276
-
277
- int blockIdx_x = 0;
278
- int blockIdx_y = blockIdx.x % ((M + 16 - 1) / 16 * j_factors1);
279
- int blockIdx_z = blockIdx.x / ((M + 16 - 1) / 16 * j_factors1);
280
-
281
- half A_shared_warp[8];
282
- half B_shared_warp[16];
283
- for (int j_0_4_init = 0; j_0_4_init < 2; ++j_0_4_init) {
284
- for (int i = 0; i < 8; ++i) {
285
- C_warp[(j_0_4_init * 8) + i] = 0.0;
286
- }
287
- }
288
-
289
- static constexpr int row_stride_warp = 32 * 8 / 32;
290
- static constexpr int row_stride = 2 * 32 * 8 / 64;
291
- bool ld_zero_flag = (threadIdx.y * 32 + threadIdx.x) * 8 < 64;
292
- // TODO: Haotian: blockIdx_y / j_factors1 in A loading to support bsz > 16
293
- bool ld_A_flag = (blockIdx_y / j_factors1 * 16 + threadIdx.y * row_stride_warp + threadIdx.x * 8 / 32) < M; // threadIdx.y is warp_id
294
- // bool wb_C_flag = (threadIdx.x / 4) < M;
295
-
296
- half* A_ptr = A
297
- + (((int)blockIdx_y) / j_factors1 * 16 + (((int)threadIdx.y) * row_stride_warp) + ((int)threadIdx.x) / (32 / 8)) * IC
298
- + (((int)threadIdx.x) % (32 / 8)) * 8;
299
-
300
- int* B_ptr = B
301
- + ((int)threadIdx.y) * (OC / 8) * 4
302
- + (((int)threadIdx.x) / (64 / 8)) * (OC / 8)
303
- + (((int)blockIdx_y) % j_factors1) * (64 / 8)
304
- + (((int)threadIdx.x) % (64 / 8)) * 1;
305
-// Why * 1 in the above line?
306
-
307
- half* A_shared_ptr = A_shared
308
- + ((int)threadIdx.y) * row_stride_warp * (32 + 8)
309
- + (((int)threadIdx.x) / (32 / 8)) * (32 + 8)
310
- + (((int)threadIdx.x) % (32 / 8) ) * 8;
311
-
312
- half* B_shared_ptr = B_shared
313
- + ((int)threadIdx.y) * (row_stride / 2) * (64 + 8)
314
- + (((int)threadIdx.x) / (64 / 8)) * (64 + 8)
315
- + (((int)threadIdx.x) % (64 / 8)) * 8;
316
-
317
- int* zeros_ptr = zeros
318
- + (((int)blockIdx_y) % j_factors1) * (64 / 8)
319
- + ((int)threadIdx.x) % (64 / 8);
320
-
321
- half* scaling_factors_ptr = scaling_factors
322
- + (((int)blockIdx_y) % j_factors1) * (64)
323
- + (((int)threadIdx.x) % (64 / 8)) * 8;
324
-
325
- half* C_ptr = C
326
- + static_cast<long long>(blockIdx_z) * M * OC // blockIdz.x -> split_k dim
327
- + (((int)blockIdx_y) % j_factors1) * 64
328
- + ((int)threadIdx.y) * 32
329
- + (((int)threadIdx.x) % 4) * 2;
330
-
331
- // preload s.f. and zeros
332
- int k_bound = (IC / 32 + split_k_iters - 1) / split_k_iters;
333
- if ((k_bound - 1) * split_k_iters * 32 + blockIdx_z * 32 >= IC) k_bound -= 1;
334
- for (int _k_0_0 = 0; _k_0_0 < k_bound; ++_k_0_0) {
335
- int k_0_0 = _k_0_0 * split_k_iters + blockIdx_z;
336
- __syncthreads();
337
- // TODO: Haotian: blockIdx_y / j_factors1 in A loading to support bsz > 16
338
- if (ld_A_flag)
339
- {
340
- *(uint4*)(A_shared_ptr) = *(uint4*)(A_ptr + (k_0_0 * 32));
341
- }
342
- else
343
- {
344
- *(uint4*)(A_shared_ptr) = make_uint4(0, 0, 0, 0);
345
- }
346
-
347
- // for (int ax0_ax1_fused_0 = 0; ax0_ax1_fused_0 < 2; ++ax0_ax1_fused_0) {
348
- uint32_t zeros_loaded = *(uint32_t*)(zeros_ptr + k_0_0 * 32 / G * (OC / 8));
349
- uint4 B_loaded_zero = dequantize_s4_to_fp16x2(zeros_loaded);
350
- uint4 B_loaded_scale = *(uint4*)(scaling_factors_ptr + k_0_0 * 32 / G * (OC));
351
- /*
352
- if (blockIdx_z == 0 && blockIdx_y == 0 && k_0_0 == 0 && threadIdx.x == 0 && threadIdx.y == 0){
353
- printf("%x %x %x %x %x %x %x %x\n", B_loaded_scale.x, B_loaded_scale.y, B_loaded_scale.z, B_loaded_scale.w, B_loaded_zero.x, B_loaded_zero.y, B_loaded_zero.z, B_loaded_zero.w);
354
- }
355
- */
356
- // uint4 B_loaded_scale = make_uint4(0, 0, 0, 0);
357
- int* B_ptr_local = B_ptr + k_0_0 * 32 * (OC / 8);
358
-
359
- for (int ax0_ax1_fused_0 = 0; ax0_ax1_fused_0 < 4; ++ax0_ax1_fused_0) {
360
-
361
- // B: 32 x 136 (128+8) float16
362
- // each warp: 32 x 4
363
- // each thr: read 32 bit -> convert to 8xFP16 (a UINT4) -> scale and minus zero -> WB UINT4
364
- // *(uint4*)(B_shared + ((((ax0_ax1_fused_0 * 544) + (((int)threadIdx.y) * 272)) + ((((int)threadIdx.x) >> 4) * 136)) + ((((int)threadIdx.x) & 15) * 8))) = *(uint4*)(B + ((((((k_0_0 * 163840) + (ax0_ax1_fused_0 * 20480)) + (((int)threadIdx.y) * 10240)) + ((((int)threadIdx.x) >> 4) * 5120)) + (((int)blockIdx_y) * 128)) + ((((int)threadIdx.x) & 15) * 8)));
365
- // row stride in shared memory: (NWARPS * 32 * 8 / cta_N)
366
- uint32_t B_loaded = *(uint32_t*)(B_ptr_local + ax0_ax1_fused_0 * row_stride * (OC / 8));
367
- uint4 B_loaded_fp16 = dequantize_s4_to_fp16x2(B_loaded);
368
- //uint4 B_loaded_zero = *(uint4*)(zeros_shared + (threadIdx.x % (cta_N / 8)) * 8);
369
-
370
- // uint4 B_loaded_scale = *(uint4*)(scaling_factors_shared + (threadIdx.x % (cta_N / 8)) * 8);
371
- // - zero and * scale
372
- // TODO (Haotian): can save 4 assembly instructions if sormulate as deq = q * scale - zero * scale.
373
- asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.x) : "r"(B_loaded_fp16.x), "r"(B_loaded_zero.x));
374
- asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.x) : "r"(B_loaded_fp16.x), "r"(B_loaded_scale.x), "r"(ZERO));
375
- asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.y) : "r"(B_loaded_fp16.y), "r"(B_loaded_zero.y));
376
- asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.y) : "r"(B_loaded_fp16.y), "r"(B_loaded_scale.y), "r"(ZERO));
377
- asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.z) : "r"(B_loaded_fp16.z), "r"(B_loaded_zero.z));
378
- asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.z) : "r"(B_loaded_fp16.z), "r"(B_loaded_scale.z), "r"(ZERO));
379
- asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.w) : "r"(B_loaded_fp16.w), "r"(B_loaded_zero.w));
380
- asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.w) : "r"(B_loaded_fp16.w), "r"(B_loaded_scale.w), "r"(ZERO));
381
- /*
382
- if (ax0_ax1_fused_0 == 0 && blockIdx_z == 0 && blockIdx_y == 0 && k_0_0 == 0 && threadIdx.x == 17 && threadIdx.y == 0){
383
- printf("[x] %X %X %X %X\n", B_loaded_fp16.x, B_loaded_fp16.y, B_loaded_fp16.z, B_loaded_fp16.w);
384
- }
385
- */
386
-
387
- // write back
388
- *(uint4*)(B_shared_ptr + ax0_ax1_fused_0 * row_stride * (64 + 8)) = B_loaded_fp16;
389
- }
390
- __syncthreads();
391
-
392
- for (int k_0_1 = 0; k_0_1 < 2; ++k_0_1)
393
- {
394
- {
395
- unsigned int addr;
396
- __asm__ __volatile__(
397
- "{ .reg .u64 addr; cvta.to.shared.u64 addr, %1; cvt.u32.u64 %0, addr; }\n"
398
- : "=r"(addr)
399
- : "l"((void *)((&(A_shared[(k_0_1 * 16)])) + (((((int)threadIdx.x) & 15) * 40) + ((((int)threadIdx.x) >> 4) * 8))))
400
- );
401
- __asm__ __volatile__(
402
- "ldmatrix.sync.aligned.m8n8.x4.shared.b16"
403
- "{%0, %1, %2, %3}, [%4];\n"
404
- : "=r"(((unsigned *)(A_shared_warp + 0))[0]), "=r"(((unsigned *)(A_shared_warp + 0))[1]), "=r"(((unsigned *)(A_shared_warp + 0))[2]), "=r"(((unsigned *)(A_shared_warp + 0))[3])
405
- : "r"(addr)
406
- );
407
- }
408
-
409
-
410
- for (int ax1_0 = 0; ax1_0 < 2; ++ax1_0)
411
- {
412
- {
413
- unsigned int addr;
414
- __asm__ __volatile__(
415
- "{ .reg .u64 addr; cvta.to.shared.u64 addr, %1; cvt.u32.u64 %0, addr; }\n"
416
- : "=r"(addr)
417
- : "l"((void *)((&(B_shared[(((k_0_1 * 1152) + (((int)threadIdx.y) * 32)) + (ax1_0 * 16))])) + (((((int)threadIdx.x) & 15) * 72) + ((((int)threadIdx.x) >> 4) * 8))))
418
- );
419
- __asm__ __volatile__(
420
- "ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16"
421
- "{%0, %1, %2, %3}, [%4];\n"
422
- : "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[0]), "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[1]), "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[2]), "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[3])
423
- : "r"(addr)
424
- );
425
- }
426
- }
427
-
428
- for (int j_0_4 = 0; j_0_4 < 2; ++j_0_4)
429
- {
430
-#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ == 750
431
- {
432
- __asm__ __volatile__(
433
- "mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32"
434
- "{%0, %1, %2, %3}, {%4, %5}, {%6}, {%7, %8, %9, %10};\n"
435
- : "=f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[3])
436
- : "r"(((unsigned *)(A_shared_warp + 0))[0]), "r"(((unsigned *)(A_shared_warp + 0))[1]), "r"(((unsigned *)(B_shared_warp + (j_0_4 * 8)))[0]), "f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "f"(((float *)(C_warp + (j_0_4 * 8)))[3]));
437
- }
438
-
439
- {
440
- __asm__ __volatile__(
441
- "mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32"
442
- "{%0, %1, %2, %3}, {%4, %5}, {%6}, {%7, %8, %9, %10};\n"
443
- : "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3])
444
- : "r"(((unsigned *)(A_shared_warp + 0))[0]), "r"(((unsigned *)(A_shared_warp + 0))[1]), "r"(((unsigned *)(B_shared_warp + ((j_0_4 * 8) + 4)))[0]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3]));
445
- }
446
-
447
- {
448
- __asm__ __volatile__(
449
- "mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32"
450
- "{%0, %1, %2, %3}, {%4, %5}, {%6}, {%7, %8, %9, %10};\n"
451
- : "=f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[3])
452
- : "r"(((unsigned *)(A_shared_warp + 0))[2]), "r"(((unsigned *)(A_shared_warp + 0))[3]), "r"(((unsigned *)(B_shared_warp + (j_0_4 * 8)))[1]), "f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "f"(((float *)(C_warp + (j_0_4 * 8)))[3]));
453
- }
454
-
455
- {
456
- __asm__ __volatile__(
457
- "mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32"
458
- "{%0, %1, %2, %3}, {%4, %5}, {%6}, {%7, %8, %9, %10};\n"
459
- : "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3])
460
- : "r"(((unsigned *)(A_shared_warp + 0))[2]), "r"(((unsigned *)(A_shared_warp + 0))[3]), "r"(((unsigned *)(B_shared_warp + ((j_0_4 * 8) + 4)))[1]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3]));
461
- }
462
-#else
463
- {
464
- __asm__ __volatile__(
465
- "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32"
466
- "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, %13};\n"
467
- : "=f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[3])
468
- : "r"(((unsigned *)(A_shared_warp + 0))[0]), "r"(((unsigned *)(A_shared_warp + 0))[1]), "r"(((unsigned *)(A_shared_warp + 0))[2]), "r"(((unsigned *)(A_shared_warp + 0))[3]), "r"(((unsigned *)(B_shared_warp + (j_0_4 * 8)))[0]), "r"(((unsigned *)(B_shared_warp + (j_0_4 * 8)))[1]), "f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "f"(((float *)(C_warp + (j_0_4 * 8)))[3]));
469
- }
470
-
471
- {
472
- __asm__ __volatile__(
473
- "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32"
474
- "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, %13};\n"
475
- : "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3])
476
- : "r"(((unsigned *)(A_shared_warp + 0))[0]), "r"(((unsigned *)(A_shared_warp + 0))[1]), "r"(((unsigned *)(A_shared_warp + 0))[2]), "r"(((unsigned *)(A_shared_warp + 0))[3]), "r"(((unsigned *)(B_shared_warp + ((j_0_4 * 8) + 4)))[0]), "r"(((unsigned *)(B_shared_warp + ((j_0_4 * 8) + 4)))[1]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3]));
477
- }
478
-#endif
479
- }
480
- }
481
- }
482
-
483
-// TODO: Shang: Hoist loop invariance.
484
- for (int ax1_0_1 = 0; ax1_0_1 < 2; ++ax1_0_1) {
485
- for (int local_id = 0; local_id < 8; ++local_id) {
486
- int row_offset = (((int)blockIdx_y) / j_factors1) * 16 + ((int)threadIdx.x) / 4 + (local_id % 4) / 2 * 8;
487
- if (row_offset < M)
488
- {
489
- *(C_ptr + ax1_0_1 * 16 + row_offset * OC + (local_id / 4) * 8 + local_id % 2) = __float2half(C_warp[(ax1_0_1 * 8) + local_id]);
490
- }
491
- }
492
- }
493
-#endif
494
-}
495
-
274
__global__ void __launch_bounds__(64) dequantize_weights(
275
int* __restrict__ B,
276
half* __restrict__ scaling_factors,
@@ -526,26 +304,24 @@ __global__ void __launch_bounds__(64) dequantize_weights(
304
int index4 = 8 * col + (int)(row / G) * N * 8;
305
half* scaling_factors_ptr2 = scaling_factors + index4;
306
307
+ uint32_t zeros_loaded = *(uint32_t*)(zeros_ptr2);
308
+ uint4 B_loaded_zero = dequantize_s4_to_fp16x2(zeros_loaded);
309
+ uint4 B_loaded_scale = *(uint4*)(scaling_factors_ptr2);
310
530
- uint32_t zeros_loaded = *(uint32_t*)(zeros_ptr2);
531
- uint4 B_loaded_zero = dequantize_s4_to_fp16x2(zeros_loaded);
532
- uint4 B_loaded_scale = *(uint4*)(scaling_factors_ptr2);
533
-int j=0;
311
+ uint32_t B_loaded = *(uint32_t*)B_ptr2;
312
+ uint4 B_loaded_fp16 = dequantize_s4_to_fp16x2(B_loaded);
313
+ asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.x) : "r"(B_loaded_fp16.x), "r"(B_loaded_zero.x));
314
+ asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.x) : "r"(B_loaded_fp16.x), "r"(B_loaded_scale.x), "r"(ZERO));
315
+ asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.y) : "r"(B_loaded_fp16.y), "r"(B_loaded_zero.y));
316
+ asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.y) : "r"(B_loaded_fp16.y), "r"(B_loaded_scale.y), "r"(ZERO));
317
+ asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.z) : "r"(B_loaded_fp16.z), "r"(B_loaded_zero.z));
318
+ asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.z) : "r"(B_loaded_fp16.z), "r"(B_loaded_scale.z), "r"(ZERO));
319
+ asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.w) : "r"(B_loaded_fp16.w), "r"(B_loaded_zero.w));
320
+ asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.w) : "r"(B_loaded_fp16.w), "r"(B_loaded_scale.w), "r"(ZERO));
321
535
- uint32_t B_loaded = *(uint32_t*)(B_ptr2 + j);
536
- uint4 B_loaded_fp16 = dequantize_s4_to_fp16x2(B_loaded);
537
- asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.x) : "r"(B_loaded_fp16.x), "r"(B_loaded_zero.x));
538
- asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.x) : "r"(B_loaded_fp16.x), "r"(B_loaded_scale.x), "r"(ZERO));
539
- asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.y) : "r"(B_loaded_fp16.y), "r"(B_loaded_zero.y));
540
- asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.y) : "r"(B_loaded_fp16.y), "r"(B_loaded_scale.y), "r"(ZERO));
541
- asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.z) : "r"(B_loaded_fp16.z), "r"(B_loaded_zero.z));
542
- asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.z) : "r"(B_loaded_fp16.z), "r"(B_loaded_scale.z), "r"(ZERO));
543
- asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.w) : "r"(B_loaded_fp16.w), "r"(B_loaded_zero.w));
544
- asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.w) : "r"(B_loaded_fp16.w), "r"(B_loaded_scale.w), "r"(ZERO));
545
-
546
- *(uint4*)(B_shared_ptr2 + j) = B_loaded_fp16;
322
+ *(uint4*)B_shared_ptr2 = B_loaded_fp16;
323
548
- for (int i=0; i<8; ++i) {
324
+ for (int i = 0; i < 8; ++i) {
325
*(C_ptr2 + i) = B_shared[i];
326
}
327
}
@@ -650,19 +426,21 @@ torch::Tensor awq_gemm(
426
// threadIdx.x: 32
427
// threadIdx.y: i_factors[2] * j_factors[2]
428
dim3 threads_per_block(32, 2);
653
- vllm::awq::gemm_forward_4bit_cuda_m16n128k32<<<num_blocks, threads_per_block, 0, stream>>>(
654
- group_size, split_k_iters, in_feats, kernel, scaling_factors, zeros, num_in_feats, num_in_channels, num_out_channels, out_feats);
429
+ vllm::awq::gemm_forward_4bit_cuda_m16nXk32<128><<<num_blocks, threads_per_block, 0, stream>>>(
430
+ group_size, split_k_iters, in_feats, kernel, scaling_factors, zeros, num_in_feats, num_in_channels,
431
+ num_out_channels, out_feats);
432
}
433
else if (num_out_channels % 64 == 0)
434
{
435
int j_factors1 = num_out_channels / 64 / 1;
436
dim3 num_blocks(1 * (num_out_feats + 16 - 1) / 16 * j_factors1 * split_k_iters);
660
-
437
+
438
// threadIdx.x: 32
439
// threadIdx.y: i_factors[2] * j_factors[2]
440
dim3 threads_per_block(32, 2);
664
- vllm::awq::gemm_forward_4bit_cuda_m16n64k32<<<num_blocks, threads_per_block, 0, stream>>>(
665
- group_size, split_k_iters, in_feats, kernel, scaling_factors, zeros, num_in_feats, num_in_channels, num_out_channels, out_feats);
441
+ vllm::awq::gemm_forward_4bit_cuda_m16nXk32<64><<<num_blocks, threads_per_block, 0, stream>>>(
442
+ group_size, split_k_iters, in_feats, kernel, scaling_factors, zeros, num_in_feats, num_in_channels,
443
+ num_out_channels, out_feats);
444
}
445
return _out_feats.sum(0);
446
}
vllm/model_executor/layers/quantization/awq.py
+1
-1
@@ -145,8 +145,8 @@ class AWQLinearMethod(LinearMethodBase):
145
x: torch.Tensor,
146
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
147
qweight = weights["qweight"]
148
- qzeros = weights["qzeros"]
148
scales = weights["scales"]
149
+ qzeros = weights["qzeros"]
150
pack_factor = self.quant_config.pack_factor
151
out_shape = (x.shape[:-1] + (qweight.shape[-1] * pack_factor, ))
152
reshaped_x = x.reshape(-1, x.shape[-1])