A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Kernel] Zero point support in fused MarlinMoE kernel + AWQ Fused MoE (#8973)
Co-authored-by: Dipika <dipikasikka1@gmail.com> Co-authored-by: Dipika Sikka <ds3822@columbia.edu>
ElizaWszola committed
Oct 4, 2024 at 20:34 UTC
05d686432f2e13296127962861b21c25cdcdfc8b
23 files changed
+969
-223
CMakeLists.txt
+2
@@ -433,6 +433,8 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
433
"csrc/moe/marlin_kernels/marlin_moe_kernel_ku4b8.cu"
434
"csrc/moe/marlin_kernels/marlin_moe_kernel_ku8b128.h"
435
"csrc/moe/marlin_kernels/marlin_moe_kernel_ku8b128.cu"
436
+ "csrc/moe/marlin_kernels/marlin_moe_kernel_ku4.h"
437
+ "csrc/moe/marlin_kernels/marlin_moe_kernel_ku4.cu"
438
"csrc/moe/marlin_moe_ops.cu")
439
440
set_gencode_flags_for_srcs(
csrc/moe/marlin_kernels/marlin_moe_kernel.h
+244
-53
@@ -38,6 +38,7 @@ using FragA = Vec<half2, 4>;
38
using FragB = Vec<half2, 2>;
39
using FragC = Vec<float, 4>;
40
using FragS = Vec<half2, 1>; // quantization scales
41
+using FragZP = Vec<half2, 4>;
42
43
// Predicated asynchronous global->shared copy; used for inputs A where we apply
44
// predication to handle batchsizes that are not multiples of 16.
@@ -175,6 +176,46 @@ __device__ inline FragB dequant<vllm::kU8B128.id()>(int q) {
176
return frag_b;
177
}
178
179
+template <>
180
+__device__ inline FragB dequant<vllm::kU4.id()>(int q) {
181
+ const int LO = 0x000f000f;
182
+ const int HI = 0x00f000f0;
183
+ const int EX = 0x64006400;
184
+ // Guarantee that the `(a & b) | c` operations are LOP3s.
185
+ int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX);
186
+ int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX);
187
+
188
+ const int SUB = 0x64006400;
189
+ const int MUL = 0x2c002c00;
190
+ const int ADD = 0xd400d400;
191
+ FragB frag_b;
192
+ frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
193
+ *reinterpret_cast<const half2*>(&SUB));
194
+ frag_b[1] = __hfma2(*reinterpret_cast<half2*>(&hi),
195
+ *reinterpret_cast<const half2*>(&MUL),
196
+ *reinterpret_cast<const half2*>(&ADD));
197
+ return frag_b;
198
+}
199
+
200
+template <>
201
+__device__ inline FragB dequant<vllm::kU8.id()>(int q) {
202
+ static constexpr uint32_t mask_for_elt_01 = 0x5250;
203
+ static constexpr uint32_t mask_for_elt_23 = 0x5351;
204
+ static constexpr uint32_t start_byte_for_fp16 = 0x64646464;
205
+
206
+ uint32_t lo = prmt<start_byte_for_fp16, mask_for_elt_01>(q);
207
+ uint32_t hi = prmt<start_byte_for_fp16, mask_for_elt_23>(q);
208
+
209
+ static constexpr uint32_t I8s_TO_F16s_MAGIC_NUM = 0x64006400;
210
+
211
+ FragB frag_b;
212
+ frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
213
+ *reinterpret_cast<const half2*>(&I8s_TO_F16s_MAGIC_NUM));
214
+ frag_b[1] = __hsub2(*reinterpret_cast<half2*>(&hi),
215
+ *reinterpret_cast<const half2*>(&I8s_TO_F16s_MAGIC_NUM));
216
+ return frag_b;
217
+}
218
+
219
// Multiply dequantized values by the corresponding quantization scale; used
220
// only for grouped quantization.
221
__device__ inline void scale(FragB& frag_b, FragS& frag_s, int i) {
@@ -183,11 +224,10 @@ __device__ inline void scale(FragB& frag_b, FragS& frag_s, int i) {
224
frag_b[1] = __hmul2(frag_b[1], s);
225
}
226
186
-// Given 2 floats multiply by 2 scales (halves)
187
-__device__ inline void scale_float(float* c, FragS& s) {
188
- __half* s_ptr = reinterpret_cast<__half*>(&s);
189
- c[0] = __fmul_rn(c[0], __half2float(s_ptr[0]));
190
- c[1] = __fmul_rn(c[1], __half2float(s_ptr[1]));
227
+__device__ inline void sub_zp(FragB& frag_b, half2& frag_zp, int i) {
228
+ half2 zp = __half2half2(reinterpret_cast<__half*>(&frag_zp)[i]);
229
+ frag_b[0] = __hsub2(frag_b[0], zp);
230
+ frag_b[1] = __hsub2(frag_b[1], zp);
231
}
232
233
// Same as above, but for act_order (each K is multiplied individually)
@@ -205,6 +245,13 @@ __device__ inline void scale4(FragB& frag_b, FragS& frag_s_1, FragS& frag_s_2,
245
frag_b[1] = __hmul2(frag_b[1], s_val_3_4);
246
}
247
248
+// Given 2 floats multiply by 2 scales (halves)
249
+__device__ inline void scale_float(float* c, FragS& s) {
250
+ __half* s_ptr = reinterpret_cast<__half*>(&s);
251
+ c[0] = __fmul_rn(c[0], __half2float(s_ptr[0]));
252
+ c[1] = __fmul_rn(c[1], __half2float(s_ptr[1]));
253
+}
254
+
255
// Wait until barrier reaches `count`, then lock for current threadblock.
256
__device__ inline void barrier_acquire(int* lock, int count) {
257
if (threadIdx.x == 0) {
@@ -248,10 +295,11 @@ template <const vllm::ScalarTypeId w_type_id, // weight ScalarType id
295
const int stages, // number of stages for the async global->shared
296
// fetch pipeline
297
const bool has_act_order, // whether act_order is enabled
298
+ const bool has_zp, // whether zero-points are enabled
299
const int group_blocks = -1 // number of consecutive 16x16 blocks
300
// with a separate quantization scale
301
>
254
-__device__ inline void MarlinMoESingle(
302
+__device__ void MarlinMoESingle(
303
const int4* __restrict__ A, // fp16 input matrix of shape mxk
304
const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
305
int4* __restrict__ C, // fp16 output buffer of shape mxn
@@ -259,6 +307,8 @@ __device__ inline void MarlinMoESingle(
307
const float* __restrict__ topk_weights, // float topk weights
308
const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
309
// (k/groupsize)xn
310
+ const int4* __restrict__ zp_ptr, // 4bit packed zero-points of shape
311
+ // (k/groupsize)x(n/pack_factor)
312
const int* __restrict__ g_idx, // int32 group indices of shape k
313
const int* __restrict__ expert_offsets,
314
int num_groups, // number of scale groups per output channel
@@ -400,8 +450,12 @@ __device__ inline void MarlinMoESingle(
450
int tb_n_warps = thread_n_blocks / 4;
451
int act_s_col_tb_stride = act_s_col_warp_stride * tb_n_warps;
452
403
- constexpr int sorted_sh_stride = threads;
404
- constexpr int sorted_gl_stride = threads;
453
+ // Zero-points sizes/strides
454
+ int zp_gl_stride = (prob_n / pack_factor) / 4;
455
+ constexpr int zp_sh_stride = ((16 * thread_n_blocks) / pack_factor) / 4;
456
+ constexpr int zp_tb_groups = s_tb_groups;
457
+ constexpr int zp_sh_stage = has_zp ? zp_tb_groups * zp_sh_stride : 0;
458
+ int zp_gl_rd_delta = zp_gl_stride;
459
460
// Global A read index of current thread.
461
int a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
@@ -442,6 +496,19 @@ __device__ inline void MarlinMoESingle(
496
int s_sh_wr = threadIdx.x;
497
bool s_sh_wr_pred = threadIdx.x < s_sh_stride;
498
499
+ // Zero-points
500
+ int zp_gl_rd;
501
+ if constexpr (has_zp) {
502
+ if constexpr (group_blocks == -1) {
503
+ zp_gl_rd = zp_sh_stride * slice_col + threadIdx.x;
504
+ } else {
505
+ zp_gl_rd = zp_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
506
+ zp_sh_stride * slice_col + threadIdx.x;
507
+ }
508
+ }
509
+ int zp_sh_wr = threadIdx.x;
510
+ bool zp_sh_wr_pred = threadIdx.x < zp_sh_stride;
511
+
512
// We use a different scale layout for grouped and column-wise quantization as
513
// we scale a `half2` tile in column-major layout in the former and in
514
// row-major in the latter case.
@@ -453,23 +520,29 @@ __device__ inline void MarlinMoESingle(
520
s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
521
(threadIdx.x % 32) % 4;
522
523
+ // Zero-points have the same read layout as the scales
524
+ // (without column-wise case)
525
+ constexpr int num_col_threads = 8;
526
+ constexpr int num_row_threads = 4;
527
+ constexpr int num_ints_per_thread = 8 / pack_factor;
528
+ int zp_sh_rd;
529
+ if constexpr (has_zp) {
530
+ zp_sh_rd = num_ints_per_thread * num_col_threads *
531
+ ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
532
+ num_ints_per_thread * ((threadIdx.x % 32) / num_row_threads);
533
+ }
534
+
535
int sh_first_group_id = -1;
536
int sh_num_groups = -1;
537
constexpr int sh_max_num_groups = 32;
538
460
- int shs_size;
461
- if constexpr (has_act_order)
462
- shs_size = sh_max_num_groups * s_sh_stride + threads;
463
- else
464
- shs_size = group_blocks > 0 ? stages * s_sh_stage : threads;
465
-
539
extern __shared__ int4 sh[];
540
// Shared memory storage for global fetch pipelines.
541
int4* sh_a = sh;
542
int4* sh_b = sh_a + (stages * a_sh_stage);
543
int4* sh_g_idx = sh_b + (stages * b_sh_stage);
471
- int4* sh_s = sh_g_idx + (stages * g_idx_stage);
472
- int* sh_sorted = (int*)(sh_s + shs_size);
544
+ int4* sh_zp = sh_g_idx + (stages * g_idx_stage);
545
+ int4* sh_s = sh_zp + (stages * zp_sh_stage);
546
547
// Precompute which thread should not read memory in which iterations; this is
548
// needed if there are more threads than required for a certain tilesize or
@@ -525,8 +598,10 @@ __device__ inline void MarlinMoESingle(
598
FragA frag_a[2][thread_m_blocks];
599
I4 frag_b_quant[2][b_thread_vecs];
600
FragC frag_c[thread_m_blocks][4][2];
528
- FragS frag_s[2][4]; // No act-order
529
- FragS act_frag_s[2][4][4]; // For act-order
601
+ FragS frag_s[2][4]; // No act-order
602
+ FragS act_frag_s[2][4][4]; // For act-order
603
+ int frag_qzp[2][num_ints_per_thread]; // Zero-points
604
+ FragZP frag_zp; // Zero-points in fp16
605
606
// Zero accumulators.
607
auto zero_accums = [&]() {
@@ -633,6 +708,28 @@ __device__ inline void MarlinMoESingle(
708
}
709
}
710
}
711
+
712
+ if constexpr (has_zp && group_blocks != -1) {
713
+ int4* sh_zp_stage = sh_zp + zp_sh_stage * pipe;
714
+
715
+ if constexpr (group_blocks >= thread_k_blocks) {
716
+ // Only fetch zero-points if this tile starts a new group
717
+ if (pipe % (group_blocks / thread_k_blocks) == 0) {
718
+ if (zp_sh_wr_pred) {
719
+ cp_async4(&sh_zp_stage[zp_sh_wr], &zp_ptr[zp_gl_rd]);
720
+ }
721
+ zp_gl_rd += zp_gl_rd_delta;
722
+ }
723
+ } else {
724
+ for (int i = 0; i < zp_tb_groups; i++) {
725
+ if (zp_sh_wr_pred) {
726
+ cp_async4(&sh_zp_stage[i * zp_sh_stride + zp_sh_wr],
727
+ &zp_ptr[zp_gl_rd]);
728
+ }
729
+ zp_gl_rd += zp_gl_rd_delta;
730
+ }
731
+ }
732
+ }
733
}
734
}
735
// Insert a fence even when we are winding down the pipeline to ensure that
@@ -640,15 +737,9 @@ __device__ inline void MarlinMoESingle(
737
cp_async_fence();
738
};
739
643
- // TODO we are currently hitting illegal memory accesses when fetching
644
- // sorted_ids to shared data: fix this
645
- auto fetch_sorted_ids_to_shared = [&]() {
646
- const int mpt = ceildiv(prob_m, threads);
647
- for (int i = 0; i < mpt; i++) {
648
- if ((i * sorted_gl_stride) + threadIdx.x < prob_m) {
649
- sh_sorted[(i * sorted_sh_stride) + threadIdx.x] =
650
- sorted_ids[(i * sorted_gl_stride) + threadIdx.x];
651
- }
740
+ auto fetch_zp_to_shared = [&]() {
741
+ if (zp_sh_wr_pred) {
742
+ cp_async4(&sh_zp[zp_sh_wr], &zp_ptr[zp_gl_rd]);
743
}
744
};
745
@@ -799,8 +890,83 @@ __device__ inline void MarlinMoESingle(
890
}
891
};
892
893
+ auto fetch_zp_to_registers = [&](int k, int full_pipe) {
894
+ // This code does not handle group_blocks == 0,
895
+ // which signifies act_order.
896
+ // has_zp implies AWQ, which doesn't have act_order,
897
+ static_assert(!has_zp || group_blocks != 0);
898
+
899
+ if constexpr (has_zp) {
900
+ int pipe = full_pipe % stages;
901
+
902
+ if constexpr (group_blocks == -1) {
903
+ for (int i = 0; i < num_ints_per_thread; i++) {
904
+ frag_qzp[k % 2][i] = (reinterpret_cast<int*>(sh_zp))[zp_sh_rd + i];
905
+ }
906
+
907
+ } else if constexpr (group_blocks >= thread_k_blocks) {
908
+ int4* sh_zp_stage =
909
+ sh_zp + zp_sh_stage * ((group_blocks / thread_k_blocks) *
910
+ (pipe / (group_blocks / thread_k_blocks)));
911
+ for (int i = 0; i < num_ints_per_thread; i++) {
912
+ frag_qzp[k % 2][i] =
913
+ (reinterpret_cast<int*>(sh_zp_stage))[zp_sh_rd + i];
914
+ }
915
+ } else {
916
+ int warp_id = threadIdx.x / 32;
917
+ int n_warps = thread_n_blocks / 4;
918
+
919
+ int warp_row = warp_id / n_warps;
920
+
921
+ int cur_k = warp_row * 16;
922
+ cur_k += k_iter_size * (k % b_sh_wr_iters);
923
+
924
+ int k_blocks = cur_k / 16;
925
+ int cur_group_id = 0;
926
+
927
+ // Suppress bogus and persistent divide-by-zero warning
928
+ #pragma nv_diagnostic push
929
+ #pragma nv_diag_suppress divide_by_zero
930
+ cur_group_id = k_blocks / group_blocks;
931
+ #pragma nv_diagnostic pop
932
+
933
+ int4* sh_zp_stage = sh_zp + zp_sh_stage * pipe;
934
+
935
+ sh_zp_stage += cur_group_id * zp_sh_stride;
936
+
937
+ for (int i = 0; i < num_ints_per_thread; i++) {
938
+ frag_qzp[k % 2][i] =
939
+ (reinterpret_cast<int*>(sh_zp_stage))[zp_sh_rd + i];
940
+ }
941
+ }
942
+ }
943
+ };
944
+
945
// Execute the actual tensor core matmul of a sub-tile.
946
auto matmul = [&](int k) {
947
+ if constexpr (has_zp) {
948
+ FragB frag_zp_0;
949
+ FragB frag_zp_1;
950
+ int zp_quant_0, zp_quant_1;
951
+
952
+ if constexpr (w_type.size_bits() == 4) {
953
+ zp_quant_0 = frag_qzp[k % 2][0];
954
+ zp_quant_1 = zp_quant_0 >> 8;
955
+ } else {
956
+ static_assert(w_type.size_bits() == 8);
957
+ zp_quant_0 = frag_qzp[k % 2][0];
958
+ zp_quant_1 = frag_qzp[k % 2][1];
959
+ }
960
+
961
+ frag_zp_0 = dequant<w_type_id>(zp_quant_0);
962
+ frag_zp_1 = dequant<w_type_id>(zp_quant_1);
963
+
964
+ frag_zp[0] = frag_zp_0[0];
965
+ frag_zp[1] = frag_zp_0[1];
966
+ frag_zp[2] = frag_zp_1[0];
967
+ frag_zp[3] = frag_zp_1[1];
968
+ }
969
+
970
// We have the m dimension as the inner loop in order to encourage overlapping
971
// dequantization and matmul operations.
972
#pragma unroll
@@ -818,6 +984,10 @@ __device__ inline void MarlinMoESingle(
984
985
FragB frag_b0 = dequant<w_type_id>(b_quant_0);
986
FragB frag_b1 = dequant<w_type_id>(b_quant_1);
987
+ // Apply zero-point to frag_b0
988
+ if constexpr (has_zp) {
989
+ sub_zp(frag_b0, frag_zp[j], 0);
990
+ }
991
992
// Apply scale to frag_b0
993
if constexpr (has_act_order) {
@@ -829,6 +999,11 @@ __device__ inline void MarlinMoESingle(
999
}
1000
}
1001
1002
+ // Apply zero-point to frag_b1
1003
+ if constexpr (has_zp) {
1004
+ sub_zp(frag_b1, frag_zp[j], 1);
1005
+ }
1006
+
1007
// Apply scale to frag_b1
1008
if constexpr (has_act_order) {
1009
scale4(frag_b1, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
@@ -1062,9 +1237,6 @@ __device__ inline void MarlinMoESingle(
1237
1238
// Start global fetch and register load pipelines.
1239
auto start_pipes = [&]() {
1065
- // TODO re-enable after fixing this function
1066
- // fetch_sorted_ids_to_shared();
1067
- // __syncthreads();
1240
1241
#pragma unroll
1242
for (int i = 0; i < stages - 1; i++) {
@@ -1075,6 +1247,12 @@ __device__ inline void MarlinMoESingle(
1247
}
1248
fetch_scales_to_shared(true, g_idx[slice_k_start], g_idx[last_g_idx]);
1249
}
1250
+
1251
+ if constexpr (has_zp && group_blocks == -1) {
1252
+ if (i == 0) {
1253
+ fetch_zp_to_shared();
1254
+ }
1255
+ }
1256
fetch_to_shared(i, i, i < slice_iters);
1257
}
1258
@@ -1083,6 +1261,7 @@ __device__ inline void MarlinMoESingle(
1261
init_same_group(0);
1262
fetch_to_registers(0, 0);
1263
fetch_scales_to_registers(0, 0);
1264
+ fetch_zp_to_registers(0, 0);
1265
a_gl_rd += a_gl_rd_delta_o * (stages - 1);
1266
slice_k_start_shared_fetch += tb_k * (stages - 1);
1267
};
@@ -1102,6 +1281,7 @@ __device__ inline void MarlinMoESingle(
1281
for (int k = 0; k < b_sh_wr_iters; k++) {
1282
fetch_to_registers(k + 1, pipe % stages);
1283
fetch_scales_to_registers(k + 1, pipe);
1284
+ fetch_zp_to_registers(k + 1, pipe);
1285
if (k == b_sh_wr_iters - 2) {
1286
fetch_to_shared((pipe + stages - 1) % stages, pipe,
1287
slice_iters >= stages);
@@ -1236,7 +1416,9 @@ __device__ inline void MarlinMoESingle(
1416
1417
} else {
1418
s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
1419
+ zp_gl_rd = zp_sh_stride * slice_col + threadIdx.x;
1420
}
1421
+
1422
start_pipes();
1423
}
1424
}
@@ -1250,6 +1432,7 @@ template <const vllm::ScalarTypeId w_type_id, // weight ScalarType id
1432
const int stages, // number of stages for the async global->shared
1433
// fetch pipeline
1434
const bool has_act_order, // whether act_order is enabled
1435
+ const bool has_zp, // whether zero-points are enabled
1436
const int group_blocks = -1 // number of consecutive 16x16 blocks
1437
// with a separate quantization scale
1438
>
@@ -1261,6 +1444,8 @@ __global__ void MarlinMoE(
1444
const float* __restrict__ topk_weights, // float topk weights
1445
const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
1446
// (k/groupsize)xn
1447
+ const int4* __restrict__ zp_ptr, // 4bit packed zero-points of shape
1448
+ // (k/groupsize)x(n/pack_factor)
1449
const int* __restrict__ g_idx, // int32 group indices of shape k
1450
const int* __restrict__ expert_offsets,
1451
int num_groups, // number of scale groups per output channel
@@ -1309,29 +1494,29 @@ __global__ void MarlinMoE(
1494
1495
if (max_block == 1) {
1496
MarlinMoESingle<w_type_id, threads, 1, thread_n_blocks, thread_k_blocks,
1312
- stages, has_act_order, group_blocks>(
1313
- A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1497
+ stages, has_act_order, has_zp, group_blocks>(
1498
+ A, B, C, sorted_ids_expert, topk_weights, scales_ptr, zp_ptr, g_idx,
1499
expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1500
prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1501
current_m_block);
1502
} else if (max_block == 2) {
1503
MarlinMoESingle<w_type_id, threads, 2, thread_n_blocks, thread_k_blocks,
1319
- stages, has_act_order, group_blocks>(
1320
- A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1504
+ stages, has_act_order, has_zp, group_blocks>(
1505
+ A, B, C, sorted_ids_expert, topk_weights, scales_ptr, zp_ptr, g_idx,
1506
expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1507
prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1508
current_m_block);
1509
} else if (max_block == 3) {
1510
MarlinMoESingle<w_type_id, threads, 3, thread_n_blocks, thread_k_blocks,
1326
- stages, has_act_order, group_blocks>(
1327
- A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1511
+ stages, has_act_order, has_zp, group_blocks>(
1512
+ A, B, C, sorted_ids_expert, topk_weights, scales_ptr, zp_ptr, g_idx,
1513
expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1514
prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1515
current_m_block);
1516
} else {
1517
MarlinMoESingle<w_type_id, threads, 4, thread_n_blocks, thread_k_blocks,
1333
- stages, has_act_order, group_blocks>(
1334
- A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1518
+ stages, has_act_order, has_zp, group_blocks>(
1519
+ A, B, C, sorted_ids_expert, topk_weights, scales_ptr, zp_ptr, g_idx,
1520
expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1521
prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1522
current_m_block);
@@ -1347,6 +1532,7 @@ template <const vllm::ScalarTypeId w_type_id, // weight ScalarType id
1532
const int stages, // number of stages for the async global->shared
1533
// fetch pipeline
1534
const bool has_act_order, // whether act_order is enabled
1535
+ const bool has_zp, // whether zero-points are enabled
1536
const int group_blocks = -1 // number of consecutive 16x16 blocks
1537
// with a separate quantization scale
1538
>
@@ -1358,6 +1544,8 @@ __global__ void MarlinMoE(
1544
const float* __restrict__ topk_weights, // float topk weights
1545
const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
1546
// (k/groupsize)xn
1547
+ const int4* __restrict__ zp_ptr, // 4bit packed zero-points of shape
1548
+ // (k/groupsize)x(n/pack_factor)
1549
const int* __restrict__ g_idx, // int32 group indices of shape k
1550
const int* __restrict__ expert_offsets,
1551
int num_groups, // number of scale groups per output channel
@@ -1374,7 +1562,6 @@ __global__ void MarlinMoE(
1562
int current_m_block, // current m block to start kernel computation from
1563
int max_par, // maximum parallelism
1564
int cfg_max_m_blocks // upper bound on m blocks
1377
-
1565
) {
1566
// Marlin is not implemented yet for SM < 8.0
1567
assert(false);
@@ -1389,37 +1576,41 @@ __global__ void MarlinMoE(
1576
const int USER_THREADS =
1577
256; // Note: This is only used with user-provided thread_k/n
1578
const int STAGES = 4; // 4 pipeline stages fit into shared memory
1392
-// const int SHARED_MEM =
1393
-// 96 * 1024; // max shared memory on compute capability 8.6 (< 8.0)
1579
1580
static constexpr int min_thread_n = 64;
1581
static constexpr int min_thread_k = 64;
1582
1583
#define __CALL_IF_MOE(W_TYPE, THREAD_N_BLOCKS, THREAD_K_BLOCKS, HAS_ACT_ORDER, \
1399
- GROUP_BLOCKS, NUM_THREADS) \
1584
+ HAS_ZP, GROUP_BLOCKS, NUM_THREADS) \
1585
else if (q_type == W_TYPE && thread_n_blocks == THREAD_N_BLOCKS && \
1586
thread_k_blocks == THREAD_K_BLOCKS && \
1402
- has_act_order == HAS_ACT_ORDER && group_blocks == GROUP_BLOCKS && \
1403
- num_threads == NUM_THREADS) { \
1587
+ has_act_order == HAS_ACT_ORDER && has_zp == HAS_ZP && \
1588
+ group_blocks == GROUP_BLOCKS && num_threads == NUM_THREADS) { \
1589
cudaFuncSetAttribute( \
1590
MarlinMoE<W_TYPE.id(), NUM_THREADS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1406
- STAGES, HAS_ACT_ORDER, GROUP_BLOCKS>, \
1591
+ STAGES, HAS_ACT_ORDER, HAS_ZP, GROUP_BLOCKS>, \
1592
cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
1593
MarlinMoE<W_TYPE.id(), NUM_THREADS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1409
- STAGES, HAS_ACT_ORDER, GROUP_BLOCKS> \
1594
+ STAGES, HAS_ACT_ORDER, HAS_ZP, GROUP_BLOCKS> \
1595
<<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
1596
A_ptr, B_ptr, C_ptr, sorted_ids_ptr, topk_weights_ptr, s_ptr, \
1412
- g_idx_ptr, expert_offsets_ptr, num_groups, expert_idx, \
1597
+ zp_ptr, g_idx_ptr, expert_offsets_ptr, num_groups, expert_idx, \
1598
num_experts, topk, prob_m, prob_n, prob_k, tot_m, locks, \
1599
replicate_input, apply_weights, m_block, max_par, \
1600
cfg_max_m_blocks); \
1601
}
1602
1418
-#define GPTQ_CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1419
- __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1420
- __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1421
- __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1422
- __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1423
- __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS)
1603
+#define GPTQ_CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1604
+ __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1605
+ __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1606
+ __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1607
+ __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1608
+ __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS)
1609
+
1610
+#define AWQ_CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1611
+ __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1612
+ __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1613
+ __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1614
+ __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS)
1615
1616
} // namespace marlin_moe
csrc/moe/marlin_kernels/marlin_moe_kernel_ku4.cu
new
+31
@@ -0,0 +1,31 @@
1
+#include "marlin_moe_kernel_ku4.h"
2
+
3
+namespace marlin_moe {
4
+
5
+// We return bool so we can create these different kernel calls as a sequence
6
+// of if-elseif's.
7
+bool call_marlin_moe_kernel_ku4(
8
+ vllm::ScalarType const& q_type, int thread_n_blocks, int thread_k_blocks,
9
+ bool has_act_order, int group_blocks, int num_threads, int blocks,
10
+ int max_shared_mem, cudaStream_t stream, const int4* A_ptr,
11
+ const int4* B_ptr, int4* C_ptr, const int* sorted_ids_ptr,
12
+ const float* topk_weights_ptr, const int4* s_ptr, const int4* zp_ptr,
13
+ const int* g_idx_ptr, int* expert_offsets_ptr, int num_groups,
14
+ int expert_idx, int num_experts, int topk, int prob_m, int prob_n,
15
+ int prob_k, int tot_m, int* locks, bool replicate_input, bool apply_weights,
16
+ int m_block, int max_par, int cfg_max_m_blocks) {
17
+ bool has_zp = true;
18
+
19
+ if (false) {
20
+ }
21
+ AWQ_CALL_IF_MOE(vllm::kU4, 16, 4, 256)
22
+ AWQ_CALL_IF_MOE(vllm::kU4, 8, 8, 256)
23
+ AWQ_CALL_IF_MOE(vllm::kU4, 8, 4, 128)
24
+ AWQ_CALL_IF_MOE(vllm::kU4, 4, 8, 128)
25
+ else {
26
+ return false;
27
+ }
28
+ return true;
29
+}
30
+
31
+} // namespace marlin_moe
csrc/moe/marlin_kernels/marlin_moe_kernel_ku4.h
new
+20
@@ -0,0 +1,20 @@
1
+#pragma once
2
+
3
+#include "marlin_moe_kernel.h"
4
+
5
+namespace marlin_moe {
6
+
7
+// We return bool so we can create these different kernel calls as a sequence
8
+// of if-elseif's.
9
+bool call_marlin_moe_kernel_ku4(
10
+ vllm::ScalarType const& q_type, int thread_n_blocks, int thread_k_blocks,
11
+ bool has_act_order, int group_blocks, int num_threads, int blocks,
12
+ int max_shared_mem, cudaStream_t stream, const int4* A_ptr,
13
+ const int4* B_ptr, int4* C_ptr, const int* sorted_ids_ptr,
14
+ const float* topk_weights_ptr, const int4* s_ptr, const int4* zp_ptr,
15
+ const int* g_idx_ptr, int* expert_offsets_ptr, int num_groups,
16
+ int expert_idx, int num_experts, int topk, int prob_m, int prob_n,
17
+ int prob_k, int tot_m, int* locks, bool replicate_input, bool apply_weights,
18
+ int m_block, int max_par, int cfg_max_m_blocks);
19
+
20
+} // namespace marlin_moe
csrc/moe/marlin_kernels/marlin_moe_kernel_ku4b8.cu
+7
-5
@@ -9,11 +9,13 @@ bool call_marlin_moe_kernel_ku4b8(
9
bool has_act_order, int group_blocks, int num_threads, int blocks,
10
int max_shared_mem, cudaStream_t stream, const int4* A_ptr,
11
const int4* B_ptr, int4* C_ptr, const int* sorted_ids_ptr,
12
- const float* topk_weights_ptr, const int4* s_ptr, const int* g_idx_ptr,
13
- int* expert_offsets_ptr, int num_groups, int expert_idx, int num_experts,
14
- int topk, int prob_m, int prob_n, int prob_k, int tot_m, int* locks,
15
- bool replicate_input, bool apply_weights, int m_block, int max_par,
16
- int cfg_max_m_blocks) {
12
+ const float* topk_weights_ptr, const int4* s_ptr, const int4* zp_ptr,
13
+ const int* g_idx_ptr, int* expert_offsets_ptr, int num_groups,
14
+ int expert_idx, int num_experts, int topk, int prob_m, int prob_n,
15
+ int prob_k, int tot_m, int* locks, bool replicate_input, bool apply_weights,
16
+ int m_block, int max_par, int cfg_max_m_blocks) {
17
+ bool has_zp = false;
18
+
19
if (false) {
20
}
21
GPTQ_CALL_IF_MOE(vllm::kU4B8, 16, 4, 256)
csrc/moe/marlin_kernels/marlin_moe_kernel_ku4b8.h
+5
-5
@@ -11,10 +11,10 @@ bool call_marlin_moe_kernel_ku4b8(
11
bool has_act_order, int group_blocks, int num_threads, int blocks,
12
int max_shared_mem, cudaStream_t stream, const int4* A_ptr,
13
const int4* B_ptr, int4* C_ptr, const int* sorted_ids_ptr,
14
- const float* topk_weights_ptr, const int4* s_ptr, const int* g_idx_ptr,
15
- int* expert_offsets_ptr, int num_groups, int expert_idx, int num_experts,
16
- int topk, int prob_m, int prob_n, int prob_k, int tot_m, int* locks,
17
- bool replicate_input, bool apply_weights, int m_block, int max_par,
18
- int cfg_max_m_blocks);
14
+ const float* topk_weights_ptr, const int4* s_ptr, const int4* zp_ptr,
15
+ const int* g_idx_ptr, int* expert_offsets_ptr, int num_groups,
16
+ int expert_idx, int num_experts, int topk, int prob_m, int prob_n,
17
+ int prob_k, int tot_m, int* locks, bool replicate_input, bool apply_weights,
18
+ int m_block, int max_par, int cfg_max_m_blocks);
19
20
} // namespace marlin_moe
csrc/moe/marlin_kernels/marlin_moe_kernel_ku8b128.cu
+7
-5
@@ -9,11 +9,13 @@ bool call_marlin_moe_kernel_ku8b128(
9
bool has_act_order, int group_blocks, int num_threads, int blocks,
10
int max_shared_mem, cudaStream_t stream, const int4* A_ptr,
11
const int4* B_ptr, int4* C_ptr, const int* sorted_ids_ptr,
12
- const float* topk_weights_ptr, const int4* s_ptr, const int* g_idx_ptr,
13
- int* expert_offsets_ptr, int num_groups, int expert_idx, int num_experts,
14
- int topk, int prob_m, int prob_n, int prob_k, int tot_m, int* locks,
15
- bool replicate_input, bool apply_weights, int m_block, int max_par,
16
- int cfg_max_m_blocks) {
12
+ const float* topk_weights_ptr, const int4* s_ptr, const int4* zp_ptr,
13
+ const int* g_idx_ptr, int* expert_offsets_ptr, int num_groups,
14
+ int expert_idx, int num_experts, int topk, int prob_m, int prob_n,
15
+ int prob_k, int tot_m, int* locks, bool replicate_input, bool apply_weights,
16
+ int m_block, int max_par, int cfg_max_m_blocks) {
17
+ bool has_zp = false;
18
+
19
if (false) {
20
}
21
GPTQ_CALL_IF_MOE(vllm::kU8B128, 16, 4, 256)
csrc/moe/marlin_kernels/marlin_moe_kernel_ku8b128.h
+5
-5
@@ -9,10 +9,10 @@ bool call_marlin_moe_kernel_ku8b128(
9
bool has_act_order, int group_blocks, int num_threads, int blocks,
10
int max_shared_mem, cudaStream_t stream, const int4* A_ptr,
11
const int4* B_ptr, int4* C_ptr, const int* sorted_ids_ptr,
12
- const float* topk_weights_ptr, const int4* s_ptr, const int* g_idx_ptr,
13
- int* expert_offsets_ptr, int num_groups, int expert_idx, int num_experts,
14
- int topk, int prob_m, int prob_n, int prob_k, int tot_m, int* locks,
15
- bool replicate_input, bool apply_weights, int m_block, int max_par,
16
- int cfg_max_m_blocks);
12
+ const float* topk_weights_ptr, const int4* s_ptr, const int4* zp_ptr,
13
+ const int* g_idx_ptr, int* expert_offsets_ptr, int num_groups,
14
+ int expert_idx, int num_experts, int topk, int prob_m, int prob_n,
15
+ int prob_k, int tot_m, int* locks, bool replicate_input, bool apply_weights,
16
+ int m_block, int max_par, int cfg_max_m_blocks);
17
18
}
csrc/moe/marlin_moe_ops.cu
+56
-28
@@ -30,6 +30,7 @@
30
#include "core/registration.h"
31
#include "marlin_kernels/marlin_moe_kernel_ku4b8.h"
32
#include "marlin_kernels/marlin_moe_kernel_ku8b128.h"
33
+#include "marlin_kernels/marlin_moe_kernel_ku4.h"
34
35
template <typename T>
36
inline std::string str(T x) {
@@ -157,6 +158,7 @@ thread_config_t small_batch_thread_configs[] = {
158
{128, 64, 128}, // Reduce N 2X, same K
159
{64, 256, 256}, // Reduce K 2X, increase N 2X
160
{64, 128, 128}, // Reduce K 2X, same N
161
+ {64, 64, 128}, // Reduce both 2X
162
};
163
164
thread_config_t large_batch_thread_configs[] = {
@@ -167,6 +169,7 @@ thread_config_t large_batch_thread_configs[] = {
169
{128, 128, 256}, // Reduce N 2X, increase K 2X
170
{64, 128, 128}, // Reduce N 2X, same K
171
{128, 64, 128}, // Reduce N 4X, increase K 2X
172
+ {64, 64, 128}, // Reduce N 4X, same K
173
};
174
175
int get_scales_cache_size(thread_config_t const& th_config, int prob_m,
@@ -312,27 +315,28 @@ exec_config_t determine_thread_config(int prob_m, int prob_n, int prob_k,
315
return exec_config_t{0, {-1, -1, -1}};
316
}
317
315
-#define CALL_MOE_KERNEL_FUNCTION(KERNEL_FUNCTION) \
316
- else if (KERNEL_FUNCTION(q_type, thread_n_blocks, thread_k_blocks, \
317
- has_act_order, group_blocks, num_threads, blocks, \
318
- max_shared_mem, stream, A_ptr, B_ptr, C_ptr, \
319
- sorted_ids_ptr, topk_weights_ptr, s_ptr, g_idx_ptr, \
320
- expert_offsets_ptr, num_groups, expert_idx, \
321
- num_experts, topk, prob_m, prob_n, prob_k, tot_m, \
322
- locks, replicate_input, apply_weights, m_block, \
323
- max_par, exec_cfg.max_m_blocks)) { \
318
+#define CALL_MOE_KERNEL_FUNCTION(KERNEL_FUNCTION) \
319
+ else if (KERNEL_FUNCTION( \
320
+ q_type, thread_n_blocks, thread_k_blocks, has_act_order, \
321
+ group_blocks, num_threads, blocks, max_shared_mem, stream, \
322
+ A_ptr, B_ptr, C_ptr, sorted_ids_ptr, topk_weights_ptr, s_ptr, \
323
+ zp_ptr, g_idx_ptr, expert_offsets_ptr, num_groups, expert_idx, \
324
+ num_experts, topk, prob_m, prob_n, prob_k, tot_m, locks, \
325
+ replicate_input, apply_weights, m_block, max_par, \
326
+ exec_cfg.max_m_blocks)) { \
327
}
328
329
void marlin_mm_moe(const void* A, const void* B, void* C,
330
const void* sorted_ids, const void* topk_weights,
328
- const void* topk_ids, const void* s, const void* g_idx,
329
- const void* perm, void* a_tmp, void* expert_offsets,
330
- int prob_m, int prob_n, int prob_k, void* workspace,
331
- vllm::ScalarType const& q_type, bool has_act_order,
332
- bool is_k_full, int num_groups, int group_size,
333
- int num_experts, int topk, int moe_block_size, int dev,
334
- cudaStream_t stream, int thread_k, int thread_n, int sms,
335
- int max_par, bool replicate_input, bool apply_weights) {
331
+ const void* topk_ids, const void* s, void* zp,
332
+ const void* g_idx, const void* perm, void* a_tmp,
333
+ void* expert_offsets, int prob_m, int prob_n, int prob_k,
334
+ void* workspace, vllm::ScalarType const& q_type,
335
+ bool has_act_order, bool is_k_full, bool has_zp,
336
+ int num_groups, int group_size, int num_experts, int topk,
337
+ int moe_block_size, int dev, cudaStream_t stream,
338
+ int thread_k, int thread_n, int sms, int max_par,
339
+ bool replicate_input, bool apply_weights) {
340
TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m,
341
", ", prob_n, ", ", prob_k, "]");
342
@@ -436,6 +440,8 @@ void marlin_mm_moe(const void* A, const void* B, void* C,
440
const float* topk_weights_ptr = (const float*)topk_weights;
441
const int* sorted_ids_ptr = (const int*)sorted_ids;
442
const int4* s_ptr = (const int4*)s + num_groups * prob_n / 8 * expert_idx;
443
+ const int4* zp_ptr =
444
+ (const int4*)zp + num_groups * prob_n / (pack_factor * 4) * expert_idx;
445
const int* g_idx_ptr = (const int*)g_idx + prob_k * expert_idx;
446
const int* perm_ptr = (const int*)perm + prob_k * expert_idx;
447
int* locks = (int*)workspace;
@@ -456,6 +462,7 @@ void marlin_mm_moe(const void* A, const void* B, void* C,
462
}
463
CALL_MOE_KERNEL_FUNCTION(call_marlin_moe_kernel_ku4b8)
464
CALL_MOE_KERNEL_FUNCTION(call_marlin_moe_kernel_ku8b128)
465
+ CALL_MOE_KERNEL_FUNCTION(call_marlin_moe_kernel_ku4)
466
else {
467
TORCH_CHECK(false, "Unsupported shapes: MNK = [" + str(prob_m) + ", " +
468
str(prob_n) + ", " + str(prob_k) + "]" +
@@ -475,13 +482,21 @@ torch::Tensor marlin_gemm_moe(
482
const torch::Tensor& a, const torch::Tensor& b_q_weights,
483
const torch::Tensor& sorted_ids, const torch::Tensor& topk_weights,
484
const torch::Tensor& topk_ids, const torch::Tensor& b_scales,
478
- const torch::Tensor& g_idx, const torch::Tensor& perm,
479
- torch::Tensor& workspace, vllm::ScalarTypeTorchPtr const& b_q_type,
480
- int64_t size_m, int64_t size_n, int64_t size_k, bool is_k_full,
481
- int64_t num_experts, int64_t topk, int64_t moe_block_size,
482
- bool replicate_input, bool apply_weights) {
483
- TORCH_CHECK(*b_q_type == vllm::kU4B8 || *b_q_type == vllm::kU8B128,
484
- "b_q_type must be uint4b8 or uint8b128. Got = ", b_q_type->str());
485
+ torch::Tensor& b_zeros, const torch::Tensor& g_idx,
486
+ const torch::Tensor& perm, torch::Tensor& workspace,
487
+ vllm::ScalarTypeTorchPtr const& b_q_type, int64_t size_m, int64_t size_n,
488
+ int64_t size_k, bool is_k_full, int64_t num_experts, int64_t topk,
489
+ int64_t moe_block_size, bool replicate_input, bool apply_weights) {
490
+ bool has_zp = b_zeros.size(1) != 0;
491
+ if (has_zp) {
492
+ TORCH_CHECK(
493
+ *b_q_type == vllm::kU4,
494
+ "b_q_type must be u4 when has_zp = True. Got = ", b_q_type->str());
495
+ } else {
496
+ TORCH_CHECK(
497
+ *b_q_type == vllm::kU4B8 || *b_q_type == vllm::kU8B128,
498
+ "b_q_type must be uint4b8 or uint8b128. Got = ", b_q_type->str());
499
+ }
500
501
int pack_factor = 32 / b_q_type->size_bits();
502
@@ -543,14 +558,27 @@ torch::Tensor marlin_gemm_moe(
558
}
559
}
560
561
+ // Verify b_zeros
562
+ if (has_zp) {
563
+ int rank = b_zeros.sizes().size();
564
+ TORCH_CHECK(rank == 3, "b_zeros rank = ", rank, " is not 3");
565
+ TORCH_CHECK(b_zeros.size(1) == num_groups,
566
+ "b_zeros dim 1 = ", b_zeros.size(1),
567
+ " is not num_groups = ", num_groups);
568
+ TORCH_CHECK(b_zeros.size(2) == size_n / pack_factor,
569
+ "b_zeros dim 2 = ", b_zeros.size(2),
570
+ " is not size_n / pack_factor = ", size_n / pack_factor);
571
+ }
572
+
573
marlin_moe::marlin_mm_moe(
574
a.data_ptr(), b_q_weights.data_ptr(), c.data_ptr(), sorted_ids.data_ptr(),
575
topk_weights.data_ptr(), topk_ids.data_ptr(), b_scales.data_ptr(),
549
- g_idx.data_ptr(), perm.data_ptr(), a_tmp.data_ptr(),
576
+ b_zeros.data_ptr(), g_idx.data_ptr(), perm.data_ptr(), a_tmp.data_ptr(),
577
expert_offsets.data_ptr(), size_m, size_n, size_k, workspace.data_ptr(),
551
- *b_q_type, has_act_order, is_k_full, num_groups, group_size, num_experts,
552
- topk, moe_block_size, dev, at::cuda::getCurrentCUDAStream(dev), thread_k,
553
- thread_n, sms, max_par, replicate_input, apply_weights);
578
+ *b_q_type, has_act_order, is_k_full, has_zp, num_groups, group_size,
579
+ num_experts, topk, moe_block_size, dev,
580
+ at::cuda::getCurrentCUDAStream(dev), thread_k, thread_n, sms, max_par,
581
+ replicate_input, apply_weights);
582
return c;
583
}
584
csrc/moe/torch_bindings.cpp
+1
-1
@@ -12,7 +12,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
12
m.def(
13
"marlin_gemm_moe(Tensor! a, Tensor! b_q_weights, Tensor! sorted_ids, "
14
"Tensor! topk_weights, Tensor! topk_ids, Tensor! b_scales, Tensor! "
15
- "g_idx, Tensor! perm, Tensor! workspace, "
15
+ "b_zeros, Tensor! g_idx, Tensor! perm, Tensor! workspace, "
16
"__torch__.torch.classes._core_C.ScalarType b_q_type, int size_m, "
17
"int size_n, int size_k, bool is_k_full, int num_experts, int topk, "
18
"int moe_block_size, bool replicate_input, bool apply_weights)"
csrc/quantization/gptq_marlin/gptq_marlin.cu
+1
-1
@@ -2260,7 +2260,7 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2260
"b_zeros dim 0 = ", b_zeros.size(0),
2261
" is not num_groups = ", num_groups);
2262
TORCH_CHECK(b_zeros.size(1) == size_n / pack_factor,
2263
- "b_zeros dim 1 = ", b_scales.size(1),
2263
+ "b_zeros dim 1 = ", b_zeros.size(1),
2264
" is not size_n / pack_factor = ", size_n / pack_factor);
2265
}
2266
tests/kernels/test_awq_marlin.py
new
+160
@@ -0,0 +1,160 @@
1
+"""Test AWQ with fused MoE Marlin kernels.
2
+
3
+Run `pytest tests/kernels/test_awq_marlin.py`.
4
+"""
5
+import pytest
6
+import torch
7
+
8
+from tests.kernels.utils import (compute_max_diff, stack_and_dev, torch_moe,
9
+ torch_moe_single)
10
+from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
11
+ fused_marlin_moe, single_marlin_moe)
12
+from vllm.model_executor.layers.fused_moe.fused_moe import fused_topk
13
+from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
14
+ awq_marlin_quantize)
15
+from vllm.scalar_type import scalar_types
16
+
17
+
18
+@pytest.mark.parametrize("m", [64, 512, 222, 33, 1])
19
+@pytest.mark.parametrize("n", [128, 2048, 256, 1024])
20
+@pytest.mark.parametrize("k", [128, 1024, 512])
21
+@pytest.mark.parametrize("e", [8, 64])
22
+@pytest.mark.parametrize("topk", [2, 6])
23
+@pytest.mark.parametrize("group_size", [-1, 32, 64, 128])
24
+def test_fused_marlin_moe_awq(
25
+ m: int,
26
+ n: int,
27
+ k: int,
28
+ e: int,
29
+ topk: int,
30
+ group_size: int,
31
+):
32
+ torch.manual_seed(7)
33
+
34
+ num_bits = 4
35
+ quant_type = scalar_types.uint4
36
+ dtype = torch.float16
37
+ a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
38
+ w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10
39
+ w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10
40
+
41
+ w_ref1_l = []
42
+ qweights1_l = []
43
+ scales1_l = []
44
+ zp1_l = []
45
+
46
+ for i in range(w1.shape[0]):
47
+ w_ref1, qweight1, scales1, zp1 = awq_marlin_quantize(
48
+ w1[i].transpose(1, 0), quant_type, group_size)
49
+ w_ref1_l.append(w_ref1)
50
+ qweights1_l.append(qweight1)
51
+ scales1_l.append(scales1)
52
+ zp1_l.append(zp1)
53
+
54
+ w_ref1 = stack_and_dev(w_ref1_l)
55
+ qweight1 = stack_and_dev(qweights1_l).contiguous()
56
+ scales1 = stack_and_dev(scales1_l)
57
+ zp1 = stack_and_dev(zp1_l)
58
+
59
+ w_ref2_l = []
60
+ qweights2_l = []
61
+ scales2_l = []
62
+ zp2_l = []
63
+
64
+ for i in range(w2.shape[0]):
65
+ w_ref2, qweight2, scales2, zp2 = awq_marlin_quantize(
66
+ w2[i].transpose(1, 0), quant_type, group_size)
67
+ w_ref2_l.append(w_ref2)
68
+ qweights2_l.append(qweight2)
69
+ scales2_l.append(scales2)
70
+ zp2_l.append(zp2)
71
+
72
+ w_ref2 = stack_and_dev(w_ref2_l)
73
+ qweight2 = stack_and_dev(qweights2_l).contiguous()
74
+ scales2 = stack_and_dev(scales2_l)
75
+ zp2 = stack_and_dev(zp2_l)
76
+
77
+ score = torch.randn((m, e), device="cuda", dtype=dtype)
78
+
79
+ topk_weights, topk_ids = fused_topk(a, score, topk, False)
80
+ marlin_output = fused_marlin_moe(
81
+ a,
82
+ qweight1,
83
+ qweight2,
84
+ scales1,
85
+ scales2,
86
+ score,
87
+ topk_weights,
88
+ topk_ids,
89
+ w1_zeros=zp1,
90
+ w2_zeros=zp2,
91
+ num_bits=num_bits,
92
+ )
93
+
94
+ torch_output = torch_moe(
95
+ a,
96
+ w_ref1.transpose(1, 2),
97
+ w_ref2.transpose(1, 2),
98
+ score,
99
+ topk,
100
+ )
101
+
102
+ assert compute_max_diff(marlin_output, torch_output) < 4e-2
103
+
104
+
105
+@pytest.mark.skip("This test is here for the sake of debugging, "
106
+ "don't run it in automated tests.")
107
+@pytest.mark.parametrize("m", [64, 512, 222, 33, 1])
108
+@pytest.mark.parametrize("n", [128, 2048, 256, 1024])
109
+@pytest.mark.parametrize("k", [128, 1024, 512])
110
+@pytest.mark.parametrize("e", [8, 64])
111
+@pytest.mark.parametrize("topk", [2, 6])
112
+@pytest.mark.parametrize("group_size", [-1, 32, 64, 128])
113
+def test_single_marlin_moe_multiply_awq(
114
+ m: int,
115
+ n: int,
116
+ k: int,
117
+ e: int,
118
+ topk: int,
119
+ group_size: int,
120
+):
121
+ torch.manual_seed(7)
122
+
123
+ num_bits = 4
124
+ quant_type = scalar_types.uint4
125
+ dtype = torch.float16
126
+ a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
127
+ w = torch.randn((e, n, k), device="cuda", dtype=dtype) / 10
128
+
129
+ w_ref_l = []
130
+ qweights_l = []
131
+ scales_l = []
132
+ zp_l = []
133
+
134
+ for i in range(w.shape[0]):
135
+ w_ref, qweight, scales, zp = awq_marlin_quantize(
136
+ w[i].transpose(1, 0), quant_type, group_size)
137
+ w_ref_l.append(w_ref)
138
+ qweights_l.append(qweight)
139
+ scales_l.append(scales)
140
+ zp_l.append(zp)
141
+
142
+ w_ref = stack_and_dev(w_ref_l)
143
+ qweight = stack_and_dev(qweights_l).contiguous()
144
+ scales = stack_and_dev(scales_l).contiguous()
145
+ zp = stack_and_dev(zp_l).contiguous()
146
+
147
+ score = torch.randn((m, e), device="cuda", dtype=dtype)
148
+
149
+ marlin_output = single_marlin_moe(a,
150
+ qweight,
151
+ scales,
152
+ score,
153
+ topk,
154
+ renormalize=False,
155
+ w_zeros=zp,
156
+ num_bits=num_bits)
157
+
158
+ torch_output = torch_moe_single(a, w_ref.transpose(1, 2), score, topk)
159
+
160
+ assert compute_max_diff(marlin_output, torch_output) < 1e-2
tests/kernels/test_moe.py
+18
-61
@@ -2,16 +2,14 @@
2
3
Run `pytest tests/kernels/test_moe.py`.
4
"""
5
-from typing import List
6
-
5
import pytest
6
import torch
7
from transformers import MixtralConfig
8
from transformers.models.mixtral.modeling_mixtral import MixtralSparseMoeBlock
9
12
-from tests.kernels.utils import opcheck
10
+from tests.kernels.utils import (compute_max_diff, opcheck, stack_and_dev,
11
+ torch_moe, torch_moe_single)
12
from vllm import _custom_ops as ops
14
-from vllm.model_executor.layers.activation import SiluAndMul
13
from vllm.model_executor.layers.fused_moe import fused_moe
14
from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
15
fused_marlin_moe, single_marlin_moe)
@@ -24,37 +22,6 @@ from vllm.scalar_type import scalar_types
22
from vllm.utils import seed_everything
23
24
27
-def torch_moe(a, w1, w2, score, topk):
28
- B, D = a.shape
29
- a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
30
- out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device)
31
- score = torch.softmax(score, dim=-1, dtype=torch.float32)
32
- topk_weight, topk_ids = torch.topk(score, topk)
33
- topk_weight = topk_weight.view(-1)
34
- topk_ids = topk_ids.view(-1)
35
- for i in range(w1.shape[0]):
36
- mask = topk_ids == i
37
- if mask.sum():
38
- out[mask] = SiluAndMul()(
39
- a[mask] @ w1[i].transpose(0, 1)) @ w2[i].transpose(0, 1)
40
- return (out.view(B, -1, w2.shape[1]) *
41
- topk_weight.view(B, -1, 1).to(out.dtype)).sum(dim=1)
42
-
43
-
44
-def torch_moe_single(a, w, score, topk):
45
- B, D = a.shape
46
- a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
47
- out = torch.zeros(B * topk, w.shape[1], dtype=a.dtype, device=a.device)
48
- score = torch.softmax(score, dim=-1, dtype=torch.float32)
49
- _, topk_ids = torch.topk(score, topk)
50
- topk_ids = topk_ids.view(-1)
51
- for i in range(w.shape[0]):
52
- mask = topk_ids == i
53
- if mask.sum():
54
- out[mask] = a[mask] @ w[i].transpose(0, 1)
55
- return (out.view(B, -1, w.shape[1])).sum(dim=1)
56
-
57
-
25
@pytest.mark.parametrize("m", [1024 * 128, 512, 222, 33, 1])
26
@pytest.mark.parametrize("n", [2048, 256, 1024])
27
@pytest.mark.parametrize("k", [128, 511, 1024])
@@ -127,20 +94,10 @@ def test_mixtral_moe(dtype: torch.dtype):
94
atol=mixtral_moe_tol[dtype])
95
96
130
-def stack_and_dev(tensors: List[torch.Tensor]):
131
- dev = tensors[0].device
132
- return torch.stack(tensors, dim=0).to(dev)
133
-
134
-
135
-def compute_max_diff(output, output_ref):
136
- return torch.mean(torch.abs(output - output_ref)) / torch.mean(
137
- torch.abs(output_ref))
138
-
139
-
97
@pytest.mark.parametrize("m", [64, 512, 222, 33, 1])
98
@pytest.mark.parametrize("n", [128, 2048, 256, 1024])
99
@pytest.mark.parametrize("k", [128, 1024, 512])
143
-@pytest.mark.parametrize("e", [4, 8, 64])
100
+@pytest.mark.parametrize("e", [8, 64])
101
@pytest.mark.parametrize("topk", [2, 6])
102
@pytest.mark.parametrize("group_size", [-1, 32, 64, 128])
103
@pytest.mark.parametrize("act_order", [True, False])
@@ -159,9 +116,6 @@ def test_fused_marlin_moe(
116
):
117
seed_everything(7)
118
162
- if topk > e:
163
- return
164
-
119
# Filter act_order
120
if act_order:
121
if group_size == -1:
@@ -241,15 +195,15 @@ def test_fused_marlin_moe(
195
a,
196
qweight1,
197
qweight2,
198
+ scales1,
199
+ scales2,
200
score,
245
- g_idx1,
246
- g_idx2,
247
- sort_indices1,
248
- sort_indices2,
201
topk_weights,
202
topk_ids,
251
- w1_scale=scales1,
252
- w2_scale=scales2,
203
+ g_idx1=g_idx1,
204
+ g_idx2=g_idx2,
205
+ sort_indices1=sort_indices1,
206
+ sort_indices2=sort_indices2,
207
num_bits=num_bits,
208
is_k_full=is_k_full,
209
)
@@ -280,9 +234,13 @@ def test_fused_marlin_moe(
234
device="cuda",
235
requires_grad=False)
236
237
+ zp = torch.empty((0, 0),
238
+ dtype=dtype,
239
+ device="cuda",
240
+ requires_grad=False)
241
opcheck(torch.ops._moe_C.marlin_gemm_moe,
242
(a, qweight1, sorted_token_ids, topk_weights, topk_ids,
285
- scales1, g_idx1, sort_indices1, workspace, quant_type, m,
243
+ scales1, zp, g_idx1, sort_indices1, workspace, quant_type, m,
244
2 * n, k, True, e, topk, block_size_m, True, False))
245
246
@@ -291,7 +249,7 @@ def test_fused_marlin_moe(
249
@pytest.mark.parametrize("m", [64, 512, 222, 33, 1])
250
@pytest.mark.parametrize("n", [128, 2048, 256, 1024])
251
@pytest.mark.parametrize("k", [128, 1024, 512])
294
-@pytest.mark.parametrize("e", [4, 8, 64])
252
+@pytest.mark.parametrize("e", [8, 64])
253
@pytest.mark.parametrize("topk", [2, 6])
254
@pytest.mark.parametrize("group_size", [-1, 32, 64, 128])
255
@pytest.mark.parametrize("act_order", [True, False])
@@ -308,8 +266,6 @@ def test_single_marlin_moe_multiply(
266
num_bits: int,
267
is_k_full: bool,
268
):
311
- if topk > e:
312
- return
269
270
# Filter act_order
271
if act_order:
@@ -355,13 +311,14 @@ def test_single_marlin_moe_multiply(
311
qweight,
312
scales,
313
score,
358
- g_idx,
359
- sort_indices,
314
topk,
315
renormalize=False,
316
+ g_idx=g_idx,
317
+ sort_indices=sort_indices,
318
num_bits=num_bits,
319
is_k_full=is_k_full,
320
)
321
+
322
torch_output = torch_moe_single(a, w_ref.transpose(1, 2), score, topk)
323
324
assert compute_max_diff(marlin_output, torch_output) < 1e-2
tests/kernels/utils.py
+45
@@ -12,6 +12,7 @@ import torch
12
from torch._prims_common import TensorLikeType
13
14
from vllm.attention import AttentionBackend, AttentionMetadata, AttentionType
15
+from vllm.model_executor.layers.activation import SiluAndMul
16
from vllm.utils import (STR_BACKEND_ENV_VAR, STR_XFORMERS_ATTN_VAL,
17
make_tensor_with_pad)
18
@@ -974,6 +975,50 @@ def fp8_allclose(
975
equal_nan=equal_nan)).item())
976
977
978
+# Marlin MoE test utils
979
+
980
+
981
+def stack_and_dev(tensors: List[torch.Tensor]):
982
+ dev = tensors[0].device
983
+ return torch.stack(tensors, dim=0).to(dev)
984
+
985
+
986
+def compute_max_diff(output, output_ref):
987
+ return torch.mean(torch.abs(output - output_ref)) / torch.mean(
988
+ torch.abs(output_ref))
989
+
990
+
991
+def torch_moe(a, w1, w2, score, topk):
992
+ B, D = a.shape
993
+ a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
994
+ out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device)
995
+ score = torch.softmax(score, dim=-1, dtype=torch.float32)
996
+ topk_weight, topk_ids = torch.topk(score, topk)
997
+ topk_weight = topk_weight.view(-1)
998
+ topk_ids = topk_ids.view(-1)
999
+ for i in range(w1.shape[0]):
1000
+ mask = topk_ids == i
1001
+ if mask.sum():
1002
+ out[mask] = SiluAndMul()(
1003
+ a[mask] @ w1[i].transpose(0, 1)) @ w2[i].transpose(0, 1)
1004
+ return (out.view(B, -1, w2.shape[1]) *
1005
+ topk_weight.view(B, -1, 1).to(out.dtype)).sum(dim=1)
1006
+
1007
+
1008
+def torch_moe_single(a, w, score, topk):
1009
+ B, D = a.shape
1010
+ a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
1011
+ out = torch.zeros(B * topk, w.shape[1], dtype=a.dtype, device=a.device)
1012
+ score = torch.softmax(score, dim=-1, dtype=torch.float32)
1013
+ _, topk_ids = torch.topk(score, topk)
1014
+ topk_ids = topk_ids.view(-1)
1015
+ for i in range(w.shape[0]):
1016
+ mask = topk_ids == i
1017
+ if mask.sum():
1018
+ out[mask] = a[mask] @ w[i].transpose(0, 1)
1019
+ return (out.view(B, -1, w.shape[1])).sum(dim=1)
1020
+
1021
+
1022
# A special version of op check that has a restricted default set of test_utils
1023
# and a patched version of allclose that supports fp8 types.
1024
def opcheck(op: Union[torch._ops.OpOverload, torch._ops.OpOverloadPacket,
tests/weight_loading/models-large.txt
+1
@@ -3,3 +3,4 @@ compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-channel-quantize
3
compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W8A16-quantized, main
4
compressed-tensors, mgoin/DeepSeek-Coder-V2-Lite-Instruct-FP8, main
5
gptq_marlin, TheBloke/Mixtral-8x7B-v0.1-GPTQ, main
6
+awq_marlin, casperhansen/deepseek-coder-v2-instruct-awq, main
\ No newline at end of file
tests/weight_loading/run_model_weight_loading_test.sh
+14
-1
@@ -1,7 +1,20 @@
1
#!/bin/bash
2
SUCCESS=0
3
4
-IFS=$'\n' read -d '' -r -a MODEL_CONFIGS < "weight_loading/models.txt"
4
+while getopts "c:" OPT; do
5
+ case ${OPT} in
6
+ c )
7
+ CONFIG="$OPTARG"
8
+ ;;
9
+ \? )
10
+ usage
11
+ exit 1
12
+ ;;
13
+ esac
14
+done
15
+
16
+
17
+IFS=$'\n' read -d '' -r -a MODEL_CONFIGS < $CONFIG
18
19
for MODEL_CONFIG in "${MODEL_CONFIGS[@]}"
20
do
vllm/_custom_ops.py
+20
-5
@@ -568,6 +568,20 @@ def gptq_marlin_moe_repack(b_q_weight: torch.Tensor, perm: torch.Tensor,
568
return output
569
570
571
+def awq_marlin_moe_repack(b_q_weight: torch.Tensor, perm: torch.Tensor,
572
+ size_k: int, size_n: int,
573
+ num_bits: int) -> torch.Tensor:
574
+ num_experts = b_q_weight.shape[0]
575
+ assert size_k % 16 == 0
576
+ output = torch.empty((num_experts, size_k // 16, size_n * (num_bits // 2)),
577
+ device=b_q_weight.device,
578
+ dtype=b_q_weight.dtype)
579
+ for e in range(num_experts):
580
+ output[e] = torch.ops._C.awq_marlin_repack(b_q_weight[e], size_k,
581
+ size_n, num_bits)
582
+ return output
583
+
584
+
585
def gptq_marlin_gemm(a: torch.Tensor,
586
b_q_weight: torch.Tensor,
587
b_scales: torch.Tensor,
@@ -828,11 +842,12 @@ if supports_moe_ops and hasattr(torch.ops._moe_C, "marlin_gemm_moe"):
842
sorted_ids: torch.Tensor,
843
topk_weights: torch.Tensor,
844
topk_ids: torch.Tensor, b_scales: torch.Tensor,
831
- g_idx: torch.Tensor, perm: torch.Tensor,
832
- workspace: torch.Tensor, b_q_type: ScalarType,
833
- size_m: int, size_n: int, size_k: int,
834
- is_k_full: bool, num_experts: int, topk: int,
835
- moe_block_size: int, replicate_input: bool,
845
+ b_zero_points: torch.Tensor, g_idx: torch.Tensor,
846
+ perm: torch.Tensor, workspace: torch.Tensor,
847
+ b_q_type: ScalarType, size_m: int, size_n: int,
848
+ size_k: int, is_k_full: bool, num_experts: int,
849
+ topk: int, moe_block_size: int,
850
+ replicate_input: bool,
851
apply_weights: bool) -> torch.Tensor:
852
return torch.empty((size_m, topk, size_n),
853
dtype=a.dtype,
vllm/model_executor/layers/fused_moe/fused_marlin_moe.py
+107
-31
@@ -10,15 +10,24 @@ from vllm.model_executor.layers.fused_moe.fused_moe import (
10
from vllm.scalar_type import scalar_types
11
12
13
+def get_scalar_type(num_bits: int, has_zp: bool):
14
+ if has_zp:
15
+ assert num_bits == 4
16
+ return scalar_types.uint4
17
+ else:
18
+ return scalar_types.uint4b8 if num_bits == 4 else scalar_types.uint8b128
19
+
20
+
21
def single_marlin_moe(
22
hidden_states: torch.Tensor,
23
w: torch.Tensor,
24
scales: torch.Tensor,
25
gating_output: torch.Tensor,
18
- g_idx: torch.Tensor,
19
- perm: torch.Tensor,
26
topk: int,
27
renormalize: bool,
28
+ g_idx: Optional[torch.Tensor] = None,
29
+ sort_indices: Optional[torch.Tensor] = None,
30
+ w_zeros: Optional[torch.Tensor] = None,
31
override_config: Optional[Dict[str, Any]] = None,
32
num_bits: int = 8,
33
is_k_full: bool = True,
@@ -34,10 +43,12 @@ def single_marlin_moe(
43
- scales (torch.Tensor): The quantization scales.
44
- gating_output (torch.Tensor): The output of the gating operation
45
(before softmax).
37
- - g_idx (torch.Tensor): The act_order indices.
38
- - perm (torch.Tensor): The act_order input permutation.
46
+ - g_idx (Optional[torch.Tensor]): Optional act_order indices.
47
+ - sort_indices (Optional[torch.Tensor]): Optional act_order input
48
+ permutation.
49
- topk (int): The number of top-k experts to select.
50
- renormalize (bool): If True, renormalize the top-k weights to sum to 1.
51
+ - w_zeros (Optional[torch.Tensor]): Optional zero points to be used for w.
52
- override_config (Optional[Dict[str, Any]]): Optional override
53
for the kernel configuration.
54
- num_bits (bool): The number of bits in expert weights quantization.
@@ -79,16 +90,34 @@ def single_marlin_moe(
90
max_workspace_size = (N // 64) * 16
91
workspace = torch.zeros(max_workspace_size,
92
dtype=torch.int,
82
- device="cuda",
93
+ device=hidden_states.device,
94
+ requires_grad=False)
95
+
96
+ has_zero_point = w_zeros is not None
97
+ if w_zeros is None:
98
+ w_zeros = torch.empty((0, 0),
99
+ dtype=hidden_states.dtype,
100
+ device=hidden_states.device,
101
+ requires_grad=False)
102
+
103
+ if g_idx is None:
104
+ g_idx = torch.empty((0, 0),
105
+ dtype=torch.int32,
106
+ device=hidden_states.device,
107
requires_grad=False)
108
85
- scalar_type = (scalar_types.uint4b8
86
- if num_bits == 4 else scalar_types.uint8b128)
109
+ if sort_indices is None:
110
+ sort_indices = torch.empty((0),
111
+ dtype=torch.int32,
112
+ device=hidden_states.device,
113
+ requires_grad=False)
114
+
115
+ scalar_type = get_scalar_type(num_bits, has_zero_point)
116
117
intermediate_cache = torch.ops._moe_C.marlin_gemm_moe(
118
hidden_states, w, sorted_token_ids, topk_weights, topk_ids, scales,
90
- g_idx, perm, workspace, scalar_type, M, N, K, is_k_full, E, topk,
91
- block_size_m, True, False)
119
+ w_zeros, g_idx, sort_indices, workspace, scalar_type, M, N, K,
120
+ is_k_full, E, topk, block_size_m, True, False)
121
122
return torch.sum(intermediate_cache.view(*intermediate_cache.shape), dim=1)
123
@@ -97,16 +126,18 @@ def fused_marlin_moe(
126
hidden_states: torch.Tensor,
127
w1: torch.Tensor,
128
w2: torch.Tensor,
129
+ w1_scale: torch.Tensor,
130
+ w2_scale: torch.Tensor,
131
gating_output: torch.Tensor,
101
- g_idx1: torch.Tensor,
102
- g_idx2: torch.Tensor,
103
- perm1: torch.Tensor,
104
- perm2: torch.Tensor,
132
topk_weights: torch.Tensor,
133
topk_ids: torch.Tensor,
134
+ g_idx1: Optional[torch.Tensor] = None,
135
+ g_idx2: Optional[torch.Tensor] = None,
136
+ sort_indices1: Optional[torch.Tensor] = None,
137
+ sort_indices2: Optional[torch.Tensor] = None,
138
+ w1_zeros: Optional[torch.Tensor] = None,
139
+ w2_zeros: Optional[torch.Tensor] = None,
140
override_config: Optional[Dict[str, Any]] = None,
108
- w1_scale: Optional[torch.Tensor] = None,
109
- w2_scale: Optional[torch.Tensor] = None,
141
num_bits: int = 8,
142
is_k_full: bool = True,
143
) -> torch.Tensor:
@@ -118,21 +149,22 @@ def fused_marlin_moe(
149
- hidden_states (torch.Tensor): The input tensor to the MoE layer.
150
- w1 (torch.Tensor): The first set of expert weights.
151
- w2 (torch.Tensor): The second set of expert weights.
152
+ - w1_scale (torch.Tensor): Scale to be used for w1.
153
+ - w2_scale (torch.Tensor): Scale to be used for w2.
154
- gating_output (torch.Tensor): The output of the gating operation
155
(before softmax).
123
- - g_idx1 (torch.Tensor): The first set of act_order indices.
124
- - g_idx2 (torch.Tensor): The second set of act_order indices.
125
- - perm1 (torch.Tensor): The first act_order input permutation.
126
- - perm2 (torch.Tensor): The second act_order input permutation.
156
+ - g_idx1 (Optional[torch.Tensor]): The first set of act_order indices.
157
+ - g_idx2 (Optional[torch.Tensor]): The second set of act_order indices.
158
+ - sort_indices1 (Optional[torch.Tensor]): The first act_order input
159
+ permutation.
160
+ - sort_indices2 (Optional[torch.Tensor]): The second act_order input
161
+ permutation.
162
- topk_weights (torch.Tensor): Top-k weights.
163
- topk_ids (torch.Tensor): Indices of topk-k elements.
129
- - renormalize (bool): If True, renormalize the top-k weights to sum to 1.
164
- override_config (Optional[Dict[str, Any]]): Optional override
165
for the kernel configuration.
132
- - w1_scale (Optional[torch.Tensor]): Optional scale to be used for
133
- w1.
134
- - w2_scale (Optional[torch.Tensor]): Optional scale to be used for
135
- w2.
166
+ - w1_zeros (Optional[torch.Tensor]): Optional zero points to be used for w1.
167
+ - w2_zeros (Optional[torch.Tensor]): Optional zero points to be used for w2.
168
- num_bits (bool): The number of bits in expert weights quantization.
169
170
Returns:
@@ -152,6 +184,20 @@ def fused_marlin_moe(
184
assert hidden_states.dtype == torch.float16
185
assert num_bits in [4, 8]
186
187
+ has_no_act_order = (g_idx1 is None and g_idx2 is None
188
+ and sort_indices1 is None and sort_indices2 is None)
189
+ has_all_act_order = (g_idx1 is not None and g_idx2 is not None
190
+ and sort_indices1 is not None
191
+ and sort_indices2 is not None)
192
+ assert has_no_act_order or has_all_act_order, (
193
+ "g_idx and sorted_indices "
194
+ "must be all not None or must be all None")
195
+
196
+ has_no_zp = w1_zeros is None and w2_zeros is None
197
+ has_all_zp = w1_zeros is not None and w2_zeros is not None
198
+ assert has_no_zp or has_all_zp, ("zero points must be both not None or "
199
+ "must be both None")
200
+
201
M, K = hidden_states.shape
202
E = w1.shape[0]
203
N = w2.shape[1] * 16
@@ -172,14 +218,42 @@ def fused_marlin_moe(
218
219
sorted_token_ids, _, _ = moe_align_block_size(topk_ids, block_size_m, E)
220
175
- max_workspace_size = ((M + 255) // 256) * (max(2 * N, K) // 64) * 16
221
+ max_workspace_size = (max(2 * N, K) // 64) * 16
222
workspace = torch.zeros(max_workspace_size,
223
dtype=torch.int,
224
device="cuda",
225
requires_grad=False)
226
181
- scalar_type = (scalar_types.uint4b8
182
- if num_bits == 4 else scalar_types.uint8b128)
227
+ if has_no_zp:
228
+ w1_zeros = torch.empty((0, 0),
229
+ dtype=hidden_states.dtype,
230
+ device=hidden_states.device,
231
+ requires_grad=False)
232
+ w2_zeros = torch.empty((0, 0),
233
+ dtype=hidden_states.dtype,
234
+ device=hidden_states.device,
235
+ requires_grad=False)
236
+
237
+ if has_no_act_order:
238
+ g_idx1 = torch.empty((0, 0),
239
+ dtype=torch.int32,
240
+ device=hidden_states.device,
241
+ requires_grad=False)
242
+ g_idx2 = torch.empty((0, 0),
243
+ dtype=torch.int32,
244
+ device=hidden_states.device,
245
+ requires_grad=False)
246
+ sort_indices1 = torch.empty((0),
247
+ dtype=torch.int32,
248
+ device=hidden_states.device,
249
+ requires_grad=False)
250
+ sort_indices2 = torch.empty((0, 0),
251
+ dtype=torch.int32,
252
+ device=hidden_states.device,
253
+ requires_grad=False)
254
+
255
+ scalar_type1 = get_scalar_type(num_bits, has_all_zp)
256
+ scalar_type2 = get_scalar_type(num_bits, has_all_zp)
257
258
intermediate_cache2 = torch.empty(
259
(M * topk_ids.shape[1], N),
@@ -194,10 +268,11 @@ def fused_marlin_moe(
268
topk_weights,
269
topk_ids,
270
w1_scale,
271
+ w1_zeros,
272
g_idx1,
198
- perm1,
273
+ sort_indices1,
274
workspace,
200
- scalar_type,
275
+ scalar_type1,
276
M,
277
2 * N,
278
K,
@@ -218,10 +293,11 @@ def fused_marlin_moe(
293
topk_weights,
294
topk_ids,
295
w2_scale,
296
+ w2_zeros,
297
g_idx2,
222
- perm2,
298
+ sort_indices2,
299
workspace,
224
- scalar_type,
300
+ scalar_type2,
301
M,
302
K,
303
N,
vllm/model_executor/layers/quantization/awq_marlin.py
+195
-9
@@ -1,16 +1,21 @@
1
-from typing import Any, Dict, List, Optional
1
+from typing import Any, Callable, Dict, List, Optional
2
3
import torch
4
+from torch.nn import Parameter
5
6
from vllm import _custom_ops as ops
7
from vllm.logger import init_logger
7
-from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
8
+from vllm.model_executor.layers.fused_moe.layer import (
9
+ FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported)
10
+from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
11
+ set_weight_attrs)
12
from vllm.model_executor.layers.quantization.base_config import (
9
- QuantizationConfig)
13
+ QuantizationConfig, QuantizeMethodBase)
14
from vllm.model_executor.layers.quantization.utils import replace_parameter
15
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
16
apply_awq_marlin_linear, awq_to_marlin_zero_points, check_marlin_supported,
13
- marlin_make_empty_g_idx, marlin_make_workspace, marlin_permute_scales,
17
+ marlin_make_empty_g_idx, marlin_make_workspace, marlin_moe_permute_scales,
18
+ marlin_permute_scales, moe_awq_to_marlin_zero_points,
19
verify_marlin_supported, verify_marlin_supports_shape)
20
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
21
from vllm.model_executor.parameter import (GroupQuantScaleParameter,
@@ -35,12 +40,13 @@ class AWQMarlinConfig(QuantizationConfig):
40
self.group_size = group_size
41
self.has_zp = has_zp
42
self.lm_head_quantized = lm_head_quantized
43
+ self.weight_bits = weight_bits
44
39
- if weight_bits not in self.TYPE_MAP:
40
- raise ValueError(f"Unsupported num_bits = {weight_bits}. "
45
+ if self.weight_bits not in self.TYPE_MAP:
46
+ raise ValueError(f"Unsupported num_bits = {self.weight_bits}. "
47
f"Supported num_bits = {self.TYPE_MAP.keys()}")
48
43
- self.quant_type = self.TYPE_MAP[weight_bits]
49
+ self.quant_type = self.TYPE_MAP[self.weight_bits]
50
51
verify_marlin_supported(self.quant_type,
52
group_size=self.group_size,
@@ -98,10 +104,12 @@ class AWQMarlinConfig(QuantizationConfig):
104
return None
105
106
def get_quant_method(self, layer: torch.nn.Module,
101
- prefix: str) -> Optional["AWQMarlinLinearMethod"]:
107
+ prefix: str) -> Optional["QuantizeMethodBase"]:
108
if (isinstance(layer, LinearBase) or
109
(isinstance(layer, ParallelLMHead) and self.lm_head_quantized)):
110
return AWQMarlinLinearMethod(self)
111
+ elif isinstance(layer, FusedMoE):
112
+ return AWQMoEMethod(self)
113
return None
114
115
def get_scaled_act_names(self) -> List[str]:
@@ -271,4 +279,182 @@ class AWQMarlinLinearMethod(LinearMethodBase):
279
quant_type=self.quant_config.quant_type,
280
output_size_per_partition=layer.output_size_per_partition,
281
input_size_per_partition=layer.input_size_per_partition,
274
- bias=bias)
\ No newline at end of file
282
+ bias=bias)
283
+
284
+
285
+class AWQMoEMethod(FusedMoEMethodBase):
286
+
287
+ def __init__(self, quant_config: AWQMarlinConfig):
288
+ self.quant_config = quant_config
289
+
290
+ def create_weights(self, layer: torch.nn.Module, num_experts: int,
291
+ hidden_size: int, intermediate_size: int,
292
+ params_dtype: torch.dtype, **extra_weight_attrs):
293
+ extra_weight_attrs.update({
294
+ "is_transposed":
295
+ True,
296
+ "quant_method":
297
+ FusedMoeWeightScaleSupported.GROUP.value,
298
+ })
299
+
300
+ w13_qweight = Parameter(torch.empty(num_experts,
301
+ hidden_size,
302
+ 2 * intermediate_size //
303
+ self.quant_config.pack_factor,
304
+ dtype=torch.int32),
305
+ requires_grad=False)
306
+ layer.register_parameter("w13_qweight", w13_qweight)
307
+ set_weight_attrs(w13_qweight, extra_weight_attrs)
308
+
309
+ w2_qweight = Parameter(torch.empty(num_experts,
310
+ intermediate_size,
311
+ hidden_size //
312
+ self.quant_config.pack_factor,
313
+ dtype=torch.int32),
314
+ requires_grad=False)
315
+ layer.register_parameter("w2_qweight", w2_qweight)
316
+ set_weight_attrs(w2_qweight, extra_weight_attrs)
317
+
318
+ num_groups_w13 = hidden_size // self.quant_config.group_size
319
+ num_groups_w2 = intermediate_size // self.quant_config.group_size
320
+
321
+ # WEIGHT_SCALES
322
+ # Allocate 2 scales for w1 and w3 respectively.
323
+ w13_scales = Parameter(torch.empty(num_experts,
324
+ num_groups_w13,
325
+ intermediate_size * 2,
326
+ dtype=params_dtype),
327
+ requires_grad=False)
328
+ layer.register_parameter("w13_scales", w13_scales)
329
+ set_weight_attrs(w13_scales, extra_weight_attrs)
330
+
331
+ w2_scales = Parameter(torch.empty(num_experts,
332
+ num_groups_w2,
333
+ hidden_size,
334
+ dtype=params_dtype),
335
+ requires_grad=False)
336
+ layer.register_parameter("w2_scales", w2_scales)
337
+ set_weight_attrs(w2_scales, extra_weight_attrs)
338
+
339
+ # WEIGHT_ZERO_POINT
340
+ # Allocate 2 zero points for w1 and w3 respectively.
341
+ w13_qzeros = Parameter(torch.empty(num_experts,
342
+ num_groups_w13,
343
+ 2 * intermediate_size //
344
+ self.quant_config.pack_factor,
345
+ dtype=torch.int32),
346
+ requires_grad=False)
347
+ layer.register_parameter("w13_qzeros", w13_qzeros)
348
+ set_weight_attrs(w13_qzeros, extra_weight_attrs)
349
+
350
+ w2_qzeros = Parameter(torch.empty(num_experts,
351
+ num_groups_w2,
352
+ hidden_size //
353
+ self.quant_config.pack_factor,
354
+ dtype=torch.int32),
355
+ requires_grad=False)
356
+ layer.register_parameter("w2_qzeros", w2_qzeros)
357
+ set_weight_attrs(w2_qzeros, extra_weight_attrs)
358
+
359
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
360
+ num_experts = layer.w13_qweight.shape[0]
361
+ device = layer.w13_qweight.device
362
+
363
+ layer.w13_g_idx_sort_indices = torch.nn.Parameter(
364
+ torch.empty((num_experts, 0), dtype=torch.int32, device=device),
365
+ requires_grad=False,
366
+ )
367
+ layer.w2_g_idx_sort_indices = torch.nn.Parameter(
368
+ torch.empty((num_experts, 0), dtype=torch.int32, device=device),
369
+ requires_grad=False,
370
+ )
371
+
372
+ marlin_w13_qweight = ops.awq_marlin_moe_repack(
373
+ layer.w13_qweight,
374
+ layer.w13_g_idx_sort_indices,
375
+ size_k=layer.w13_qweight.shape[1],
376
+ size_n=layer.w13_qweight.shape[2] * self.quant_config.pack_factor,
377
+ num_bits=self.quant_config.weight_bits,
378
+ )
379
+ replace_parameter(layer, "w13_qweight", marlin_w13_qweight)
380
+
381
+ marlin_w2_qweight = ops.awq_marlin_moe_repack(
382
+ layer.w2_qweight,
383
+ layer.w2_g_idx_sort_indices,
384
+ size_k=layer.w2_qweight.shape[1],
385
+ size_n=layer.w2_qweight.shape[2] * self.quant_config.pack_factor,
386
+ num_bits=self.quant_config.weight_bits,
387
+ )
388
+ replace_parameter(layer, "w2_qweight", marlin_w2_qweight)
389
+
390
+ # Why does this take the intermediate size for size_k?
391
+ marlin_w13_scales = marlin_moe_permute_scales(
392
+ s=layer.w13_scales,
393
+ size_k=layer.intermediate_size_per_partition,
394
+ size_n=layer.w13_scales.shape[2],
395
+ group_size=self.quant_config.group_size,
396
+ )
397
+
398
+ replace_parameter(layer, "w13_scales", marlin_w13_scales)
399
+
400
+ marlin_w2_scales = marlin_moe_permute_scales(
401
+ s=layer.w2_scales,
402
+ size_k=layer.intermediate_size_per_partition,
403
+ size_n=layer.w2_scales.shape[2],
404
+ group_size=self.quant_config.group_size,
405
+ )
406
+ replace_parameter(layer, "w2_scales", marlin_w2_scales)
407
+
408
+ marlin_w13_zp = moe_awq_to_marlin_zero_points(
409
+ layer.w13_qzeros,
410
+ size_k=layer.w13_qzeros.shape[1],
411
+ size_n=layer.w13_qzeros.shape[2] * self.quant_config.pack_factor,
412
+ num_bits=self.quant_config.weight_bits)
413
+ replace_parameter(layer, "w13_qzeros", marlin_w13_zp)
414
+
415
+ marlin_w2_zp = moe_awq_to_marlin_zero_points(
416
+ layer.w2_qzeros,
417
+ size_k=layer.w2_qzeros.shape[1],
418
+ size_n=layer.w2_qzeros.shape[2] * self.quant_config.pack_factor,
419
+ num_bits=self.quant_config.weight_bits)
420
+ replace_parameter(layer, "w2_qzeros", marlin_w2_zp)
421
+
422
+ def apply(
423
+ self,
424
+ layer: torch.nn.Module,
425
+ x: torch.Tensor,
426
+ router_logits: torch.Tensor,
427
+ top_k: int,
428
+ renormalize: bool = True,
429
+ use_grouped_topk: bool = False,
430
+ num_expert_group: Optional[int] = None,
431
+ topk_group: Optional[int] = None,
432
+ custom_routing_function: Optional[Callable] = None,
433
+ ) -> torch.Tensor:
434
+
435
+ from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
436
+ fused_marlin_moe)
437
+
438
+ topk_weights, topk_ids = FusedMoE.select_experts(
439
+ hidden_states=x,
440
+ router_logits=router_logits,
441
+ use_grouped_topk=use_grouped_topk,
442
+ top_k=top_k,
443
+ renormalize=renormalize,
444
+ topk_group=topk_group,
445
+ num_expert_group=num_expert_group,
446
+ custom_routing_function=custom_routing_function)
447
+
448
+ return fused_marlin_moe(
449
+ x,
450
+ layer.w13_qweight,
451
+ layer.w2_qweight,
452
+ layer.w13_scales,
453
+ layer.w2_scales,
454
+ router_logits,
455
+ topk_weights,
456
+ topk_ids,
457
+ w1_zeros=layer.w13_qzeros,
458
+ w2_zeros=layer.w2_qzeros,
459
+ num_bits=self.quant_config.weight_bits,
460
+ )
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+6
-6
@@ -498,14 +498,14 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
498
x,
499
layer.w13_weight_packed,
500
layer.w2_weight_packed,
501
+ layer.w13_weight_scale,
502
+ layer.w2_weight_scale,
503
router_logits,
502
- layer.w13_g_idx,
503
- layer.w2_g_idx,
504
- layer.w13_g_idx_sort_indices,
505
- layer.w2_g_idx_sort_indices,
504
topk_weights,
505
topk_ids,
508
- w1_scale=layer.w13_weight_scale,
509
- w2_scale=layer.w2_weight_scale,
506
+ g_idx1=layer.w13_g_idx,
507
+ g_idx2=layer.w2_g_idx,
508
+ sort_indices1=layer.w13_g_idx_sort_indices,
509
+ sort_indices2=layer.w2_g_idx_sort_indices,
510
num_bits=self.num_bits,
511
)
vllm/model_executor/layers/quantization/gptq_marlin.py
+6
-6
@@ -557,14 +557,14 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
557
x,
558
layer.w13_qweight,
559
layer.w2_qweight,
560
+ layer.w13_scales,
561
+ layer.w2_scales,
562
router_logits,
561
- layer.w13_g_idx,
562
- layer.w2_g_idx,
563
- layer.w13_g_idx_sort_indices,
564
- layer.w2_g_idx_sort_indices,
563
topk_weights,
564
topk_ids,
567
- w1_scale=layer.w13_scales,
568
- w2_scale=layer.w2_scales,
565
+ g_idx1=layer.w13_g_idx,
566
+ g_idx2=layer.w2_g_idx,
567
+ sort_indices1=layer.w13_g_idx_sort_indices,
568
+ sort_indices2=layer.w2_g_idx_sort_indices,
569
num_bits=self.quant_config.quant_type.size_bits,
570
).to(orig_dtype)
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+15
@@ -208,6 +208,7 @@ def marlin_moe_permute_scales(
208
device=s.device,
209
dtype=s.dtype,
210
)
211
+
212
for e in range(num_experts):
213
output[e] = marlin_permute_scales(s[e], size_k, size_n, group_size)
214
return output
@@ -258,6 +259,20 @@ def awq_to_marlin_zero_points(q_zp_packed: torch.Tensor, size_k: int,
259
return marlin_zp
260
261
262
+def moe_awq_to_marlin_zero_points(q_zp_packed: torch.Tensor, size_k: int,
263
+ size_n: int, num_bits: int):
264
+ num_experts = q_zp_packed.shape[0]
265
+ output = torch.empty(
266
+ (num_experts, q_zp_packed.shape[1], q_zp_packed.shape[2]),
267
+ device=q_zp_packed.device,
268
+ dtype=q_zp_packed.dtype,
269
+ )
270
+ for e in range(num_experts):
271
+ output[e] = awq_to_marlin_zero_points(q_zp_packed[e], size_k, size_n,
272
+ num_bits)
273
+ return output
274
+
275
+
276
def apply_gptq_marlin_linear(
277
input: torch.Tensor,
278
weight: torch.Tensor,
vllm/model_executor/model_loader/utils.py
+3
-1
@@ -23,7 +23,9 @@ def get_model_architecture(
23
architectures = getattr(model_config.hf_config, "architectures", [])
24
# Special handling for quantized Mixtral.
25
# FIXME(woosuk): This is a temporary hack.
26
- mixtral_supported = ["fp8", "compressed-tensors", "gptq_marlin"]
26
+ mixtral_supported = [
27
+ "fp8", "compressed-tensors", "gptq_marlin", "awq_marlin"
28
+ ]
29
30
if (model_config.quantization is not None
31
and model_config.quantization not in mixtral_supported