@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 05d686432f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Kernel] Zero point support in fused MarlinMoE kernel + AWQ Fused MoE (#8973)

Co-authored-by: Dipika <dipikasikka1@gmail.com> Co-authored-by: Dipika Sikka <ds3822@columbia.edu>

ElizaWszola committed Oct 4, 2024 at 20:34 UTC 05d686432f2e13296127962861b21c25cdcdfc8b
23 files changed +969 -223
CMakeLists.txt
+2
@@ -433,6 +433,8 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
433 "csrc/moe/marlin_kernels/marlin_moe_kernel_ku4b8.cu"
434 "csrc/moe/marlin_kernels/marlin_moe_kernel_ku8b128.h"
435 "csrc/moe/marlin_kernels/marlin_moe_kernel_ku8b128.cu"
436 + "csrc/moe/marlin_kernels/marlin_moe_kernel_ku4.h"
437 + "csrc/moe/marlin_kernels/marlin_moe_kernel_ku4.cu"
438 "csrc/moe/marlin_moe_ops.cu")
439
440 set_gencode_flags_for_srcs(
csrc/moe/marlin_kernels/marlin_moe_kernel.h
+244 -53
@@ -38,6 +38,7 @@ using FragA = Vec<half2, 4>;
38 using FragB = Vec<half2, 2>;
39 using FragC = Vec<float, 4>;
40 using FragS = Vec<half2, 1>; // quantization scales
41 +using FragZP = Vec<half2, 4>;
42
43 // Predicated asynchronous global->shared copy; used for inputs A where we apply
44 // predication to handle batchsizes that are not multiples of 16.
@@ -175,6 +176,46 @@ __device__ inline FragB dequant<vllm::kU8B128.id()>(int q) {
176 return frag_b;
177 }
178
179 +template <>
180 +__device__ inline FragB dequant<vllm::kU4.id()>(int q) {
181 + const int LO = 0x000f000f;
182 + const int HI = 0x00f000f0;
183 + const int EX = 0x64006400;
184 + // Guarantee that the `(a & b) | c` operations are LOP3s.
185 + int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX);
186 + int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX);
187 +
188 + const int SUB = 0x64006400;
189 + const int MUL = 0x2c002c00;
190 + const int ADD = 0xd400d400;
191 + FragB frag_b;
192 + frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
193 + *reinterpret_cast<const half2*>(&SUB));
194 + frag_b[1] = __hfma2(*reinterpret_cast<half2*>(&hi),
195 + *reinterpret_cast<const half2*>(&MUL),
196 + *reinterpret_cast<const half2*>(&ADD));
197 + return frag_b;
198 +}
199 +
200 +template <>
201 +__device__ inline FragB dequant<vllm::kU8.id()>(int q) {
202 + static constexpr uint32_t mask_for_elt_01 = 0x5250;
203 + static constexpr uint32_t mask_for_elt_23 = 0x5351;
204 + static constexpr uint32_t start_byte_for_fp16 = 0x64646464;
205 +
206 + uint32_t lo = prmt<start_byte_for_fp16, mask_for_elt_01>(q);
207 + uint32_t hi = prmt<start_byte_for_fp16, mask_for_elt_23>(q);
208 +
209 + static constexpr uint32_t I8s_TO_F16s_MAGIC_NUM = 0x64006400;
210 +
211 + FragB frag_b;
212 + frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
213 + *reinterpret_cast<const half2*>(&I8s_TO_F16s_MAGIC_NUM));
214 + frag_b[1] = __hsub2(*reinterpret_cast<half2*>(&hi),
215 + *reinterpret_cast<const half2*>(&I8s_TO_F16s_MAGIC_NUM));
216 + return frag_b;
217 +}
218 +
219 // Multiply dequantized values by the corresponding quantization scale; used
220 // only for grouped quantization.
221 __device__ inline void scale(FragB& frag_b, FragS& frag_s, int i) {
@@ -183,11 +224,10 @@ __device__ inline void scale(FragB& frag_b, FragS& frag_s, int i) {
224 frag_b[1] = __hmul2(frag_b[1], s);
225 }
226
186 -// Given 2 floats multiply by 2 scales (halves)
187 -__device__ inline void scale_float(float* c, FragS& s) {
188 - __half* s_ptr = reinterpret_cast<__half*>(&s);
189 - c[0] = __fmul_rn(c[0], __half2float(s_ptr[0]));
190 - c[1] = __fmul_rn(c[1], __half2float(s_ptr[1]));
227 +__device__ inline void sub_zp(FragB& frag_b, half2& frag_zp, int i) {
228 + half2 zp = __half2half2(reinterpret_cast<__half*>(&frag_zp)[i]);
229 + frag_b[0] = __hsub2(frag_b[0], zp);
230 + frag_b[1] = __hsub2(frag_b[1], zp);
231 }
232
233 // Same as above, but for act_order (each K is multiplied individually)
@@ -205,6 +245,13 @@ __device__ inline void scale4(FragB& frag_b, FragS& frag_s_1, FragS& frag_s_2,
245 frag_b[1] = __hmul2(frag_b[1], s_val_3_4);
246 }
247
248 +// Given 2 floats multiply by 2 scales (halves)
249 +__device__ inline void scale_float(float* c, FragS& s) {
250 + __half* s_ptr = reinterpret_cast<__half*>(&s);
251 + c[0] = __fmul_rn(c[0], __half2float(s_ptr[0]));
252 + c[1] = __fmul_rn(c[1], __half2float(s_ptr[1]));
253 +}
254 +
255 // Wait until barrier reaches `count`, then lock for current threadblock.
256 __device__ inline void barrier_acquire(int* lock, int count) {
257 if (threadIdx.x == 0) {
@@ -248,10 +295,11 @@ template <const vllm::ScalarTypeId w_type_id, // weight ScalarType id
295 const int stages, // number of stages for the async global->shared
296 // fetch pipeline
297 const bool has_act_order, // whether act_order is enabled
298 + const bool has_zp, // whether zero-points are enabled
299 const int group_blocks = -1 // number of consecutive 16x16 blocks
300 // with a separate quantization scale
301 >
254 -__device__ inline void MarlinMoESingle(
302 +__device__ void MarlinMoESingle(
303 const int4* __restrict__ A, // fp16 input matrix of shape mxk
304 const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
305 int4* __restrict__ C, // fp16 output buffer of shape mxn
@@ -259,6 +307,8 @@ __device__ inline void MarlinMoESingle(
307 const float* __restrict__ topk_weights, // float topk weights
308 const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
309 // (k/groupsize)xn
310 + const int4* __restrict__ zp_ptr, // 4bit packed zero-points of shape
311 + // (k/groupsize)x(n/pack_factor)
312 const int* __restrict__ g_idx, // int32 group indices of shape k
313 const int* __restrict__ expert_offsets,
314 int num_groups, // number of scale groups per output channel
@@ -400,8 +450,12 @@ __device__ inline void MarlinMoESingle(
450 int tb_n_warps = thread_n_blocks / 4;
451 int act_s_col_tb_stride = act_s_col_warp_stride * tb_n_warps;
452
403 - constexpr int sorted_sh_stride = threads;
404 - constexpr int sorted_gl_stride = threads;
453 + // Zero-points sizes/strides
454 + int zp_gl_stride = (prob_n / pack_factor) / 4;
455 + constexpr int zp_sh_stride = ((16 * thread_n_blocks) / pack_factor) / 4;
456 + constexpr int zp_tb_groups = s_tb_groups;
457 + constexpr int zp_sh_stage = has_zp ? zp_tb_groups * zp_sh_stride : 0;
458 + int zp_gl_rd_delta = zp_gl_stride;
459
460 // Global A read index of current thread.
461 int a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
@@ -442,6 +496,19 @@ __device__ inline void MarlinMoESingle(
496 int s_sh_wr = threadIdx.x;
497 bool s_sh_wr_pred = threadIdx.x < s_sh_stride;
498
499 + // Zero-points
500 + int zp_gl_rd;
501 + if constexpr (has_zp) {
502 + if constexpr (group_blocks == -1) {
503 + zp_gl_rd = zp_sh_stride * slice_col + threadIdx.x;
504 + } else {
505 + zp_gl_rd = zp_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
506 + zp_sh_stride * slice_col + threadIdx.x;
507 + }
508 + }
509 + int zp_sh_wr = threadIdx.x;
510 + bool zp_sh_wr_pred = threadIdx.x < zp_sh_stride;
511 +
512 // We use a different scale layout for grouped and column-wise quantization as
513 // we scale a `half2` tile in column-major layout in the former and in
514 // row-major in the latter case.
@@ -453,23 +520,29 @@ __device__ inline void MarlinMoESingle(
520 s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
521 (threadIdx.x % 32) % 4;
522
523 + // Zero-points have the same read layout as the scales
524 + // (without column-wise case)
525 + constexpr int num_col_threads = 8;
526 + constexpr int num_row_threads = 4;
527 + constexpr int num_ints_per_thread = 8 / pack_factor;
528 + int zp_sh_rd;
529 + if constexpr (has_zp) {
530 + zp_sh_rd = num_ints_per_thread * num_col_threads *
531 + ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
532 + num_ints_per_thread * ((threadIdx.x % 32) / num_row_threads);
533 + }
534 +
535 int sh_first_group_id = -1;
536 int sh_num_groups = -1;
537 constexpr int sh_max_num_groups = 32;
538
460 - int shs_size;
461 - if constexpr (has_act_order)
462 - shs_size = sh_max_num_groups * s_sh_stride + threads;
463 - else
464 - shs_size = group_blocks > 0 ? stages * s_sh_stage : threads;
465 -
539 extern __shared__ int4 sh[];
540 // Shared memory storage for global fetch pipelines.
541 int4* sh_a = sh;
542 int4* sh_b = sh_a + (stages * a_sh_stage);
543 int4* sh_g_idx = sh_b + (stages * b_sh_stage);
471 - int4* sh_s = sh_g_idx + (stages * g_idx_stage);
472 - int* sh_sorted = (int*)(sh_s + shs_size);
544 + int4* sh_zp = sh_g_idx + (stages * g_idx_stage);
545 + int4* sh_s = sh_zp + (stages * zp_sh_stage);
546
547 // Precompute which thread should not read memory in which iterations; this is
548 // needed if there are more threads than required for a certain tilesize or
@@ -525,8 +598,10 @@ __device__ inline void MarlinMoESingle(
598 FragA frag_a[2][thread_m_blocks];
599 I4 frag_b_quant[2][b_thread_vecs];
600 FragC frag_c[thread_m_blocks][4][2];
528 - FragS frag_s[2][4]; // No act-order
529 - FragS act_frag_s[2][4][4]; // For act-order
601 + FragS frag_s[2][4]; // No act-order
602 + FragS act_frag_s[2][4][4]; // For act-order
603 + int frag_qzp[2][num_ints_per_thread]; // Zero-points
604 + FragZP frag_zp; // Zero-points in fp16
605
606 // Zero accumulators.
607 auto zero_accums = [&]() {
@@ -633,6 +708,28 @@ __device__ inline void MarlinMoESingle(
708 }
709 }
710 }
711 +
712 + if constexpr (has_zp && group_blocks != -1) {
713 + int4* sh_zp_stage = sh_zp + zp_sh_stage * pipe;
714 +
715 + if constexpr (group_blocks >= thread_k_blocks) {
716 + // Only fetch zero-points if this tile starts a new group
717 + if (pipe % (group_blocks / thread_k_blocks) == 0) {
718 + if (zp_sh_wr_pred) {
719 + cp_async4(&sh_zp_stage[zp_sh_wr], &zp_ptr[zp_gl_rd]);
720 + }
721 + zp_gl_rd += zp_gl_rd_delta;
722 + }
723 + } else {
724 + for (int i = 0; i < zp_tb_groups; i++) {
725 + if (zp_sh_wr_pred) {
726 + cp_async4(&sh_zp_stage[i * zp_sh_stride + zp_sh_wr],
727 + &zp_ptr[zp_gl_rd]);
728 + }
729 + zp_gl_rd += zp_gl_rd_delta;
730 + }
731 + }
732 + }
733 }
734 }
735 // Insert a fence even when we are winding down the pipeline to ensure that
@@ -640,15 +737,9 @@ __device__ inline void MarlinMoESingle(
737 cp_async_fence();
738 };
739
643 - // TODO we are currently hitting illegal memory accesses when fetching
644 - // sorted_ids to shared data: fix this
645 - auto fetch_sorted_ids_to_shared = [&]() {
646 - const int mpt = ceildiv(prob_m, threads);
647 - for (int i = 0; i < mpt; i++) {
648 - if ((i * sorted_gl_stride) + threadIdx.x < prob_m) {
649 - sh_sorted[(i * sorted_sh_stride) + threadIdx.x] =
650 - sorted_ids[(i * sorted_gl_stride) + threadIdx.x];
651 - }
740 + auto fetch_zp_to_shared = [&]() {
741 + if (zp_sh_wr_pred) {
742 + cp_async4(&sh_zp[zp_sh_wr], &zp_ptr[zp_gl_rd]);
743 }
744 };
745
@@ -799,8 +890,83 @@ __device__ inline void MarlinMoESingle(
890 }
891 };
892
893 + auto fetch_zp_to_registers = [&](int k, int full_pipe) {
894 + // This code does not handle group_blocks == 0,
895 + // which signifies act_order.
896 + // has_zp implies AWQ, which doesn't have act_order,
897 + static_assert(!has_zp || group_blocks != 0);
898 +
899 + if constexpr (has_zp) {
900 + int pipe = full_pipe % stages;
901 +
902 + if constexpr (group_blocks == -1) {
903 + for (int i = 0; i < num_ints_per_thread; i++) {
904 + frag_qzp[k % 2][i] = (reinterpret_cast<int*>(sh_zp))[zp_sh_rd + i];
905 + }
906 +
907 + } else if constexpr (group_blocks >= thread_k_blocks) {
908 + int4* sh_zp_stage =
909 + sh_zp + zp_sh_stage * ((group_blocks / thread_k_blocks) *
910 + (pipe / (group_blocks / thread_k_blocks)));
911 + for (int i = 0; i < num_ints_per_thread; i++) {
912 + frag_qzp[k % 2][i] =
913 + (reinterpret_cast<int*>(sh_zp_stage))[zp_sh_rd + i];
914 + }
915 + } else {
916 + int warp_id = threadIdx.x / 32;
917 + int n_warps = thread_n_blocks / 4;
918 +
919 + int warp_row = warp_id / n_warps;
920 +
921 + int cur_k = warp_row * 16;
922 + cur_k += k_iter_size * (k % b_sh_wr_iters);
923 +
924 + int k_blocks = cur_k / 16;
925 + int cur_group_id = 0;
926 +
927 + // Suppress bogus and persistent divide-by-zero warning
928 + #pragma nv_diagnostic push
929 + #pragma nv_diag_suppress divide_by_zero
930 + cur_group_id = k_blocks / group_blocks;
931 + #pragma nv_diagnostic pop
932 +
933 + int4* sh_zp_stage = sh_zp + zp_sh_stage * pipe;
934 +
935 + sh_zp_stage += cur_group_id * zp_sh_stride;
936 +
937 + for (int i = 0; i < num_ints_per_thread; i++) {
938 + frag_qzp[k % 2][i] =
939 + (reinterpret_cast<int*>(sh_zp_stage))[zp_sh_rd + i];
940 + }
941 + }
942 + }
943 + };
944 +
945 // Execute the actual tensor core matmul of a sub-tile.
946 auto matmul = [&](int k) {
947 + if constexpr (has_zp) {
948 + FragB frag_zp_0;
949 + FragB frag_zp_1;
950 + int zp_quant_0, zp_quant_1;
951 +
952 + if constexpr (w_type.size_bits() == 4) {
953 + zp_quant_0 = frag_qzp[k % 2][0];
954 + zp_quant_1 = zp_quant_0 >> 8;
955 + } else {
956 + static_assert(w_type.size_bits() == 8);
957 + zp_quant_0 = frag_qzp[k % 2][0];
958 + zp_quant_1 = frag_qzp[k % 2][1];
959 + }
960 +
961 + frag_zp_0 = dequant<w_type_id>(zp_quant_0);
962 + frag_zp_1 = dequant<w_type_id>(zp_quant_1);
963 +
964 + frag_zp[0] = frag_zp_0[0];
965 + frag_zp[1] = frag_zp_0[1];
966 + frag_zp[2] = frag_zp_1[0];
967 + frag_zp[3] = frag_zp_1[1];
968 + }
969 +
970 // We have the m dimension as the inner loop in order to encourage overlapping
971 // dequantization and matmul operations.
972 #pragma unroll
@@ -818,6 +984,10 @@ __device__ inline void MarlinMoESingle(
984
985 FragB frag_b0 = dequant<w_type_id>(b_quant_0);
986 FragB frag_b1 = dequant<w_type_id>(b_quant_1);
987 + // Apply zero-point to frag_b0
988 + if constexpr (has_zp) {
989 + sub_zp(frag_b0, frag_zp[j], 0);
990 + }
991
992 // Apply scale to frag_b0
993 if constexpr (has_act_order) {
@@ -829,6 +999,11 @@ __device__ inline void MarlinMoESingle(
999 }
1000 }
1001
1002 + // Apply zero-point to frag_b1
1003 + if constexpr (has_zp) {
1004 + sub_zp(frag_b1, frag_zp[j], 1);
1005 + }
1006 +
1007 // Apply scale to frag_b1
1008 if constexpr (has_act_order) {
1009 scale4(frag_b1, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
@@ -1062,9 +1237,6 @@ __device__ inline void MarlinMoESingle(
1237
1238 // Start global fetch and register load pipelines.
1239 auto start_pipes = [&]() {
1065 - // TODO re-enable after fixing this function
1066 - // fetch_sorted_ids_to_shared();
1067 - // __syncthreads();
1240
1241 #pragma unroll
1242 for (int i = 0; i < stages - 1; i++) {
@@ -1075,6 +1247,12 @@ __device__ inline void MarlinMoESingle(
1247 }
1248 fetch_scales_to_shared(true, g_idx[slice_k_start], g_idx[last_g_idx]);
1249 }
1250 +
1251 + if constexpr (has_zp && group_blocks == -1) {
1252 + if (i == 0) {
1253 + fetch_zp_to_shared();
1254 + }
1255 + }
1256 fetch_to_shared(i, i, i < slice_iters);
1257 }
1258
@@ -1083,6 +1261,7 @@ __device__ inline void MarlinMoESingle(
1261 init_same_group(0);
1262 fetch_to_registers(0, 0);
1263 fetch_scales_to_registers(0, 0);
1264 + fetch_zp_to_registers(0, 0);
1265 a_gl_rd += a_gl_rd_delta_o * (stages - 1);
1266 slice_k_start_shared_fetch += tb_k * (stages - 1);
1267 };
@@ -1102,6 +1281,7 @@ __device__ inline void MarlinMoESingle(
1281 for (int k = 0; k < b_sh_wr_iters; k++) {
1282 fetch_to_registers(k + 1, pipe % stages);
1283 fetch_scales_to_registers(k + 1, pipe);
1284 + fetch_zp_to_registers(k + 1, pipe);
1285 if (k == b_sh_wr_iters - 2) {
1286 fetch_to_shared((pipe + stages - 1) % stages, pipe,
1287 slice_iters >= stages);
@@ -1236,7 +1416,9 @@ __device__ inline void MarlinMoESingle(
1416
1417 } else {
1418 s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
1419 + zp_gl_rd = zp_sh_stride * slice_col + threadIdx.x;
1420 }
1421 +
1422 start_pipes();
1423 }
1424 }
@@ -1250,6 +1432,7 @@ template <const vllm::ScalarTypeId w_type_id, // weight ScalarType id
1432 const int stages, // number of stages for the async global->shared
1433 // fetch pipeline
1434 const bool has_act_order, // whether act_order is enabled
1435 + const bool has_zp, // whether zero-points are enabled
1436 const int group_blocks = -1 // number of consecutive 16x16 blocks
1437 // with a separate quantization scale
1438 >
@@ -1261,6 +1444,8 @@ __global__ void MarlinMoE(
1444 const float* __restrict__ topk_weights, // float topk weights
1445 const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
1446 // (k/groupsize)xn
1447 + const int4* __restrict__ zp_ptr, // 4bit packed zero-points of shape
1448 + // (k/groupsize)x(n/pack_factor)
1449 const int* __restrict__ g_idx, // int32 group indices of shape k
1450 const int* __restrict__ expert_offsets,
1451 int num_groups, // number of scale groups per output channel
@@ -1309,29 +1494,29 @@ __global__ void MarlinMoE(
1494
1495 if (max_block == 1) {
1496 MarlinMoESingle<w_type_id, threads, 1, thread_n_blocks, thread_k_blocks,
1312 - stages, has_act_order, group_blocks>(
1313 - A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1497 + stages, has_act_order, has_zp, group_blocks>(
1498 + A, B, C, sorted_ids_expert, topk_weights, scales_ptr, zp_ptr, g_idx,
1499 expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1500 prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1501 current_m_block);
1502 } else if (max_block == 2) {
1503 MarlinMoESingle<w_type_id, threads, 2, thread_n_blocks, thread_k_blocks,
1319 - stages, has_act_order, group_blocks>(
1320 - A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1504 + stages, has_act_order, has_zp, group_blocks>(
1505 + A, B, C, sorted_ids_expert, topk_weights, scales_ptr, zp_ptr, g_idx,
1506 expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1507 prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1508 current_m_block);
1509 } else if (max_block == 3) {
1510 MarlinMoESingle<w_type_id, threads, 3, thread_n_blocks, thread_k_blocks,
1326 - stages, has_act_order, group_blocks>(
1327 - A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1511 + stages, has_act_order, has_zp, group_blocks>(
1512 + A, B, C, sorted_ids_expert, topk_weights, scales_ptr, zp_ptr, g_idx,
1513 expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1514 prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1515 current_m_block);
1516 } else {
1517 MarlinMoESingle<w_type_id, threads, 4, thread_n_blocks, thread_k_blocks,
1333 - stages, has_act_order, group_blocks>(
1334 - A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1518 + stages, has_act_order, has_zp, group_blocks>(
1519 + A, B, C, sorted_ids_expert, topk_weights, scales_ptr, zp_ptr, g_idx,
1520 expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1521 prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1522 current_m_block);
@@ -1347,6 +1532,7 @@ template <const vllm::ScalarTypeId w_type_id, // weight ScalarType id
1532 const int stages, // number of stages for the async global->shared
1533 // fetch pipeline
1534 const bool has_act_order, // whether act_order is enabled
1535 + const bool has_zp, // whether zero-points are enabled
1536 const int group_blocks = -1 // number of consecutive 16x16 blocks
1537 // with a separate quantization scale
1538 >
@@ -1358,6 +1544,8 @@ __global__ void MarlinMoE(
1544 const float* __restrict__ topk_weights, // float topk weights
1545 const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
1546 // (k/groupsize)xn
1547 + const int4* __restrict__ zp_ptr, // 4bit packed zero-points of shape
1548 + // (k/groupsize)x(n/pack_factor)
1549 const int* __restrict__ g_idx, // int32 group indices of shape k
1550 const int* __restrict__ expert_offsets,
1551 int num_groups, // number of scale groups per output channel
@@ -1374,7 +1562,6 @@ __global__ void MarlinMoE(
1562 int current_m_block, // current m block to start kernel computation from
1563 int max_par, // maximum parallelism
1564 int cfg_max_m_blocks // upper bound on m blocks
1377 -
1565 ) {
1566 // Marlin is not implemented yet for SM < 8.0
1567 assert(false);
@@ -1389,37 +1576,41 @@ __global__ void MarlinMoE(
1576 const int USER_THREADS =
1577 256; // Note: This is only used with user-provided thread_k/n
1578 const int STAGES = 4; // 4 pipeline stages fit into shared memory
1392 -// const int SHARED_MEM =
1393 -// 96 * 1024; // max shared memory on compute capability 8.6 (< 8.0)
1579
1580 static constexpr int min_thread_n = 64;
1581 static constexpr int min_thread_k = 64;
1582
1583 #define __CALL_IF_MOE(W_TYPE, THREAD_N_BLOCKS, THREAD_K_BLOCKS, HAS_ACT_ORDER, \
1399 - GROUP_BLOCKS, NUM_THREADS) \
1584 + HAS_ZP, GROUP_BLOCKS, NUM_THREADS) \
1585 else if (q_type == W_TYPE && thread_n_blocks == THREAD_N_BLOCKS && \
1586 thread_k_blocks == THREAD_K_BLOCKS && \
1402 - has_act_order == HAS_ACT_ORDER && group_blocks == GROUP_BLOCKS && \
1403 - num_threads == NUM_THREADS) { \
1587 + has_act_order == HAS_ACT_ORDER && has_zp == HAS_ZP && \
1588 + group_blocks == GROUP_BLOCKS && num_threads == NUM_THREADS) { \
1589 cudaFuncSetAttribute( \
1590 MarlinMoE<W_TYPE.id(), NUM_THREADS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1406 - STAGES, HAS_ACT_ORDER, GROUP_BLOCKS>, \
1591 + STAGES, HAS_ACT_ORDER, HAS_ZP, GROUP_BLOCKS>, \
1592 cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
1593 MarlinMoE<W_TYPE.id(), NUM_THREADS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1409 - STAGES, HAS_ACT_ORDER, GROUP_BLOCKS> \
1594 + STAGES, HAS_ACT_ORDER, HAS_ZP, GROUP_BLOCKS> \
1595 <<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
1596 A_ptr, B_ptr, C_ptr, sorted_ids_ptr, topk_weights_ptr, s_ptr, \
1412 - g_idx_ptr, expert_offsets_ptr, num_groups, expert_idx, \
1597 + zp_ptr, g_idx_ptr, expert_offsets_ptr, num_groups, expert_idx, \
1598 num_experts, topk, prob_m, prob_n, prob_k, tot_m, locks, \
1599 replicate_input, apply_weights, m_block, max_par, \
1600 cfg_max_m_blocks); \
1601 }
1602
1418 -#define GPTQ_CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1419 - __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1420 - __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1421 - __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1422 - __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1423 - __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS)
1603 +#define GPTQ_CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1604 + __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1605 + __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1606 + __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1607 + __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1608 + __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS)
1609 +
1610 +#define AWQ_CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1611 + __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1612 + __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1613 + __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1614 + __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS)
1615
1616 } // namespace marlin_moe
csrc/moe/marlin_kernels/marlin_moe_kernel_ku4.cu new
+31
@@ -0,0 +1,31 @@
1 +#include "marlin_moe_kernel_ku4.h"
2 +
3 +namespace marlin_moe {
4 +
5 +// We return bool so we can create these different kernel calls as a sequence
6 +// of if-elseif's.
7 +bool call_marlin_moe_kernel_ku4(
8 + vllm::ScalarType const& q_type, int thread_n_blocks, int thread_k_blocks,
9 + bool has_act_order, int group_blocks, int num_threads, int blocks,
10 + int max_shared_mem, cudaStream_t stream, const int4* A_ptr,
11 + const int4* B_ptr, int4* C_ptr, const int* sorted_ids_ptr,
12 + const float* topk_weights_ptr, const int4* s_ptr, const int4* zp_ptr,
13 + const int* g_idx_ptr, int* expert_offsets_ptr, int num_groups,
14 + int expert_idx, int num_experts, int topk, int prob_m, int prob_n,
15 + int prob_k, int tot_m, int* locks, bool replicate_input, bool apply_weights,
16 + int m_block, int max_par, int cfg_max_m_blocks) {
17 + bool has_zp = true;
18 +
19 + if (false) {
20 + }
21 + AWQ_CALL_IF_MOE(vllm::kU4, 16, 4, 256)
22 + AWQ_CALL_IF_MOE(vllm::kU4, 8, 8, 256)
23 + AWQ_CALL_IF_MOE(vllm::kU4, 8, 4, 128)
24 + AWQ_CALL_IF_MOE(vllm::kU4, 4, 8, 128)
25 + else {
26 + return false;
27 + }
28 + return true;
29 +}
30 +
31 +} // namespace marlin_moe
csrc/moe/marlin_kernels/marlin_moe_kernel_ku4.h new
+20
@@ -0,0 +1,20 @@
1 +#pragma once
2 +
3 +#include "marlin_moe_kernel.h"
4 +
5 +namespace marlin_moe {
6 +
7 +// We return bool so we can create these different kernel calls as a sequence
8 +// of if-elseif's.
9 +bool call_marlin_moe_kernel_ku4(
10 + vllm::ScalarType const& q_type, int thread_n_blocks, int thread_k_blocks,
11 + bool has_act_order, int group_blocks, int num_threads, int blocks,
12 + int max_shared_mem, cudaStream_t stream, const int4* A_ptr,
13 + const int4* B_ptr, int4* C_ptr, const int* sorted_ids_ptr,
14 + const float* topk_weights_ptr, const int4* s_ptr, const int4* zp_ptr,
15 + const int* g_idx_ptr, int* expert_offsets_ptr, int num_groups,
16 + int expert_idx, int num_experts, int topk, int prob_m, int prob_n,
17 + int prob_k, int tot_m, int* locks, bool replicate_input, bool apply_weights,
18 + int m_block, int max_par, int cfg_max_m_blocks);
19 +
20 +} // namespace marlin_moe
csrc/moe/marlin_kernels/marlin_moe_kernel_ku4b8.cu
+7 -5
@@ -9,11 +9,13 @@ bool call_marlin_moe_kernel_ku4b8(
9 bool has_act_order, int group_blocks, int num_threads, int blocks,
10 int max_shared_mem, cudaStream_t stream, const int4* A_ptr,
11 const int4* B_ptr, int4* C_ptr, const int* sorted_ids_ptr,
12 - const float* topk_weights_ptr, const int4* s_ptr, const int* g_idx_ptr,
13 - int* expert_offsets_ptr, int num_groups, int expert_idx, int num_experts,
14 - int topk, int prob_m, int prob_n, int prob_k, int tot_m, int* locks,
15 - bool replicate_input, bool apply_weights, int m_block, int max_par,
16 - int cfg_max_m_blocks) {
12 + const float* topk_weights_ptr, const int4* s_ptr, const int4* zp_ptr,
13 + const int* g_idx_ptr, int* expert_offsets_ptr, int num_groups,
14 + int expert_idx, int num_experts, int topk, int prob_m, int prob_n,
15 + int prob_k, int tot_m, int* locks, bool replicate_input, bool apply_weights,
16 + int m_block, int max_par, int cfg_max_m_blocks) {
17 + bool has_zp = false;
18 +
19 if (false) {
20 }
21 GPTQ_CALL_IF_MOE(vllm::kU4B8, 16, 4, 256)
csrc/moe/marlin_kernels/marlin_moe_kernel_ku4b8.h
+5 -5
@@ -11,10 +11,10 @@ bool call_marlin_moe_kernel_ku4b8(
11 bool has_act_order, int group_blocks, int num_threads, int blocks,
12 int max_shared_mem, cudaStream_t stream, const int4* A_ptr,
13 const int4* B_ptr, int4* C_ptr, const int* sorted_ids_ptr,
14 - const float* topk_weights_ptr, const int4* s_ptr, const int* g_idx_ptr,
15 - int* expert_offsets_ptr, int num_groups, int expert_idx, int num_experts,
16 - int topk, int prob_m, int prob_n, int prob_k, int tot_m, int* locks,
17 - bool replicate_input, bool apply_weights, int m_block, int max_par,
18 - int cfg_max_m_blocks);
14 + const float* topk_weights_ptr, const int4* s_ptr, const int4* zp_ptr,
15 + const int* g_idx_ptr, int* expert_offsets_ptr, int num_groups,
16 + int expert_idx, int num_experts, int topk, int prob_m, int prob_n,
17 + int prob_k, int tot_m, int* locks, bool replicate_input, bool apply_weights,
18 + int m_block, int max_par, int cfg_max_m_blocks);
19
20 } // namespace marlin_moe
csrc/moe/marlin_kernels/marlin_moe_kernel_ku8b128.cu
+7 -5
@@ -9,11 +9,13 @@ bool call_marlin_moe_kernel_ku8b128(
9 bool has_act_order, int group_blocks, int num_threads, int blocks,
10 int max_shared_mem, cudaStream_t stream, const int4* A_ptr,
11 const int4* B_ptr, int4* C_ptr, const int* sorted_ids_ptr,
12 - const float* topk_weights_ptr, const int4* s_ptr, const int* g_idx_ptr,
13 - int* expert_offsets_ptr, int num_groups, int expert_idx, int num_experts,
14 - int topk, int prob_m, int prob_n, int prob_k, int tot_m, int* locks,
15 - bool replicate_input, bool apply_weights, int m_block, int max_par,
16 - int cfg_max_m_blocks) {
12 + const float* topk_weights_ptr, const int4* s_ptr, const int4* zp_ptr,
13 + const int* g_idx_ptr, int* expert_offsets_ptr, int num_groups,
14 + int expert_idx, int num_experts, int topk, int prob_m, int prob_n,
15 + int prob_k, int tot_m, int* locks, bool replicate_input, bool apply_weights,
16 + int m_block, int max_par, int cfg_max_m_blocks) {
17 + bool has_zp = false;
18 +
19 if (false) {
20 }
21 GPTQ_CALL_IF_MOE(vllm::kU8B128, 16, 4, 256)
csrc/moe/marlin_kernels/marlin_moe_kernel_ku8b128.h
+5 -5
@@ -9,10 +9,10 @@ bool call_marlin_moe_kernel_ku8b128(
9 bool has_act_order, int group_blocks, int num_threads, int blocks,
10 int max_shared_mem, cudaStream_t stream, const int4* A_ptr,
11 const int4* B_ptr, int4* C_ptr, const int* sorted_ids_ptr,
12 - const float* topk_weights_ptr, const int4* s_ptr, const int* g_idx_ptr,
13 - int* expert_offsets_ptr, int num_groups, int expert_idx, int num_experts,
14 - int topk, int prob_m, int prob_n, int prob_k, int tot_m, int* locks,
15 - bool replicate_input, bool apply_weights, int m_block, int max_par,
16 - int cfg_max_m_blocks);
12 + const float* topk_weights_ptr, const int4* s_ptr, const int4* zp_ptr,
13 + const int* g_idx_ptr, int* expert_offsets_ptr, int num_groups,
14 + int expert_idx, int num_experts, int topk, int prob_m, int prob_n,
15 + int prob_k, int tot_m, int* locks, bool replicate_input, bool apply_weights,
16 + int m_block, int max_par, int cfg_max_m_blocks);
17
18 }
csrc/moe/marlin_moe_ops.cu
+56 -28
@@ -30,6 +30,7 @@
30 #include "core/registration.h"
31 #include "marlin_kernels/marlin_moe_kernel_ku4b8.h"
32 #include "marlin_kernels/marlin_moe_kernel_ku8b128.h"
33 +#include "marlin_kernels/marlin_moe_kernel_ku4.h"
34
35 template <typename T>
36 inline std::string str(T x) {
@@ -157,6 +158,7 @@ thread_config_t small_batch_thread_configs[] = {
158 {128, 64, 128}, // Reduce N 2X, same K
159 {64, 256, 256}, // Reduce K 2X, increase N 2X
160 {64, 128, 128}, // Reduce K 2X, same N
161 + {64, 64, 128}, // Reduce both 2X
162 };
163
164 thread_config_t large_batch_thread_configs[] = {
@@ -167,6 +169,7 @@ thread_config_t large_batch_thread_configs[] = {
169 {128, 128, 256}, // Reduce N 2X, increase K 2X
170 {64, 128, 128}, // Reduce N 2X, same K
171 {128, 64, 128}, // Reduce N 4X, increase K 2X
172 + {64, 64, 128}, // Reduce N 4X, same K
173 };
174
175 int get_scales_cache_size(thread_config_t const& th_config, int prob_m,
@@ -312,27 +315,28 @@ exec_config_t determine_thread_config(int prob_m, int prob_n, int prob_k,
315 return exec_config_t{0, {-1, -1, -1}};
316 }
317
315 -#define CALL_MOE_KERNEL_FUNCTION(KERNEL_FUNCTION) \
316 - else if (KERNEL_FUNCTION(q_type, thread_n_blocks, thread_k_blocks, \
317 - has_act_order, group_blocks, num_threads, blocks, \
318 - max_shared_mem, stream, A_ptr, B_ptr, C_ptr, \
319 - sorted_ids_ptr, topk_weights_ptr, s_ptr, g_idx_ptr, \
320 - expert_offsets_ptr, num_groups, expert_idx, \
321 - num_experts, topk, prob_m, prob_n, prob_k, tot_m, \
322 - locks, replicate_input, apply_weights, m_block, \
323 - max_par, exec_cfg.max_m_blocks)) { \
318 +#define CALL_MOE_KERNEL_FUNCTION(KERNEL_FUNCTION) \
319 + else if (KERNEL_FUNCTION( \
320 + q_type, thread_n_blocks, thread_k_blocks, has_act_order, \
321 + group_blocks, num_threads, blocks, max_shared_mem, stream, \
322 + A_ptr, B_ptr, C_ptr, sorted_ids_ptr, topk_weights_ptr, s_ptr, \
323 + zp_ptr, g_idx_ptr, expert_offsets_ptr, num_groups, expert_idx, \
324 + num_experts, topk, prob_m, prob_n, prob_k, tot_m, locks, \
325 + replicate_input, apply_weights, m_block, max_par, \
326 + exec_cfg.max_m_blocks)) { \
327 }
328
329 void marlin_mm_moe(const void* A, const void* B, void* C,
330 const void* sorted_ids, const void* topk_weights,
328 - const void* topk_ids, const void* s, const void* g_idx,
329 - const void* perm, void* a_tmp, void* expert_offsets,
330 - int prob_m, int prob_n, int prob_k, void* workspace,
331 - vllm::ScalarType const& q_type, bool has_act_order,
332 - bool is_k_full, int num_groups, int group_size,
333 - int num_experts, int topk, int moe_block_size, int dev,
334 - cudaStream_t stream, int thread_k, int thread_n, int sms,
335 - int max_par, bool replicate_input, bool apply_weights) {
331 + const void* topk_ids, const void* s, void* zp,
332 + const void* g_idx, const void* perm, void* a_tmp,
333 + void* expert_offsets, int prob_m, int prob_n, int prob_k,
334 + void* workspace, vllm::ScalarType const& q_type,
335 + bool has_act_order, bool is_k_full, bool has_zp,
336 + int num_groups, int group_size, int num_experts, int topk,
337 + int moe_block_size, int dev, cudaStream_t stream,
338 + int thread_k, int thread_n, int sms, int max_par,
339 + bool replicate_input, bool apply_weights) {
340 TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m,
341 ", ", prob_n, ", ", prob_k, "]");
342
@@ -436,6 +440,8 @@ void marlin_mm_moe(const void* A, const void* B, void* C,
440 const float* topk_weights_ptr = (const float*)topk_weights;
441 const int* sorted_ids_ptr = (const int*)sorted_ids;
442 const int4* s_ptr = (const int4*)s + num_groups * prob_n / 8 * expert_idx;
443 + const int4* zp_ptr =
444 + (const int4*)zp + num_groups * prob_n / (pack_factor * 4) * expert_idx;
445 const int* g_idx_ptr = (const int*)g_idx + prob_k * expert_idx;
446 const int* perm_ptr = (const int*)perm + prob_k * expert_idx;
447 int* locks = (int*)workspace;
@@ -456,6 +462,7 @@ void marlin_mm_moe(const void* A, const void* B, void* C,
462 }
463 CALL_MOE_KERNEL_FUNCTION(call_marlin_moe_kernel_ku4b8)
464 CALL_MOE_KERNEL_FUNCTION(call_marlin_moe_kernel_ku8b128)
465 + CALL_MOE_KERNEL_FUNCTION(call_marlin_moe_kernel_ku4)
466 else {
467 TORCH_CHECK(false, "Unsupported shapes: MNK = [" + str(prob_m) + ", " +
468 str(prob_n) + ", " + str(prob_k) + "]" +
@@ -475,13 +482,21 @@ torch::Tensor marlin_gemm_moe(
482 const torch::Tensor& a, const torch::Tensor& b_q_weights,
483 const torch::Tensor& sorted_ids, const torch::Tensor& topk_weights,
484 const torch::Tensor& topk_ids, const torch::Tensor& b_scales,
478 - const torch::Tensor& g_idx, const torch::Tensor& perm,
479 - torch::Tensor& workspace, vllm::ScalarTypeTorchPtr const& b_q_type,
480 - int64_t size_m, int64_t size_n, int64_t size_k, bool is_k_full,
481 - int64_t num_experts, int64_t topk, int64_t moe_block_size,
482 - bool replicate_input, bool apply_weights) {
483 - TORCH_CHECK(*b_q_type == vllm::kU4B8 || *b_q_type == vllm::kU8B128,
484 - "b_q_type must be uint4b8 or uint8b128. Got = ", b_q_type->str());
485 + torch::Tensor& b_zeros, const torch::Tensor& g_idx,
486 + const torch::Tensor& perm, torch::Tensor& workspace,
487 + vllm::ScalarTypeTorchPtr const& b_q_type, int64_t size_m, int64_t size_n,
488 + int64_t size_k, bool is_k_full, int64_t num_experts, int64_t topk,
489 + int64_t moe_block_size, bool replicate_input, bool apply_weights) {
490 + bool has_zp = b_zeros.size(1) != 0;
491 + if (has_zp) {
492 + TORCH_CHECK(
493 + *b_q_type == vllm::kU4,
494 + "b_q_type must be u4 when has_zp = True. Got = ", b_q_type->str());
495 + } else {
496 + TORCH_CHECK(
497 + *b_q_type == vllm::kU4B8 || *b_q_type == vllm::kU8B128,
498 + "b_q_type must be uint4b8 or uint8b128. Got = ", b_q_type->str());
499 + }
500
501 int pack_factor = 32 / b_q_type->size_bits();
502
@@ -543,14 +558,27 @@ torch::Tensor marlin_gemm_moe(
558 }
559 }
560
561 + // Verify b_zeros
562 + if (has_zp) {
563 + int rank = b_zeros.sizes().size();
564 + TORCH_CHECK(rank == 3, "b_zeros rank = ", rank, " is not 3");
565 + TORCH_CHECK(b_zeros.size(1) == num_groups,
566 + "b_zeros dim 1 = ", b_zeros.size(1),
567 + " is not num_groups = ", num_groups);
568 + TORCH_CHECK(b_zeros.size(2) == size_n / pack_factor,
569 + "b_zeros dim 2 = ", b_zeros.size(2),
570 + " is not size_n / pack_factor = ", size_n / pack_factor);
571 + }
572 +
573 marlin_moe::marlin_mm_moe(
574 a.data_ptr(), b_q_weights.data_ptr(), c.data_ptr(), sorted_ids.data_ptr(),
575 topk_weights.data_ptr(), topk_ids.data_ptr(), b_scales.data_ptr(),
549 - g_idx.data_ptr(), perm.data_ptr(), a_tmp.data_ptr(),
576 + b_zeros.data_ptr(), g_idx.data_ptr(), perm.data_ptr(), a_tmp.data_ptr(),
577 expert_offsets.data_ptr(), size_m, size_n, size_k, workspace.data_ptr(),
551 - *b_q_type, has_act_order, is_k_full, num_groups, group_size, num_experts,
552 - topk, moe_block_size, dev, at::cuda::getCurrentCUDAStream(dev), thread_k,
553 - thread_n, sms, max_par, replicate_input, apply_weights);
578 + *b_q_type, has_act_order, is_k_full, has_zp, num_groups, group_size,
579 + num_experts, topk, moe_block_size, dev,
580 + at::cuda::getCurrentCUDAStream(dev), thread_k, thread_n, sms, max_par,
581 + replicate_input, apply_weights);
582 return c;
583 }
584
csrc/moe/torch_bindings.cpp
+1 -1
@@ -12,7 +12,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
12 m.def(
13 "marlin_gemm_moe(Tensor! a, Tensor! b_q_weights, Tensor! sorted_ids, "
14 "Tensor! topk_weights, Tensor! topk_ids, Tensor! b_scales, Tensor! "
15 - "g_idx, Tensor! perm, Tensor! workspace, "
15 + "b_zeros, Tensor! g_idx, Tensor! perm, Tensor! workspace, "
16 "__torch__.torch.classes._core_C.ScalarType b_q_type, int size_m, "
17 "int size_n, int size_k, bool is_k_full, int num_experts, int topk, "
18 "int moe_block_size, bool replicate_input, bool apply_weights)"
csrc/quantization/gptq_marlin/gptq_marlin.cu
+1 -1
@@ -2260,7 +2260,7 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2260 "b_zeros dim 0 = ", b_zeros.size(0),
2261 " is not num_groups = ", num_groups);
2262 TORCH_CHECK(b_zeros.size(1) == size_n / pack_factor,
2263 - "b_zeros dim 1 = ", b_scales.size(1),
2263 + "b_zeros dim 1 = ", b_zeros.size(1),
2264 " is not size_n / pack_factor = ", size_n / pack_factor);
2265 }
2266
tests/kernels/test_awq_marlin.py new
+160
@@ -0,0 +1,160 @@
1 +"""Test AWQ with fused MoE Marlin kernels.
2 +
3 +Run `pytest tests/kernels/test_awq_marlin.py`.
4 +"""
5 +import pytest
6 +import torch
7 +
8 +from tests.kernels.utils import (compute_max_diff, stack_and_dev, torch_moe,
9 + torch_moe_single)
10 +from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
11 + fused_marlin_moe, single_marlin_moe)
12 +from vllm.model_executor.layers.fused_moe.fused_moe import fused_topk
13 +from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
14 + awq_marlin_quantize)
15 +from vllm.scalar_type import scalar_types
16 +
17 +
18 +@pytest.mark.parametrize("m", [64, 512, 222, 33, 1])
19 +@pytest.mark.parametrize("n", [128, 2048, 256, 1024])
20 +@pytest.mark.parametrize("k", [128, 1024, 512])
21 +@pytest.mark.parametrize("e", [8, 64])
22 +@pytest.mark.parametrize("topk", [2, 6])
23 +@pytest.mark.parametrize("group_size", [-1, 32, 64, 128])
24 +def test_fused_marlin_moe_awq(
25 + m: int,
26 + n: int,
27 + k: int,
28 + e: int,
29 + topk: int,
30 + group_size: int,
31 +):
32 + torch.manual_seed(7)
33 +
34 + num_bits = 4
35 + quant_type = scalar_types.uint4
36 + dtype = torch.float16
37 + a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
38 + w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10
39 + w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10
40 +
41 + w_ref1_l = []
42 + qweights1_l = []
43 + scales1_l = []
44 + zp1_l = []
45 +
46 + for i in range(w1.shape[0]):
47 + w_ref1, qweight1, scales1, zp1 = awq_marlin_quantize(
48 + w1[i].transpose(1, 0), quant_type, group_size)
49 + w_ref1_l.append(w_ref1)
50 + qweights1_l.append(qweight1)
51 + scales1_l.append(scales1)
52 + zp1_l.append(zp1)
53 +
54 + w_ref1 = stack_and_dev(w_ref1_l)
55 + qweight1 = stack_and_dev(qweights1_l).contiguous()
56 + scales1 = stack_and_dev(scales1_l)
57 + zp1 = stack_and_dev(zp1_l)
58 +
59 + w_ref2_l = []
60 + qweights2_l = []
61 + scales2_l = []
62 + zp2_l = []
63 +
64 + for i in range(w2.shape[0]):
65 + w_ref2, qweight2, scales2, zp2 = awq_marlin_quantize(
66 + w2[i].transpose(1, 0), quant_type, group_size)
67 + w_ref2_l.append(w_ref2)
68 + qweights2_l.append(qweight2)
69 + scales2_l.append(scales2)
70 + zp2_l.append(zp2)
71 +
72 + w_ref2 = stack_and_dev(w_ref2_l)
73 + qweight2 = stack_and_dev(qweights2_l).contiguous()
74 + scales2 = stack_and_dev(scales2_l)
75 + zp2 = stack_and_dev(zp2_l)
76 +
77 + score = torch.randn((m, e), device="cuda", dtype=dtype)
78 +
79 + topk_weights, topk_ids = fused_topk(a, score, topk, False)
80 + marlin_output = fused_marlin_moe(
81 + a,
82 + qweight1,
83 + qweight2,
84 + scales1,
85 + scales2,
86 + score,
87 + topk_weights,
88 + topk_ids,
89 + w1_zeros=zp1,
90 + w2_zeros=zp2,
91 + num_bits=num_bits,
92 + )
93 +
94 + torch_output = torch_moe(
95 + a,
96 + w_ref1.transpose(1, 2),
97 + w_ref2.transpose(1, 2),
98 + score,
99 + topk,
100 + )
101 +
102 + assert compute_max_diff(marlin_output, torch_output) < 4e-2
103 +
104 +
105 +@pytest.mark.skip("This test is here for the sake of debugging, "
106 + "don't run it in automated tests.")
107 +@pytest.mark.parametrize("m", [64, 512, 222, 33, 1])
108 +@pytest.mark.parametrize("n", [128, 2048, 256, 1024])
109 +@pytest.mark.parametrize("k", [128, 1024, 512])
110 +@pytest.mark.parametrize("e", [8, 64])
111 +@pytest.mark.parametrize("topk", [2, 6])
112 +@pytest.mark.parametrize("group_size", [-1, 32, 64, 128])
113 +def test_single_marlin_moe_multiply_awq(
114 + m: int,
115 + n: int,
116 + k: int,
117 + e: int,
118 + topk: int,
119 + group_size: int,
120 +):
121 + torch.manual_seed(7)
122 +
123 + num_bits = 4
124 + quant_type = scalar_types.uint4
125 + dtype = torch.float16
126 + a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
127 + w = torch.randn((e, n, k), device="cuda", dtype=dtype) / 10
128 +
129 + w_ref_l = []
130 + qweights_l = []
131 + scales_l = []
132 + zp_l = []
133 +
134 + for i in range(w.shape[0]):
135 + w_ref, qweight, scales, zp = awq_marlin_quantize(
136 + w[i].transpose(1, 0), quant_type, group_size)
137 + w_ref_l.append(w_ref)
138 + qweights_l.append(qweight)
139 + scales_l.append(scales)
140 + zp_l.append(zp)
141 +
142 + w_ref = stack_and_dev(w_ref_l)
143 + qweight = stack_and_dev(qweights_l).contiguous()
144 + scales = stack_and_dev(scales_l).contiguous()
145 + zp = stack_and_dev(zp_l).contiguous()
146 +
147 + score = torch.randn((m, e), device="cuda", dtype=dtype)
148 +
149 + marlin_output = single_marlin_moe(a,
150 + qweight,
151 + scales,
152 + score,
153 + topk,
154 + renormalize=False,
155 + w_zeros=zp,
156 + num_bits=num_bits)
157 +
158 + torch_output = torch_moe_single(a, w_ref.transpose(1, 2), score, topk)
159 +
160 + assert compute_max_diff(marlin_output, torch_output) < 1e-2
tests/kernels/test_moe.py
+18 -61
@@ -2,16 +2,14 @@
2
3 Run `pytest tests/kernels/test_moe.py`.
4 """
5 -from typing import List
6 -
5 import pytest
6 import torch
7 from transformers import MixtralConfig
8 from transformers.models.mixtral.modeling_mixtral import MixtralSparseMoeBlock
9
12 -from tests.kernels.utils import opcheck
10 +from tests.kernels.utils import (compute_max_diff, opcheck, stack_and_dev,
11 + torch_moe, torch_moe_single)
12 from vllm import _custom_ops as ops
14 -from vllm.model_executor.layers.activation import SiluAndMul
13 from vllm.model_executor.layers.fused_moe import fused_moe
14 from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
15 fused_marlin_moe, single_marlin_moe)
@@ -24,37 +22,6 @@ from vllm.scalar_type import scalar_types
22 from vllm.utils import seed_everything
23
24
27 -def torch_moe(a, w1, w2, score, topk):
28 - B, D = a.shape
29 - a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
30 - out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device)
31 - score = torch.softmax(score, dim=-1, dtype=torch.float32)
32 - topk_weight, topk_ids = torch.topk(score, topk)
33 - topk_weight = topk_weight.view(-1)
34 - topk_ids = topk_ids.view(-1)
35 - for i in range(w1.shape[0]):
36 - mask = topk_ids == i
37 - if mask.sum():
38 - out[mask] = SiluAndMul()(
39 - a[mask] @ w1[i].transpose(0, 1)) @ w2[i].transpose(0, 1)
40 - return (out.view(B, -1, w2.shape[1]) *
41 - topk_weight.view(B, -1, 1).to(out.dtype)).sum(dim=1)
42 -
43 -
44 -def torch_moe_single(a, w, score, topk):
45 - B, D = a.shape
46 - a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
47 - out = torch.zeros(B * topk, w.shape[1], dtype=a.dtype, device=a.device)
48 - score = torch.softmax(score, dim=-1, dtype=torch.float32)
49 - _, topk_ids = torch.topk(score, topk)
50 - topk_ids = topk_ids.view(-1)
51 - for i in range(w.shape[0]):
52 - mask = topk_ids == i
53 - if mask.sum():
54 - out[mask] = a[mask] @ w[i].transpose(0, 1)
55 - return (out.view(B, -1, w.shape[1])).sum(dim=1)
56 -
57 -
25 @pytest.mark.parametrize("m", [1024 * 128, 512, 222, 33, 1])
26 @pytest.mark.parametrize("n", [2048, 256, 1024])
27 @pytest.mark.parametrize("k", [128, 511, 1024])
@@ -127,20 +94,10 @@ def test_mixtral_moe(dtype: torch.dtype):
94 atol=mixtral_moe_tol[dtype])
95
96
130 -def stack_and_dev(tensors: List[torch.Tensor]):
131 - dev = tensors[0].device
132 - return torch.stack(tensors, dim=0).to(dev)
133 -
134 -
135 -def compute_max_diff(output, output_ref):
136 - return torch.mean(torch.abs(output - output_ref)) / torch.mean(
137 - torch.abs(output_ref))
138 -
139 -
97 @pytest.mark.parametrize("m", [64, 512, 222, 33, 1])
98 @pytest.mark.parametrize("n", [128, 2048, 256, 1024])
99 @pytest.mark.parametrize("k", [128, 1024, 512])
143 -@pytest.mark.parametrize("e", [4, 8, 64])
100 +@pytest.mark.parametrize("e", [8, 64])
101 @pytest.mark.parametrize("topk", [2, 6])
102 @pytest.mark.parametrize("group_size", [-1, 32, 64, 128])
103 @pytest.mark.parametrize("act_order", [True, False])
@@ -159,9 +116,6 @@ def test_fused_marlin_moe(
116 ):
117 seed_everything(7)
118
162 - if topk > e:
163 - return
164 -
119 # Filter act_order
120 if act_order:
121 if group_size == -1:
@@ -241,15 +195,15 @@ def test_fused_marlin_moe(
195 a,
196 qweight1,
197 qweight2,
198 + scales1,
199 + scales2,
200 score,
245 - g_idx1,
246 - g_idx2,
247 - sort_indices1,
248 - sort_indices2,
201 topk_weights,
202 topk_ids,
251 - w1_scale=scales1,
252 - w2_scale=scales2,
203 + g_idx1=g_idx1,
204 + g_idx2=g_idx2,
205 + sort_indices1=sort_indices1,
206 + sort_indices2=sort_indices2,
207 num_bits=num_bits,
208 is_k_full=is_k_full,
209 )
@@ -280,9 +234,13 @@ def test_fused_marlin_moe(
234 device="cuda",
235 requires_grad=False)
236
237 + zp = torch.empty((0, 0),
238 + dtype=dtype,
239 + device="cuda",
240 + requires_grad=False)
241 opcheck(torch.ops._moe_C.marlin_gemm_moe,
242 (a, qweight1, sorted_token_ids, topk_weights, topk_ids,
285 - scales1, g_idx1, sort_indices1, workspace, quant_type, m,
243 + scales1, zp, g_idx1, sort_indices1, workspace, quant_type, m,
244 2 * n, k, True, e, topk, block_size_m, True, False))
245
246
@@ -291,7 +249,7 @@ def test_fused_marlin_moe(
249 @pytest.mark.parametrize("m", [64, 512, 222, 33, 1])
250 @pytest.mark.parametrize("n", [128, 2048, 256, 1024])
251 @pytest.mark.parametrize("k", [128, 1024, 512])
294 -@pytest.mark.parametrize("e", [4, 8, 64])
252 +@pytest.mark.parametrize("e", [8, 64])
253 @pytest.mark.parametrize("topk", [2, 6])
254 @pytest.mark.parametrize("group_size", [-1, 32, 64, 128])
255 @pytest.mark.parametrize("act_order", [True, False])
@@ -308,8 +266,6 @@ def test_single_marlin_moe_multiply(
266 num_bits: int,
267 is_k_full: bool,
268 ):
311 - if topk > e:
312 - return
269
270 # Filter act_order
271 if act_order:
@@ -355,13 +311,14 @@ def test_single_marlin_moe_multiply(
311 qweight,
312 scales,
313 score,
358 - g_idx,
359 - sort_indices,
314 topk,
315 renormalize=False,
316 + g_idx=g_idx,
317 + sort_indices=sort_indices,
318 num_bits=num_bits,
319 is_k_full=is_k_full,
320 )
321 +
322 torch_output = torch_moe_single(a, w_ref.transpose(1, 2), score, topk)
323
324 assert compute_max_diff(marlin_output, torch_output) < 1e-2
tests/kernels/utils.py
+45
@@ -12,6 +12,7 @@ import torch
12 from torch._prims_common import TensorLikeType
13
14 from vllm.attention import AttentionBackend, AttentionMetadata, AttentionType
15 +from vllm.model_executor.layers.activation import SiluAndMul
16 from vllm.utils import (STR_BACKEND_ENV_VAR, STR_XFORMERS_ATTN_VAL,
17 make_tensor_with_pad)
18
@@ -974,6 +975,50 @@ def fp8_allclose(
975 equal_nan=equal_nan)).item())
976
977
978 +# Marlin MoE test utils
979 +
980 +
981 +def stack_and_dev(tensors: List[torch.Tensor]):
982 + dev = tensors[0].device
983 + return torch.stack(tensors, dim=0).to(dev)
984 +
985 +
986 +def compute_max_diff(output, output_ref):
987 + return torch.mean(torch.abs(output - output_ref)) / torch.mean(
988 + torch.abs(output_ref))
989 +
990 +
991 +def torch_moe(a, w1, w2, score, topk):
992 + B, D = a.shape
993 + a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
994 + out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device)
995 + score = torch.softmax(score, dim=-1, dtype=torch.float32)
996 + topk_weight, topk_ids = torch.topk(score, topk)
997 + topk_weight = topk_weight.view(-1)
998 + topk_ids = topk_ids.view(-1)
999 + for i in range(w1.shape[0]):
1000 + mask = topk_ids == i
1001 + if mask.sum():
1002 + out[mask] = SiluAndMul()(
1003 + a[mask] @ w1[i].transpose(0, 1)) @ w2[i].transpose(0, 1)
1004 + return (out.view(B, -1, w2.shape[1]) *
1005 + topk_weight.view(B, -1, 1).to(out.dtype)).sum(dim=1)
1006 +
1007 +
1008 +def torch_moe_single(a, w, score, topk):
1009 + B, D = a.shape
1010 + a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
1011 + out = torch.zeros(B * topk, w.shape[1], dtype=a.dtype, device=a.device)
1012 + score = torch.softmax(score, dim=-1, dtype=torch.float32)
1013 + _, topk_ids = torch.topk(score, topk)
1014 + topk_ids = topk_ids.view(-1)
1015 + for i in range(w.shape[0]):
1016 + mask = topk_ids == i
1017 + if mask.sum():
1018 + out[mask] = a[mask] @ w[i].transpose(0, 1)
1019 + return (out.view(B, -1, w.shape[1])).sum(dim=1)
1020 +
1021 +
1022 # A special version of op check that has a restricted default set of test_utils
1023 # and a patched version of allclose that supports fp8 types.
1024 def opcheck(op: Union[torch._ops.OpOverload, torch._ops.OpOverloadPacket,
tests/weight_loading/models-large.txt
+1
@@ -3,3 +3,4 @@ compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-channel-quantize
3 compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W8A16-quantized, main
4 compressed-tensors, mgoin/DeepSeek-Coder-V2-Lite-Instruct-FP8, main
5 gptq_marlin, TheBloke/Mixtral-8x7B-v0.1-GPTQ, main
6 +awq_marlin, casperhansen/deepseek-coder-v2-instruct-awq, main
\ No newline at end of file
tests/weight_loading/run_model_weight_loading_test.sh
+14 -1
@@ -1,7 +1,20 @@
1 #!/bin/bash
2 SUCCESS=0
3
4 -IFS=$'\n' read -d '' -r -a MODEL_CONFIGS < "weight_loading/models.txt"
4 +while getopts "c:" OPT; do
5 + case ${OPT} in
6 + c )
7 + CONFIG="$OPTARG"
8 + ;;
9 + \? )
10 + usage
11 + exit 1
12 + ;;
13 + esac
14 +done
15 +
16 +
17 +IFS=$'\n' read -d '' -r -a MODEL_CONFIGS < $CONFIG
18
19 for MODEL_CONFIG in "${MODEL_CONFIGS[@]}"
20 do
vllm/_custom_ops.py
+20 -5
@@ -568,6 +568,20 @@ def gptq_marlin_moe_repack(b_q_weight: torch.Tensor, perm: torch.Tensor,
568 return output
569
570
571 +def awq_marlin_moe_repack(b_q_weight: torch.Tensor, perm: torch.Tensor,
572 + size_k: int, size_n: int,
573 + num_bits: int) -> torch.Tensor:
574 + num_experts = b_q_weight.shape[0]
575 + assert size_k % 16 == 0
576 + output = torch.empty((num_experts, size_k // 16, size_n * (num_bits // 2)),
577 + device=b_q_weight.device,
578 + dtype=b_q_weight.dtype)
579 + for e in range(num_experts):
580 + output[e] = torch.ops._C.awq_marlin_repack(b_q_weight[e], size_k,
581 + size_n, num_bits)
582 + return output
583 +
584 +
585 def gptq_marlin_gemm(a: torch.Tensor,
586 b_q_weight: torch.Tensor,
587 b_scales: torch.Tensor,
@@ -828,11 +842,12 @@ if supports_moe_ops and hasattr(torch.ops._moe_C, "marlin_gemm_moe"):
842 sorted_ids: torch.Tensor,
843 topk_weights: torch.Tensor,
844 topk_ids: torch.Tensor, b_scales: torch.Tensor,
831 - g_idx: torch.Tensor, perm: torch.Tensor,
832 - workspace: torch.Tensor, b_q_type: ScalarType,
833 - size_m: int, size_n: int, size_k: int,
834 - is_k_full: bool, num_experts: int, topk: int,
835 - moe_block_size: int, replicate_input: bool,
845 + b_zero_points: torch.Tensor, g_idx: torch.Tensor,
846 + perm: torch.Tensor, workspace: torch.Tensor,
847 + b_q_type: ScalarType, size_m: int, size_n: int,
848 + size_k: int, is_k_full: bool, num_experts: int,
849 + topk: int, moe_block_size: int,
850 + replicate_input: bool,
851 apply_weights: bool) -> torch.Tensor:
852 return torch.empty((size_m, topk, size_n),
853 dtype=a.dtype,
vllm/model_executor/layers/fused_moe/fused_marlin_moe.py
+107 -31
@@ -10,15 +10,24 @@ from vllm.model_executor.layers.fused_moe.fused_moe import (
10 from vllm.scalar_type import scalar_types
11
12
13 +def get_scalar_type(num_bits: int, has_zp: bool):
14 + if has_zp:
15 + assert num_bits == 4
16 + return scalar_types.uint4
17 + else:
18 + return scalar_types.uint4b8 if num_bits == 4 else scalar_types.uint8b128
19 +
20 +
21 def single_marlin_moe(
22 hidden_states: torch.Tensor,
23 w: torch.Tensor,
24 scales: torch.Tensor,
25 gating_output: torch.Tensor,
18 - g_idx: torch.Tensor,
19 - perm: torch.Tensor,
26 topk: int,
27 renormalize: bool,
28 + g_idx: Optional[torch.Tensor] = None,
29 + sort_indices: Optional[torch.Tensor] = None,
30 + w_zeros: Optional[torch.Tensor] = None,
31 override_config: Optional[Dict[str, Any]] = None,
32 num_bits: int = 8,
33 is_k_full: bool = True,
@@ -34,10 +43,12 @@ def single_marlin_moe(
43 - scales (torch.Tensor): The quantization scales.
44 - gating_output (torch.Tensor): The output of the gating operation
45 (before softmax).
37 - - g_idx (torch.Tensor): The act_order indices.
38 - - perm (torch.Tensor): The act_order input permutation.
46 + - g_idx (Optional[torch.Tensor]): Optional act_order indices.
47 + - sort_indices (Optional[torch.Tensor]): Optional act_order input
48 + permutation.
49 - topk (int): The number of top-k experts to select.
50 - renormalize (bool): If True, renormalize the top-k weights to sum to 1.
51 + - w_zeros (Optional[torch.Tensor]): Optional zero points to be used for w.
52 - override_config (Optional[Dict[str, Any]]): Optional override
53 for the kernel configuration.
54 - num_bits (bool): The number of bits in expert weights quantization.
@@ -79,16 +90,34 @@ def single_marlin_moe(
90 max_workspace_size = (N // 64) * 16
91 workspace = torch.zeros(max_workspace_size,
92 dtype=torch.int,
82 - device="cuda",
93 + device=hidden_states.device,
94 + requires_grad=False)
95 +
96 + has_zero_point = w_zeros is not None
97 + if w_zeros is None:
98 + w_zeros = torch.empty((0, 0),
99 + dtype=hidden_states.dtype,
100 + device=hidden_states.device,
101 + requires_grad=False)
102 +
103 + if g_idx is None:
104 + g_idx = torch.empty((0, 0),
105 + dtype=torch.int32,
106 + device=hidden_states.device,
107 requires_grad=False)
108
85 - scalar_type = (scalar_types.uint4b8
86 - if num_bits == 4 else scalar_types.uint8b128)
109 + if sort_indices is None:
110 + sort_indices = torch.empty((0),
111 + dtype=torch.int32,
112 + device=hidden_states.device,
113 + requires_grad=False)
114 +
115 + scalar_type = get_scalar_type(num_bits, has_zero_point)
116
117 intermediate_cache = torch.ops._moe_C.marlin_gemm_moe(
118 hidden_states, w, sorted_token_ids, topk_weights, topk_ids, scales,
90 - g_idx, perm, workspace, scalar_type, M, N, K, is_k_full, E, topk,
91 - block_size_m, True, False)
119 + w_zeros, g_idx, sort_indices, workspace, scalar_type, M, N, K,
120 + is_k_full, E, topk, block_size_m, True, False)
121
122 return torch.sum(intermediate_cache.view(*intermediate_cache.shape), dim=1)
123
@@ -97,16 +126,18 @@ def fused_marlin_moe(
126 hidden_states: torch.Tensor,
127 w1: torch.Tensor,
128 w2: torch.Tensor,
129 + w1_scale: torch.Tensor,
130 + w2_scale: torch.Tensor,
131 gating_output: torch.Tensor,
101 - g_idx1: torch.Tensor,
102 - g_idx2: torch.Tensor,
103 - perm1: torch.Tensor,
104 - perm2: torch.Tensor,
132 topk_weights: torch.Tensor,
133 topk_ids: torch.Tensor,
134 + g_idx1: Optional[torch.Tensor] = None,
135 + g_idx2: Optional[torch.Tensor] = None,
136 + sort_indices1: Optional[torch.Tensor] = None,
137 + sort_indices2: Optional[torch.Tensor] = None,
138 + w1_zeros: Optional[torch.Tensor] = None,
139 + w2_zeros: Optional[torch.Tensor] = None,
140 override_config: Optional[Dict[str, Any]] = None,
108 - w1_scale: Optional[torch.Tensor] = None,
109 - w2_scale: Optional[torch.Tensor] = None,
141 num_bits: int = 8,
142 is_k_full: bool = True,
143 ) -> torch.Tensor:
@@ -118,21 +149,22 @@ def fused_marlin_moe(
149 - hidden_states (torch.Tensor): The input tensor to the MoE layer.
150 - w1 (torch.Tensor): The first set of expert weights.
151 - w2 (torch.Tensor): The second set of expert weights.
152 + - w1_scale (torch.Tensor): Scale to be used for w1.
153 + - w2_scale (torch.Tensor): Scale to be used for w2.
154 - gating_output (torch.Tensor): The output of the gating operation
155 (before softmax).
123 - - g_idx1 (torch.Tensor): The first set of act_order indices.
124 - - g_idx2 (torch.Tensor): The second set of act_order indices.
125 - - perm1 (torch.Tensor): The first act_order input permutation.
126 - - perm2 (torch.Tensor): The second act_order input permutation.
156 + - g_idx1 (Optional[torch.Tensor]): The first set of act_order indices.
157 + - g_idx2 (Optional[torch.Tensor]): The second set of act_order indices.
158 + - sort_indices1 (Optional[torch.Tensor]): The first act_order input
159 + permutation.
160 + - sort_indices2 (Optional[torch.Tensor]): The second act_order input
161 + permutation.
162 - topk_weights (torch.Tensor): Top-k weights.
163 - topk_ids (torch.Tensor): Indices of topk-k elements.
129 - - renormalize (bool): If True, renormalize the top-k weights to sum to 1.
164 - override_config (Optional[Dict[str, Any]]): Optional override
165 for the kernel configuration.
132 - - w1_scale (Optional[torch.Tensor]): Optional scale to be used for
133 - w1.
134 - - w2_scale (Optional[torch.Tensor]): Optional scale to be used for
135 - w2.
166 + - w1_zeros (Optional[torch.Tensor]): Optional zero points to be used for w1.
167 + - w2_zeros (Optional[torch.Tensor]): Optional zero points to be used for w2.
168 - num_bits (bool): The number of bits in expert weights quantization.
169
170 Returns:
@@ -152,6 +184,20 @@ def fused_marlin_moe(
184 assert hidden_states.dtype == torch.float16
185 assert num_bits in [4, 8]
186
187 + has_no_act_order = (g_idx1 is None and g_idx2 is None
188 + and sort_indices1 is None and sort_indices2 is None)
189 + has_all_act_order = (g_idx1 is not None and g_idx2 is not None
190 + and sort_indices1 is not None
191 + and sort_indices2 is not None)
192 + assert has_no_act_order or has_all_act_order, (
193 + "g_idx and sorted_indices "
194 + "must be all not None or must be all None")
195 +
196 + has_no_zp = w1_zeros is None and w2_zeros is None
197 + has_all_zp = w1_zeros is not None and w2_zeros is not None
198 + assert has_no_zp or has_all_zp, ("zero points must be both not None or "
199 + "must be both None")
200 +
201 M, K = hidden_states.shape
202 E = w1.shape[0]
203 N = w2.shape[1] * 16
@@ -172,14 +218,42 @@ def fused_marlin_moe(
218
219 sorted_token_ids, _, _ = moe_align_block_size(topk_ids, block_size_m, E)
220
175 - max_workspace_size = ((M + 255) // 256) * (max(2 * N, K) // 64) * 16
221 + max_workspace_size = (max(2 * N, K) // 64) * 16
222 workspace = torch.zeros(max_workspace_size,
223 dtype=torch.int,
224 device="cuda",
225 requires_grad=False)
226
181 - scalar_type = (scalar_types.uint4b8
182 - if num_bits == 4 else scalar_types.uint8b128)
227 + if has_no_zp:
228 + w1_zeros = torch.empty((0, 0),
229 + dtype=hidden_states.dtype,
230 + device=hidden_states.device,
231 + requires_grad=False)
232 + w2_zeros = torch.empty((0, 0),
233 + dtype=hidden_states.dtype,
234 + device=hidden_states.device,
235 + requires_grad=False)
236 +
237 + if has_no_act_order:
238 + g_idx1 = torch.empty((0, 0),
239 + dtype=torch.int32,
240 + device=hidden_states.device,
241 + requires_grad=False)
242 + g_idx2 = torch.empty((0, 0),
243 + dtype=torch.int32,
244 + device=hidden_states.device,
245 + requires_grad=False)
246 + sort_indices1 = torch.empty((0),
247 + dtype=torch.int32,
248 + device=hidden_states.device,
249 + requires_grad=False)
250 + sort_indices2 = torch.empty((0, 0),
251 + dtype=torch.int32,
252 + device=hidden_states.device,
253 + requires_grad=False)
254 +
255 + scalar_type1 = get_scalar_type(num_bits, has_all_zp)
256 + scalar_type2 = get_scalar_type(num_bits, has_all_zp)
257
258 intermediate_cache2 = torch.empty(
259 (M * topk_ids.shape[1], N),
@@ -194,10 +268,11 @@ def fused_marlin_moe(
268 topk_weights,
269 topk_ids,
270 w1_scale,
271 + w1_zeros,
272 g_idx1,
198 - perm1,
273 + sort_indices1,
274 workspace,
200 - scalar_type,
275 + scalar_type1,
276 M,
277 2 * N,
278 K,
@@ -218,10 +293,11 @@ def fused_marlin_moe(
293 topk_weights,
294 topk_ids,
295 w2_scale,
296 + w2_zeros,
297 g_idx2,
222 - perm2,
298 + sort_indices2,
299 workspace,
224 - scalar_type,
300 + scalar_type2,
301 M,
302 K,
303 N,
vllm/model_executor/layers/quantization/awq_marlin.py
+195 -9
@@ -1,16 +1,21 @@
1 -from typing import Any, Dict, List, Optional
1 +from typing import Any, Callable, Dict, List, Optional
2
3 import torch
4 +from torch.nn import Parameter
5
6 from vllm import _custom_ops as ops
7 from vllm.logger import init_logger
7 -from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
8 +from vllm.model_executor.layers.fused_moe.layer import (
9 + FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported)
10 +from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
11 + set_weight_attrs)
12 from vllm.model_executor.layers.quantization.base_config import (
9 - QuantizationConfig)
13 + QuantizationConfig, QuantizeMethodBase)
14 from vllm.model_executor.layers.quantization.utils import replace_parameter
15 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
16 apply_awq_marlin_linear, awq_to_marlin_zero_points, check_marlin_supported,
13 - marlin_make_empty_g_idx, marlin_make_workspace, marlin_permute_scales,
17 + marlin_make_empty_g_idx, marlin_make_workspace, marlin_moe_permute_scales,
18 + marlin_permute_scales, moe_awq_to_marlin_zero_points,
19 verify_marlin_supported, verify_marlin_supports_shape)
20 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
21 from vllm.model_executor.parameter import (GroupQuantScaleParameter,
@@ -35,12 +40,13 @@ class AWQMarlinConfig(QuantizationConfig):
40 self.group_size = group_size
41 self.has_zp = has_zp
42 self.lm_head_quantized = lm_head_quantized
43 + self.weight_bits = weight_bits
44
39 - if weight_bits not in self.TYPE_MAP:
40 - raise ValueError(f"Unsupported num_bits = {weight_bits}. "
45 + if self.weight_bits not in self.TYPE_MAP:
46 + raise ValueError(f"Unsupported num_bits = {self.weight_bits}. "
47 f"Supported num_bits = {self.TYPE_MAP.keys()}")
48
43 - self.quant_type = self.TYPE_MAP[weight_bits]
49 + self.quant_type = self.TYPE_MAP[self.weight_bits]
50
51 verify_marlin_supported(self.quant_type,
52 group_size=self.group_size,
@@ -98,10 +104,12 @@ class AWQMarlinConfig(QuantizationConfig):
104 return None
105
106 def get_quant_method(self, layer: torch.nn.Module,
101 - prefix: str) -> Optional["AWQMarlinLinearMethod"]:
107 + prefix: str) -> Optional["QuantizeMethodBase"]:
108 if (isinstance(layer, LinearBase) or
109 (isinstance(layer, ParallelLMHead) and self.lm_head_quantized)):
110 return AWQMarlinLinearMethod(self)
111 + elif isinstance(layer, FusedMoE):
112 + return AWQMoEMethod(self)
113 return None
114
115 def get_scaled_act_names(self) -> List[str]:
@@ -271,4 +279,182 @@ class AWQMarlinLinearMethod(LinearMethodBase):
279 quant_type=self.quant_config.quant_type,
280 output_size_per_partition=layer.output_size_per_partition,
281 input_size_per_partition=layer.input_size_per_partition,
274 - bias=bias)
\ No newline at end of file
282 + bias=bias)
283 +
284 +
285 +class AWQMoEMethod(FusedMoEMethodBase):
286 +
287 + def __init__(self, quant_config: AWQMarlinConfig):
288 + self.quant_config = quant_config
289 +
290 + def create_weights(self, layer: torch.nn.Module, num_experts: int,
291 + hidden_size: int, intermediate_size: int,
292 + params_dtype: torch.dtype, **extra_weight_attrs):
293 + extra_weight_attrs.update({
294 + "is_transposed":
295 + True,
296 + "quant_method":
297 + FusedMoeWeightScaleSupported.GROUP.value,
298 + })
299 +
300 + w13_qweight = Parameter(torch.empty(num_experts,
301 + hidden_size,
302 + 2 * intermediate_size //
303 + self.quant_config.pack_factor,
304 + dtype=torch.int32),
305 + requires_grad=False)
306 + layer.register_parameter("w13_qweight", w13_qweight)
307 + set_weight_attrs(w13_qweight, extra_weight_attrs)
308 +
309 + w2_qweight = Parameter(torch.empty(num_experts,
310 + intermediate_size,
311 + hidden_size //
312 + self.quant_config.pack_factor,
313 + dtype=torch.int32),
314 + requires_grad=False)
315 + layer.register_parameter("w2_qweight", w2_qweight)
316 + set_weight_attrs(w2_qweight, extra_weight_attrs)
317 +
318 + num_groups_w13 = hidden_size // self.quant_config.group_size
319 + num_groups_w2 = intermediate_size // self.quant_config.group_size
320 +
321 + # WEIGHT_SCALES
322 + # Allocate 2 scales for w1 and w3 respectively.
323 + w13_scales = Parameter(torch.empty(num_experts,
324 + num_groups_w13,
325 + intermediate_size * 2,
326 + dtype=params_dtype),
327 + requires_grad=False)
328 + layer.register_parameter("w13_scales", w13_scales)
329 + set_weight_attrs(w13_scales, extra_weight_attrs)
330 +
331 + w2_scales = Parameter(torch.empty(num_experts,
332 + num_groups_w2,
333 + hidden_size,
334 + dtype=params_dtype),
335 + requires_grad=False)
336 + layer.register_parameter("w2_scales", w2_scales)
337 + set_weight_attrs(w2_scales, extra_weight_attrs)
338 +
339 + # WEIGHT_ZERO_POINT
340 + # Allocate 2 zero points for w1 and w3 respectively.
341 + w13_qzeros = Parameter(torch.empty(num_experts,
342 + num_groups_w13,
343 + 2 * intermediate_size //
344 + self.quant_config.pack_factor,
345 + dtype=torch.int32),
346 + requires_grad=False)
347 + layer.register_parameter("w13_qzeros", w13_qzeros)
348 + set_weight_attrs(w13_qzeros, extra_weight_attrs)
349 +
350 + w2_qzeros = Parameter(torch.empty(num_experts,
351 + num_groups_w2,
352 + hidden_size //
353 + self.quant_config.pack_factor,
354 + dtype=torch.int32),
355 + requires_grad=False)
356 + layer.register_parameter("w2_qzeros", w2_qzeros)
357 + set_weight_attrs(w2_qzeros, extra_weight_attrs)
358 +
359 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
360 + num_experts = layer.w13_qweight.shape[0]
361 + device = layer.w13_qweight.device
362 +
363 + layer.w13_g_idx_sort_indices = torch.nn.Parameter(
364 + torch.empty((num_experts, 0), dtype=torch.int32, device=device),
365 + requires_grad=False,
366 + )
367 + layer.w2_g_idx_sort_indices = torch.nn.Parameter(
368 + torch.empty((num_experts, 0), dtype=torch.int32, device=device),
369 + requires_grad=False,
370 + )
371 +
372 + marlin_w13_qweight = ops.awq_marlin_moe_repack(
373 + layer.w13_qweight,
374 + layer.w13_g_idx_sort_indices,
375 + size_k=layer.w13_qweight.shape[1],
376 + size_n=layer.w13_qweight.shape[2] * self.quant_config.pack_factor,
377 + num_bits=self.quant_config.weight_bits,
378 + )
379 + replace_parameter(layer, "w13_qweight", marlin_w13_qweight)
380 +
381 + marlin_w2_qweight = ops.awq_marlin_moe_repack(
382 + layer.w2_qweight,
383 + layer.w2_g_idx_sort_indices,
384 + size_k=layer.w2_qweight.shape[1],
385 + size_n=layer.w2_qweight.shape[2] * self.quant_config.pack_factor,
386 + num_bits=self.quant_config.weight_bits,
387 + )
388 + replace_parameter(layer, "w2_qweight", marlin_w2_qweight)
389 +
390 + # Why does this take the intermediate size for size_k?
391 + marlin_w13_scales = marlin_moe_permute_scales(
392 + s=layer.w13_scales,
393 + size_k=layer.intermediate_size_per_partition,
394 + size_n=layer.w13_scales.shape[2],
395 + group_size=self.quant_config.group_size,
396 + )
397 +
398 + replace_parameter(layer, "w13_scales", marlin_w13_scales)
399 +
400 + marlin_w2_scales = marlin_moe_permute_scales(
401 + s=layer.w2_scales,
402 + size_k=layer.intermediate_size_per_partition,
403 + size_n=layer.w2_scales.shape[2],
404 + group_size=self.quant_config.group_size,
405 + )
406 + replace_parameter(layer, "w2_scales", marlin_w2_scales)
407 +
408 + marlin_w13_zp = moe_awq_to_marlin_zero_points(
409 + layer.w13_qzeros,
410 + size_k=layer.w13_qzeros.shape[1],
411 + size_n=layer.w13_qzeros.shape[2] * self.quant_config.pack_factor,
412 + num_bits=self.quant_config.weight_bits)
413 + replace_parameter(layer, "w13_qzeros", marlin_w13_zp)
414 +
415 + marlin_w2_zp = moe_awq_to_marlin_zero_points(
416 + layer.w2_qzeros,
417 + size_k=layer.w2_qzeros.shape[1],
418 + size_n=layer.w2_qzeros.shape[2] * self.quant_config.pack_factor,
419 + num_bits=self.quant_config.weight_bits)
420 + replace_parameter(layer, "w2_qzeros", marlin_w2_zp)
421 +
422 + def apply(
423 + self,
424 + layer: torch.nn.Module,
425 + x: torch.Tensor,
426 + router_logits: torch.Tensor,
427 + top_k: int,
428 + renormalize: bool = True,
429 + use_grouped_topk: bool = False,
430 + num_expert_group: Optional[int] = None,
431 + topk_group: Optional[int] = None,
432 + custom_routing_function: Optional[Callable] = None,
433 + ) -> torch.Tensor:
434 +
435 + from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
436 + fused_marlin_moe)
437 +
438 + topk_weights, topk_ids = FusedMoE.select_experts(
439 + hidden_states=x,
440 + router_logits=router_logits,
441 + use_grouped_topk=use_grouped_topk,
442 + top_k=top_k,
443 + renormalize=renormalize,
444 + topk_group=topk_group,
445 + num_expert_group=num_expert_group,
446 + custom_routing_function=custom_routing_function)
447 +
448 + return fused_marlin_moe(
449 + x,
450 + layer.w13_qweight,
451 + layer.w2_qweight,
452 + layer.w13_scales,
453 + layer.w2_scales,
454 + router_logits,
455 + topk_weights,
456 + topk_ids,
457 + w1_zeros=layer.w13_qzeros,
458 + w2_zeros=layer.w2_qzeros,
459 + num_bits=self.quant_config.weight_bits,
460 + )
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+6 -6
@@ -498,14 +498,14 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
498 x,
499 layer.w13_weight_packed,
500 layer.w2_weight_packed,
501 + layer.w13_weight_scale,
502 + layer.w2_weight_scale,
503 router_logits,
502 - layer.w13_g_idx,
503 - layer.w2_g_idx,
504 - layer.w13_g_idx_sort_indices,
505 - layer.w2_g_idx_sort_indices,
504 topk_weights,
505 topk_ids,
508 - w1_scale=layer.w13_weight_scale,
509 - w2_scale=layer.w2_weight_scale,
506 + g_idx1=layer.w13_g_idx,
507 + g_idx2=layer.w2_g_idx,
508 + sort_indices1=layer.w13_g_idx_sort_indices,
509 + sort_indices2=layer.w2_g_idx_sort_indices,
510 num_bits=self.num_bits,
511 )
vllm/model_executor/layers/quantization/gptq_marlin.py
+6 -6
@@ -557,14 +557,14 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
557 x,
558 layer.w13_qweight,
559 layer.w2_qweight,
560 + layer.w13_scales,
561 + layer.w2_scales,
562 router_logits,
561 - layer.w13_g_idx,
562 - layer.w2_g_idx,
563 - layer.w13_g_idx_sort_indices,
564 - layer.w2_g_idx_sort_indices,
563 topk_weights,
564 topk_ids,
567 - w1_scale=layer.w13_scales,
568 - w2_scale=layer.w2_scales,
565 + g_idx1=layer.w13_g_idx,
566 + g_idx2=layer.w2_g_idx,
567 + sort_indices1=layer.w13_g_idx_sort_indices,
568 + sort_indices2=layer.w2_g_idx_sort_indices,
569 num_bits=self.quant_config.quant_type.size_bits,
570 ).to(orig_dtype)
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+15
@@ -208,6 +208,7 @@ def marlin_moe_permute_scales(
208 device=s.device,
209 dtype=s.dtype,
210 )
211 +
212 for e in range(num_experts):
213 output[e] = marlin_permute_scales(s[e], size_k, size_n, group_size)
214 return output
@@ -258,6 +259,20 @@ def awq_to_marlin_zero_points(q_zp_packed: torch.Tensor, size_k: int,
259 return marlin_zp
260
261
262 +def moe_awq_to_marlin_zero_points(q_zp_packed: torch.Tensor, size_k: int,
263 + size_n: int, num_bits: int):
264 + num_experts = q_zp_packed.shape[0]
265 + output = torch.empty(
266 + (num_experts, q_zp_packed.shape[1], q_zp_packed.shape[2]),
267 + device=q_zp_packed.device,
268 + dtype=q_zp_packed.dtype,
269 + )
270 + for e in range(num_experts):
271 + output[e] = awq_to_marlin_zero_points(q_zp_packed[e], size_k, size_n,
272 + num_bits)
273 + return output
274 +
275 +
276 def apply_gptq_marlin_linear(
277 input: torch.Tensor,
278 weight: torch.Tensor,
vllm/model_executor/model_loader/utils.py
+3 -1
@@ -23,7 +23,9 @@ def get_model_architecture(
23 architectures = getattr(model_config.hf_config, "architectures", [])
24 # Special handling for quantized Mixtral.
25 # FIXME(woosuk): This is a temporary hack.
26 - mixtral_supported = ["fp8", "compressed-tensors", "gptq_marlin"]
26 + mixtral_supported = [
27 + "fp8", "compressed-tensors", "gptq_marlin", "awq_marlin"
28 + ]
29
30 if (model_config.quantization is not None
31 and model_config.quantization not in mixtral_supported