@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a091e2da3e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Kernel] Enable 8-bit weights in Fused Marlin MoE (#8032)

Co-authored-by: Dipika <dipikasikka1@gmail.com>

ElizaWszola committed Sep 16, 2024 at 17:47 UTC a091e2da3e3fcb4c63c8206839d7240a2a2a176a
12 files changed +453 -185
csrc/moe/marlin_moe_ops.cu
+389 -148
@@ -25,6 +25,8 @@
25
26 #include <iostream>
27
28 +#include "core/scalar_type.hpp"
29 +
30 template <typename T>
31 inline std::string str(T x) {
32 return std::to_string(x);
@@ -131,11 +133,26 @@ __device__ inline int lop3(int a, int b, int c) {
133 return res;
134 }
135
134 -// Efficiently dequantize an int32 value into a full B-fragment of 4 fp16
135 -// values. We mostly follow the strategy in the link below, with some small
136 -// changes:
137 -// https://github.com/NVIDIA/FasterTransformer/blob/main/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h
138 -__device__ inline FragB dequant(int q) {
136 +// Constructs destination register by taking bytes from 2 sources (based on
137 +// mask)
138 +template <int start_byte, int mask>
139 +__device__ inline uint32_t prmt(uint32_t a) {
140 + uint32_t res;
141 + asm volatile("prmt.b32 %0, %1, %2, %3;\n"
142 + : "=r"(res)
143 + : "r"(a), "n"(start_byte), "n"(mask));
144 + return res;
145 +}
146 +
147 +template <vllm::ScalarTypeId w_type_id>
148 +__device__ inline FragB dequant(int q);
149 +
150 +// Efficiently dequantize 4bit values packed in an int32 value into a full
151 +// B-fragment of 4 fp16 values. We mostly follow the strategy in the link below,
152 +// with some small changes:
153 +// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L215-L287
154 +template <>
155 +__device__ inline FragB dequant<vllm::kU4B8.id()>(int q) {
156 const int LO = 0x000f000f;
157 const int HI = 0x00f000f0;
158 const int EX = 0x64006400;
@@ -156,6 +173,28 @@ __device__ inline FragB dequant(int q) {
173 return frag_b;
174 }
175
176 +// Fast Int8ToFp16: Efficiently dequantize 8bit int values to fp16
177 +// Reference:
178 +// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L53-L85
179 +template <>
180 +__device__ inline FragB dequant<vllm::kU8B128.id()>(int q) {
181 + static constexpr uint32_t mask_for_elt_01 = 0x5250;
182 + static constexpr uint32_t mask_for_elt_23 = 0x5351;
183 + static constexpr uint32_t start_byte_for_fp16 = 0x64646464;
184 +
185 + uint32_t lo = prmt<start_byte_for_fp16, mask_for_elt_01>(q);
186 + uint32_t hi = prmt<start_byte_for_fp16, mask_for_elt_23>(q);
187 +
188 + static constexpr uint32_t I8s_TO_F16s_MAGIC_NUM = 0x64806480;
189 +
190 + FragB frag_b;
191 + frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
192 + *reinterpret_cast<const half2*>(&I8s_TO_F16s_MAGIC_NUM));
193 + frag_b[1] = __hsub2(*reinterpret_cast<half2*>(&hi),
194 + *reinterpret_cast<const half2*>(&I8s_TO_F16s_MAGIC_NUM));
195 + return frag_b;
196 +}
197 +
198 // Multiply dequantized values by the corresponding quantization scale; used
199 // only for grouped quantization.
200 __device__ inline void scale(FragB& frag_b, FragS& frag_s, int i) {
@@ -296,7 +335,8 @@ __global__ void compute_expert_offsets(int const* __restrict__ topk_ids,
335 __syncthreads();
336 }
337
299 -template <const int threads, // number of threads in a threadblock
338 +template <const vllm::ScalarTypeId w_type_id, // weight ScalarType id
339 + const int threads, // number of threads in a threadblock
340 const int thread_m_blocks, // number of 16x16 blocks in the m
341 // dimension (batchsize) of the
342 // threadblock
@@ -331,6 +371,9 @@ __device__ inline void MarlinMoESingle(
371 bool apply_weights, // apply weights to output
372 int current_m_block // current m block to start kernel computation from
373 ) {
374 + static constexpr auto w_type = vllm::ScalarType::from_id(w_type_id);
375 + constexpr int pack_factor = 32 / w_type.size_bits();
376 +
377 // For larger GEMMs we run multiple batchsize 64 versions in parallel for a
378 // better partitioning with less reductions
379 int parallel = 1;
@@ -423,21 +466,25 @@ __device__ inline void MarlinMoESingle(
466 constexpr int a_sh_wr_iters = ceildiv(a_sh_stage, a_sh_wr_delta);
467
468 // B sizes/strides
426 - int b_gl_stride = 16 * prob_n / 32;
427 - constexpr int b_sh_stride = 32 * thread_n_blocks / 4;
469 + int b_gl_stride = 16 * prob_n / (pack_factor * 4);
470 + constexpr int b_sh_stride = ((thread_n_blocks * 16) * 16 / pack_factor) / 4;
471 + constexpr int b_thread_vecs = w_type.size_bits() == 4 ? 1 : 2;
472 + constexpr int b_sh_stride_threads = b_sh_stride / b_thread_vecs;
473 +
474 int b_gl_rd_delta_o = b_gl_stride * thread_k_blocks;
429 - int b_gl_rd_delta_i = b_gl_stride * (threads / b_sh_stride);
430 - constexpr int b_sh_wr_delta = threads;
431 - constexpr int b_sh_rd_delta = threads;
475 + int b_gl_rd_delta_i = b_gl_stride * (threads / b_sh_stride_threads);
476 + constexpr int b_sh_wr_delta = threads * b_thread_vecs;
477 + constexpr int b_sh_rd_delta = threads * b_thread_vecs;
478 constexpr int b_sh_stage = b_sh_stride * thread_k_blocks;
479 constexpr int b_sh_wr_iters = b_sh_stage / b_sh_wr_delta;
480
481 // Scale sizes/strides without act_order
482 int s_gl_stride = prob_n / 8;
483 constexpr int s_sh_stride = 16 * thread_n_blocks / 8;
438 - constexpr int s_tb_groups = !has_act_order && group_blocks < thread_k_blocks
439 - ? thread_k_blocks / group_blocks
440 - : 1;
484 + constexpr int s_tb_groups =
485 + !has_act_order && group_blocks != -1 && group_blocks < thread_k_blocks
486 + ? thread_k_blocks / group_blocks
487 + : 1;
488 constexpr int s_sh_stage = s_tb_groups * s_sh_stride;
489 int s_gl_rd_delta = s_gl_stride;
490 // Scale size/strides with act_order
@@ -465,12 +512,12 @@ __device__ inline void MarlinMoESingle(
512 a_sh_stride * ((threadIdx.x % 32) % 16) + (threadIdx.x % 32) / 16;
513 a_sh_rd += 2 * ((threadIdx.x / 32) / (thread_n_blocks / 4));
514
468 - int b_gl_rd =
469 - b_gl_stride * (threadIdx.x / b_sh_stride) + (threadIdx.x % b_sh_stride);
515 + int b_gl_rd = b_gl_stride * (threadIdx.x / b_sh_stride_threads) +
516 + (threadIdx.x % b_sh_stride_threads) * b_thread_vecs;
517 b_gl_rd += b_sh_stride * slice_col;
518 b_gl_rd += b_gl_rd_delta_o * slice_row;
472 - int b_sh_wr = threadIdx.x;
473 - int b_sh_rd = threadIdx.x;
519 + int b_sh_wr = threadIdx.x * b_thread_vecs;
520 + int b_sh_rd = threadIdx.x * b_thread_vecs;
521
522 // For act_order
523 constexpr int k_iter_size = tb_k / b_sh_wr_iters;
@@ -481,11 +528,13 @@ __device__ inline void MarlinMoESingle(
528
529 // No act_order
530 int s_gl_rd;
484 - if constexpr (group_blocks == -1 || group_blocks == 0) {
485 - s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
486 - } else {
487 - s_gl_rd = s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
488 - s_sh_stride * slice_col + threadIdx.x;
531 + if constexpr (!has_act_order) {
532 + if constexpr (group_blocks == -1) {
533 + s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
534 + } else {
535 + s_gl_rd = s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
536 + s_sh_stride * slice_col + threadIdx.x;
537 + }
538 }
539 int s_sh_wr = threadIdx.x;
540 bool s_sh_wr_pred = threadIdx.x < s_sh_stride;
@@ -571,7 +620,7 @@ __device__ inline void MarlinMoESingle(
620
621 // Register storage for double buffer of shared memory reads.
622 FragA frag_a[2][thread_m_blocks];
574 - I4 frag_b_quant[2];
623 + I4 frag_b_quant[2][b_thread_vecs];
624 FragC frag_c[thread_m_blocks][4][2];
625 FragS frag_s[2][4]; // No act-order
626 FragS act_frag_s[2][4][4]; // For act-order
@@ -637,7 +686,10 @@ __device__ inline void MarlinMoESingle(
686 int4* sh_b_stage = sh_b + b_sh_stage * pipe;
687 #pragma unroll
688 for (int i = 0; i < b_sh_wr_iters; i++) {
640 - cp_async4(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr], B_ptr[i]);
689 + #pragma unroll
690 + for (int j = 0; j < b_thread_vecs; j++) {
691 + cp_async4(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr + j], B_ptr[i] + j);
692 + }
693 B_ptr[i] += b_gl_rd_delta_o;
694 }
695
@@ -715,14 +767,24 @@ __device__ inline void MarlinMoESingle(
767 for (int i = 0; i < thread_m_blocks; i++)
768 ldsm4(frag_a[k % 2][i], &sh_a_stage[a_sh_rd_trans[k % b_sh_wr_iters][i]]);
769 int4* sh_b_stage = sh_b + b_sh_stage * pipe;
718 - frag_b_quant[k % 2] = *reinterpret_cast<I4*>(
719 - &sh_b_stage[b_sh_rd_delta * (k % b_sh_wr_iters) + b_sh_rd]);
770 +
771 + #pragma unroll
772 + for (int i = 0; i < b_thread_vecs; i++) {
773 + frag_b_quant[k % 2][i] = *reinterpret_cast<I4*>(
774 + &sh_b_stage[b_sh_rd_delta * (k % b_sh_wr_iters) + b_sh_rd + i]);
775 + }
776 };
777
778 bool is_same_group[stages];
779 int same_group_id[stages];
780
781 auto init_same_group = [&](int pipe) {
782 + if constexpr (!has_act_order) {
783 + is_same_group[pipe] = false;
784 + same_group_id[pipe] = 0;
785 + return;
786 + }
787 +
788 int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
789 int* sh_g_idx_int_ptr = reinterpret_cast<int*>(sh_g_idx_stage);
790
@@ -840,10 +902,19 @@ __device__ inline void MarlinMoESingle(
902 // dequantization and matmul operations.
903 #pragma unroll
904 for (int j = 0; j < 4; j++) {
843 - int b_quant = frag_b_quant[k % 2][j];
844 - int b_quant_shift = b_quant >> 8;
905 + int b_quant_0, b_quant_1;
906 + if constexpr (w_type.size_bits() == 4) {
907 + b_quant_0 = frag_b_quant[k % 2][0][j];
908 + b_quant_1 = b_quant_0 >> 8;
909 + } else {
910 + static_assert(w_type.size_bits() == 8);
911 + int* frag_b_quant_ptr = reinterpret_cast<int*>(frag_b_quant[k % 2]);
912 + b_quant_0 = frag_b_quant_ptr[j * 2 + 0];
913 + b_quant_1 = frag_b_quant_ptr[j * 2 + 1];
914 + }
915
846 - FragB frag_b0 = dequant(b_quant);
916 + FragB frag_b0 = dequant<w_type_id>(b_quant_0);
917 + FragB frag_b1 = dequant<w_type_id>(b_quant_1);
918
919 // Apply scale to frag_b0
920 if constexpr (has_act_order) {
@@ -855,8 +926,6 @@ __device__ inline void MarlinMoESingle(
926 }
927 }
928
858 - FragB frag_b1 = dequant(b_quant_shift);
859 -
929 // Apply scale to frag_b1
930 if constexpr (has_act_order) {
931 scale4(frag_b1, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
@@ -881,13 +950,13 @@ __device__ inline void MarlinMoESingle(
950 // multiple warps that accumulate their partial sums of the same output
951 // location; which we have to reduce over in the end. We do in shared memory.
952 auto thread_block_reduce = [&]() {
884 - constexpr int red_off = threads / b_sh_stride / 2;
953 + constexpr int red_off = threads / b_sh_stride_threads / 2;
954 if (red_off >= 1) {
886 - int red_idx = threadIdx.x / b_sh_stride;
887 - constexpr int red_sh_stride = b_sh_stride * 4 * 2;
888 - constexpr int red_sh_delta = b_sh_stride;
889 - int red_sh_rd = red_sh_stride * (threadIdx.x / b_sh_stride) +
890 - (threadIdx.x % b_sh_stride);
955 + int red_idx = threadIdx.x / b_sh_stride_threads;
956 + constexpr int red_sh_stride = b_sh_stride_threads * 4 * 2;
957 + constexpr int red_sh_delta = b_sh_stride_threads;
958 + int red_sh_rd = red_sh_stride * (threadIdx.x / b_sh_stride_threads) +
959 + (threadIdx.x % b_sh_stride_threads);
960
961 // Parallel logarithmic shared memory reduction. We make sure to avoid any
962 // unnecessary read or write iterations, e.g., for two warps we write only
@@ -1035,8 +1104,10 @@ __device__ inline void MarlinMoESingle(
1104 auto write = [&](int idx, float c0, float c1, FragS& s) {
1105 half2 res = __halves2half2(__float2half(c0), __float2half(c1));
1106
1038 - // For per-column quantization we finally apply the scale here
1039 - if constexpr (!has_act_order && group_blocks == -1) {
1107 + // For per-column quantization we finally apply the scale here (only for
1108 + // 4-bit)
1109 + if constexpr (!has_act_order && group_blocks == -1 &&
1110 + w_type.size_bits() == 4) {
1111 res = __hmul2(res, s[0]);
1112 }
1113
@@ -1088,9 +1159,9 @@ __device__ inline void MarlinMoESingle(
1159
1160 // Start global fetch and register load pipelines.
1161 auto start_pipes = [&]() {
1091 - // TODO re-enable after fixing this function
1092 - // fetch_sorted_ids_to_shared();
1093 - __syncthreads();
1162 + // TODO re-enable after fixing this function
1163 + // fetch_sorted_ids_to_shared();
1164 + // __syncthreads();
1165
1166 #pragma unroll
1167 for (int i = 0; i < stages - 1; i++) {
@@ -1166,28 +1237,70 @@ __device__ inline void MarlinMoESingle(
1237 if (slice_iters == 0) {
1238 cp_async_wait<0>();
1239 bool last = slice_idx == slice_count - 1;
1169 - // For per-column scales, we only fetch them here in the final step before
1170 - // write-out
1240 if constexpr (!has_act_order && group_blocks == -1) {
1172 - if (last) {
1241 + if constexpr (w_type.size_bits() == 8) {
1242 if (s_sh_wr_pred) {
1243 cp_async4(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]);
1244 }
1245 cp_async_fence();
1246 + } else {
1247 + // For 4-bit per-column scales, we only fetch them here in the
1248 + // final step before write-out
1249 + if (last) {
1250 + if (s_sh_wr_pred) {
1251 + cp_async4(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]);
1252 + }
1253 + cp_async_fence();
1254 + }
1255 }
1256 }
1257
1258 thread_block_reduce();
1259 if constexpr (!has_act_order && group_blocks == -1) {
1182 - if (last) {
1260 + if constexpr (w_type.size_bits() == 8) {
1261 cp_async_wait<0>();
1262 __syncthreads();
1263 if (threadIdx.x / 32 < thread_n_blocks / 4) {
1264 reinterpret_cast<int4*>(&frag_s)[0] = sh_s[s_sh_rd + 0];
1265 reinterpret_cast<int4*>(&frag_s)[1] = sh_s[s_sh_rd + 4];
1266 }
1267 +
1268 + } else {
1269 + if (last) {
1270 + cp_async_wait<0>();
1271 + __syncthreads();
1272 + if (threadIdx.x / 32 < thread_n_blocks / 4) {
1273 + reinterpret_cast<int4*>(&frag_s)[0] = sh_s[s_sh_rd + 0];
1274 + reinterpret_cast<int4*>(&frag_s)[1] = sh_s[s_sh_rd + 4];
1275 + }
1276 + }
1277 }
1278 }
1279 +
1280 + // For 8-bit channelwise, we apply the scale before the global reduction
1281 + // that converts the fp32 results to fp16 (so that we avoid possible
1282 + // overflow in fp16)
1283 + if constexpr (!has_act_order && group_blocks == -1 &&
1284 + w_type.size_bits() == 8) {
1285 + if (threadIdx.x / 32 < thread_n_blocks / 4) {
1286 + #pragma unroll
1287 + for (int i = 0; i < thread_m_blocks; i++) {
1288 + #pragma unroll
1289 + for (int j = 0; j < 4; j++) {
1290 + scale_float(reinterpret_cast<float*>(&frag_c[i][j][0][0]),
1291 + frag_s[j / 2][2 * (j % 2) + 0]);
1292 + scale_float(reinterpret_cast<float*>(&frag_c[i][j][0][2]),
1293 + frag_s[j / 2][2 * (j % 2) + 0]);
1294 +
1295 + scale_float(reinterpret_cast<float*>(&frag_c[i][j][1][0]),
1296 + frag_s[j / 2][2 * (j % 2) + 1]);
1297 + scale_float(reinterpret_cast<float*>(&frag_c[i][j][1][2]),
1298 + frag_s[j / 2][2 * (j % 2) + 1]);
1299 + }
1300 + }
1301 + }
1302 + }
1303 +
1304 if (slice_count > 1) { // only globally reduce if there is more than one
1305 // block in a slice
1306 barrier_acquire(&locks[slice_col], slice_idx);
@@ -1227,7 +1340,8 @@ __device__ inline void MarlinMoESingle(
1340 }
1341 }
1342
1230 -template <const int threads, // number of threads in a threadblock
1343 +template <const vllm::ScalarTypeId w_type_id, // weight ScalarType id
1344 + const int threads, // number of threads in a threadblock
1345 const int thread_m_blocks, // number of 16x16 blocks in the m
1346 // dimension (batchsize) of the
1347 // threadblock
@@ -1261,7 +1375,8 @@ __global__ void MarlinMoE(
1375 bool replicate_input, // do we use the same input for each expert?
1376 bool apply_weights, // apply weights to output
1377 int current_m_block, // current m block to start kernel computation from
1264 - int max_par // maximum parallelism
1378 + int max_par, // maximum parallelism
1379 + int cfg_max_m_blocks // upper bound on m blocks
1380 ) {
1381 int m_block_ctr = current_m_block;
1382
@@ -1282,40 +1397,40 @@ __global__ void MarlinMoE(
1397 prob_m = tot_its - 16 * m_block_ctr;
1398
1399 int par = 1;
1285 - if (max_block > 4) {
1400 + if (max_block > cfg_max_m_blocks) {
1401 // Note that parallel > 1 currently only works for inputs without any
1402 // padding
1288 - par = (16 * max_block - pad) / 64;
1289 - par = min((16 * max_block - pad) / 64, max_par);
1290 - prob_m = 64 * par;
1291 - m_block_ctr += 4 * (par - 1);
1292 - max_block = 4;
1403 + par = (16 * max_block - pad) / (16 * cfg_max_m_blocks);
1404 + if (par > max_par) par = max_par;
1405 + prob_m = (16 * cfg_max_m_blocks) * par;
1406 + m_block_ctr += cfg_max_m_blocks * (par - 1);
1407 + max_block = cfg_max_m_blocks;
1408 }
1409
1410 if (max_block == 1) {
1296 - MarlinMoESingle<threads, 1, thread_n_blocks, thread_k_blocks, stages,
1297 - has_act_order, group_blocks>(
1411 + MarlinMoESingle<w_type_id, threads, 1, thread_n_blocks, thread_k_blocks,
1412 + stages, has_act_order, group_blocks>(
1413 A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1414 expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1415 prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1416 current_m_block);
1417 } else if (max_block == 2) {
1303 - MarlinMoESingle<threads, 2, thread_n_blocks, thread_k_blocks, stages,
1304 - has_act_order, group_blocks>(
1418 + MarlinMoESingle<w_type_id, threads, 2, thread_n_blocks, thread_k_blocks,
1419 + stages, has_act_order, group_blocks>(
1420 A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1421 expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1422 prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1423 current_m_block);
1424 } else if (max_block == 3) {
1310 - MarlinMoESingle<threads, 3, thread_n_blocks, thread_k_blocks, stages,
1311 - has_act_order, group_blocks>(
1425 + MarlinMoESingle<w_type_id, threads, 3, thread_n_blocks, thread_k_blocks,
1426 + stages, has_act_order, group_blocks>(
1427 A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1428 expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1429 prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1430 current_m_block);
1431 } else {
1317 - MarlinMoESingle<threads, 4, thread_n_blocks, thread_k_blocks, stages,
1318 - has_act_order, group_blocks>(
1432 + MarlinMoESingle<w_type_id, threads, 4, thread_n_blocks, thread_k_blocks,
1433 + stages, has_act_order, group_blocks>(
1434 A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1435 expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1436 prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
@@ -1342,7 +1457,8 @@ __global__ void compute_expert_offsets(int const* __restrict__ topk_ids,
1457 return;
1458 }
1459
1345 -template <const int threads, // number of threads in a threadblock
1460 +template <const vllm::ScalarTypeId w_type_id, // weight ScalarType id
1461 + const int threads, // number of threads in a threadblock
1462 const int thread_m_blocks, // number of 16x16 blocks in the m
1463 // dimension (batchsize) of the
1464 // threadblock
@@ -1376,7 +1492,9 @@ __global__ void MarlinMoE(
1492 bool replicate_input, // do we use the same input for each expert?
1493 bool apply_weights, // apply weights to output
1494 int current_m_block, // current m block to start kernel computation from
1379 - int max_par // maximum parallelism
1495 + int max_par, // maximum parallelism
1496 + int cfg_max_m_blocks // upper bound on m blocks
1497 +
1498 ) {
1499 // Marlin is not implemented yet for SM < 8.0
1500 assert(false);
@@ -1397,24 +1515,26 @@ const int STAGES = 4; // 4 pipeline stages fit into shared memory
1515 static constexpr int min_thread_n = 64;
1516 static constexpr int min_thread_k = 64;
1517
1400 -#define __CALL_IF_MOE(THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1401 - HAS_ACT_ORDER, GROUP_BLOCKS, NUM_THREADS) \
1402 - else if (thread_m_blocks == THREAD_M_BLOCKS && \
1518 +#define __CALL_IF_MOE(W_TYPE, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1519 + THREAD_K_BLOCKS, HAS_ACT_ORDER, GROUP_BLOCKS, \
1520 + NUM_THREADS) \
1521 + else if (q_type == W_TYPE && thread_m_blocks == THREAD_M_BLOCKS && \
1522 thread_n_blocks == THREAD_N_BLOCKS && \
1523 thread_k_blocks == THREAD_K_BLOCKS && \
1524 has_act_order == HAS_ACT_ORDER && group_blocks == GROUP_BLOCKS && \
1525 num_threads == NUM_THREADS) { \
1526 cudaFuncSetAttribute( \
1408 - MarlinMoE<NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1527 + MarlinMoE<W_TYPE.id(), NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1528 THREAD_K_BLOCKS, STAGES, HAS_ACT_ORDER, GROUP_BLOCKS>, \
1529 cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
1411 - MarlinMoE<NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1412 - STAGES, HAS_ACT_ORDER, GROUP_BLOCKS> \
1530 + MarlinMoE<W_TYPE.id(), NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1531 + THREAD_K_BLOCKS, STAGES, HAS_ACT_ORDER, GROUP_BLOCKS> \
1532 <<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
1533 A_ptr, B_ptr, C_ptr, sorted_ids_ptr, topk_weights_ptr, s_ptr, \
1534 g_idx_ptr, expert_offsets_ptr, num_groups, expert_idx, \
1535 num_experts, topk, prob_m, prob_n, prob_k, tot_m, locks, \
1417 - replicate_input, apply_weights, m_block, max_par); \
1536 + replicate_input, apply_weights, m_block, max_par, \
1537 + exec_cfg.max_m_blocks); \
1538 }
1539
1540 typedef struct {
@@ -1423,6 +1543,11 @@ typedef struct {
1543 int num_threads;
1544 } thread_config_t;
1545
1546 +typedef struct {
1547 + int max_m_blocks;
1548 + thread_config_t tb_cfg;
1549 +} exec_config_t;
1550 +
1551 thread_config_t small_batch_thread_configs[] = {
1552 // Ordered by priority
1553
@@ -1443,8 +1568,77 @@ thread_config_t large_batch_thread_configs[] = {
1568 {128, 64, 128}, // Reduce N 4X, increase K 2X
1569 };
1570
1446 -bool is_valid_config(thread_config_t const& th_config, int prob_m, int prob_n,
1447 - int prob_k) {
1571 +int get_scales_cache_size(thread_config_t const& th_config, int prob_m,
1572 + int prob_n, int prob_k, int num_bits, int group_size,
1573 + bool has_act_order, bool is_k_full) {
1574 + bool cache_scales_chunk = has_act_order && !is_k_full;
1575 +
1576 + int tb_n = th_config.thread_n;
1577 + int tb_k = th_config.thread_k;
1578 +
1579 + // Get max scale groups per thread-block
1580 + int tb_groups;
1581 + if (group_size == -1) {
1582 + tb_groups = 1;
1583 + } else if (group_size == 0) {
1584 + tb_groups = ceildiv(tb_k, 32); // Worst case is 32 group size
1585 + } else {
1586 + tb_groups = ceildiv(tb_k, group_size);
1587 + }
1588 +
1589 + if (cache_scales_chunk) {
1590 + int load_groups =
1591 + tb_groups * STAGES * 2; // Chunk size is 2x pipeline over dim K
1592 + load_groups = max(load_groups, 32); // We load at least 32 scale groups
1593 + return load_groups * tb_n * 2;
1594 +
1595 + } else {
1596 + int tb_scales = tb_groups * tb_n * 2;
1597 +
1598 + return tb_scales * STAGES;
1599 + }
1600 +}
1601 +
1602 +bool is_valid_cache_size(thread_config_t const& th_config, int max_m_blocks,
1603 + int prob_m, int prob_n, int prob_k, int num_bits,
1604 + int scales_cache_size, int max_shared_mem) {
1605 + int pack_factor = 32 / num_bits;
1606 +
1607 + // Get B size
1608 + int tb_k = th_config.thread_k;
1609 + int tb_n = th_config.thread_n;
1610 +
1611 + int b_size = (tb_k * tb_n / pack_factor) * 4;
1612 +
1613 + // Get A size
1614 + int m_blocks = ceildiv(prob_m, 16);
1615 + int tb_max_m = 16;
1616 +
1617 + while (true) {
1618 + if (m_blocks >= max_m_blocks) {
1619 + tb_max_m *= max_m_blocks;
1620 + break;
1621 + }
1622 +
1623 + max_m_blocks--;
1624 + if (max_m_blocks == 0) {
1625 + TORCH_CHECK(false, "Unexpected m_blocks = ", m_blocks);
1626 + }
1627 + }
1628 +
1629 + int a_size = (tb_max_m * tb_k) * 2;
1630 +
1631 + float pipe_size = (a_size + b_size) * STAGES;
1632 +
1633 + TORCH_CHECK(max_shared_mem / 2 > scales_cache_size); // Sanity
1634 +
1635 + return pipe_size < 0.95f * (max_shared_mem - scales_cache_size);
1636 +}
1637 +
1638 +bool is_valid_config(thread_config_t const& th_config, int max_m_blocks,
1639 + int prob_m, int prob_n, int prob_k, int num_bits,
1640 + int group_size, bool has_act_order, bool is_k_full,
1641 + int max_shared_mem) {
1642 // Sanity
1643 if (th_config.thread_k == -1 || th_config.thread_n == -1 ||
1644 th_config.num_threads == -1) {
@@ -1472,64 +1666,88 @@ bool is_valid_config(thread_config_t const& th_config, int prob_m, int prob_n,
1666 return false;
1667 }
1668
1669 + // Determine cache for scales
1670 + int scales_cache_size =
1671 + get_scales_cache_size(th_config, prob_m, prob_n, prob_k, num_bits,
1672 + group_size, has_act_order, is_k_full);
1673 +
1674 + // Check that pipeline fits into cache
1675 + if (!is_valid_cache_size(th_config, max_m_blocks, prob_m, prob_n, prob_k,
1676 + num_bits, scales_cache_size, max_shared_mem)) {
1677 + return false;
1678 + }
1679 +
1680 return true;
1681 }
1682
1478 -thread_config_t determine_thread_config(int prob_m, int prob_n, int prob_k) {
1479 - if (prob_m <= 16) {
1480 - for (auto th_config : small_batch_thread_configs) {
1481 - if (is_valid_config(th_config, prob_m, prob_n, prob_k)) {
1482 - return th_config;
1683 +exec_config_t determine_thread_config(int prob_m, int prob_n, int prob_k,
1684 + int num_bits, int group_size,
1685 + bool has_act_order, bool is_k_full,
1686 + int max_shared_mem) {
1687 + int max_m_blocks = 4;
1688 + while (max_m_blocks > 0) {
1689 + if (prob_m <= 16) {
1690 + for (auto th_config : small_batch_thread_configs) {
1691 + if (is_valid_config(th_config, max_m_blocks, prob_m, prob_n, prob_k,
1692 + num_bits, group_size, has_act_order, is_k_full,
1693 + max_shared_mem)) {
1694 + return exec_config_t{max_m_blocks, th_config};
1695 + }
1696 }
1484 - }
1485 -
1486 - } else {
1487 - for (auto th_config : large_batch_thread_configs) {
1488 - if (is_valid_config(th_config, prob_m, prob_n, prob_k)) {
1489 - return th_config;
1697 + } else {
1698 + for (auto th_config : large_batch_thread_configs) {
1699 + if (is_valid_config(th_config, max_m_blocks, prob_m, prob_n, prob_k,
1700 + num_bits, group_size, has_act_order, is_k_full,
1701 + max_shared_mem)) {
1702 + return exec_config_t{max_m_blocks, th_config};
1703 + }
1704 }
1705 }
1706 +
1707 + max_m_blocks--; // Process less M blocks per invocation to reduce cache
1708 + // usage
1709 }
1710
1494 - return thread_config_t{-1, -1, -1};
1711 + return exec_config_t{0, {-1, -1, -1}};
1712 }
1713
1497 -#define CALL_IF_MOE(N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1498 - __CALL_IF_MOE(1, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1499 - __CALL_IF_MOE(2, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1500 - __CALL_IF_MOE(3, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1501 - __CALL_IF_MOE(4, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1502 - \
1503 - __CALL_IF_MOE(1, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1504 - __CALL_IF_MOE(1, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1505 - __CALL_IF_MOE(1, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1506 - __CALL_IF_MOE(1, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1507 - \
1508 - __CALL_IF_MOE(2, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1509 - __CALL_IF_MOE(2, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1510 - __CALL_IF_MOE(2, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1511 - __CALL_IF_MOE(2, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1512 - \
1513 - __CALL_IF_MOE(3, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1514 - __CALL_IF_MOE(3, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1515 - __CALL_IF_MOE(3, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1516 - __CALL_IF_MOE(3, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1517 - \
1518 - __CALL_IF_MOE(4, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1519 - __CALL_IF_MOE(4, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1520 - __CALL_IF_MOE(4, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1521 - __CALL_IF_MOE(4, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS)
1714 +#define CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1715 + __CALL_IF_MOE(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1716 + __CALL_IF_MOE(W_TYPE, 2, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1717 + __CALL_IF_MOE(W_TYPE, 3, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1718 + __CALL_IF_MOE(W_TYPE, 4, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1719 + \
1720 + __CALL_IF_MOE(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1721 + __CALL_IF_MOE(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1722 + __CALL_IF_MOE(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1723 + __CALL_IF_MOE(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1724 + \
1725 + __CALL_IF_MOE(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1726 + __CALL_IF_MOE(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1727 + __CALL_IF_MOE(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1728 + __CALL_IF_MOE(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1729 + \
1730 + __CALL_IF_MOE(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1731 + __CALL_IF_MOE(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1732 + __CALL_IF_MOE(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1733 + __CALL_IF_MOE(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1734 + \
1735 + __CALL_IF_MOE(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1736 + __CALL_IF_MOE(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1737 + __CALL_IF_MOE(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1738 + __CALL_IF_MOE(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS)
1739
1740 void marlin_mm_moe_f16i4(const void* A, const void* B, void* C,
1741 const void* sorted_ids, const void* topk_weights,
1742 const void* topk_ids, const void* s, const void* g_idx,
1743 const void* perm, void* a_tmp, void* expert_offsets,
1744 int prob_m, int prob_n, int prob_k, void* workspace,
1528 - bool has_act_order, bool is_k_full, int num_groups,
1529 - int group_size, int num_experts, int topk,
1530 - int moe_block_size, int dev, cudaStream_t stream,
1531 - int thread_k, int thread_n, int sms, int max_par,
1532 - bool replicate_input, bool apply_weights) {
1745 + vllm::ScalarType const& q_type, bool has_act_order,
1746 + bool is_k_full, int num_groups, int group_size,
1747 + int num_experts, int topk, int moe_block_size, int dev,
1748 + cudaStream_t stream, int thread_k, int thread_n,
1749 + int sms, int max_par, bool replicate_input,
1750 + bool apply_weights) {
1751 TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m,
1752 ", ", prob_n, ", ", prob_k, "]");
1753
@@ -1537,26 +1755,42 @@ void marlin_mm_moe_f16i4(const void* A, const void* B, void* C,
1755 cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev);
1756 }
1757
1758 + int max_shared_mem = 0;
1759 + cudaDeviceGetAttribute(&max_shared_mem,
1760 + cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
1761 + TORCH_CHECK(max_shared_mem > 0);
1762 +
1763 + int num_bits = q_type.size_bits();
1764 +
1765 // Set thread config
1541 - thread_config_t th_config;
1766 + exec_config_t exec_cfg;
1767 if (thread_k != -1 && thread_n != -1) {
1768 // User-defined config
1544 - th_config = thread_config_t{thread_k, thread_n, USER_THREADS};
1769 + exec_cfg =
1770 + exec_config_t{4, thread_config_t{thread_k, thread_n, USER_THREADS}};
1771 } else {
1772 // Auto config
1547 - th_config = determine_thread_config(prob_m, prob_n, prob_k);
1773 + exec_cfg =
1774 + determine_thread_config(prob_m, prob_n, prob_k, num_bits, group_size,
1775 + has_act_order, is_k_full, max_shared_mem);
1776 }
1777
1550 - TORCH_CHECK(is_valid_config(th_config, prob_m, prob_n, prob_k),
1551 - "Invalid thread config: thread_k = " + str(th_config.thread_k) +
1552 - ", thread_n = " + str(th_config.thread_n) +
1553 - ", num_threads = " + str(th_config.num_threads) +
1554 - " for MKN = [" + str(prob_m) + ", " + str(prob_k) + ", " +
1555 - str(prob_n) + "]");
1556 -
1557 - int num_threads = th_config.num_threads;
1558 - thread_k = th_config.thread_k;
1559 - thread_n = th_config.thread_n;
1778 + TORCH_CHECK(exec_cfg.max_m_blocks > 0 &&
1779 + is_valid_config(exec_cfg.tb_cfg, exec_cfg.max_m_blocks,
1780 + prob_m, prob_n, prob_k, num_bits, group_size,
1781 + has_act_order, is_k_full, max_shared_mem),
1782 + "Invalid thread config: max_m_blocks = ", exec_cfg.max_m_blocks,
1783 + ", thread_k = ", exec_cfg.tb_cfg.thread_k,
1784 + ", thread_n = ", exec_cfg.tb_cfg.thread_n,
1785 + ", num_threads = ", exec_cfg.tb_cfg.num_threads, " for MKN = [",
1786 + prob_m, ", ", prob_k, ", ", prob_n, "] and num_bits = ", num_bits,
1787 + ", group_size = ", group_size,
1788 + ", has_act_order = ", has_act_order, ", is_k_full = ", is_k_full,
1789 + ", max_shared_mem = ", max_shared_mem);
1790 +
1791 + int num_threads = exec_cfg.tb_cfg.num_threads;
1792 + thread_k = exec_cfg.tb_cfg.thread_k;
1793 + thread_n = exec_cfg.tb_cfg.thread_n;
1794
1795 int thread_k_blocks = thread_k / 16;
1796 int thread_n_blocks = thread_n / 16;
@@ -1590,11 +1824,6 @@ void marlin_mm_moe_f16i4(const void* A, const void* B, void* C,
1824 }
1825 }
1826
1593 - int max_shared_mem = 0;
1594 - cudaDeviceGetAttribute(&max_shared_mem,
1595 - cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
1596 - TORCH_CHECK(max_shared_mem > 0);
1597 -
1827 int tot_m = prob_m;
1828
1829 const int* topk_ids_ptr = (const int*)topk_ids;
@@ -1611,10 +1840,13 @@ void marlin_mm_moe_f16i4(const void* A, const void* B, void* C,
1840 has_act_order = false;
1841 }
1842
1843 + int pack_factor = 32 / q_type.size_bits();
1844 +
1845 for (int expert_idx = 0; expert_idx < num_experts; ++expert_idx) {
1846 const int4* A_ptr = (const int4*)A;
1847 int4* a_tmp_ptr = (int4*)a_tmp;
1617 - const int4* B_ptr = (const int4*)B + (prob_n * prob_k / 32) * expert_idx;
1848 + const int4* B_ptr =
1849 + (const int4*)B + (prob_n * prob_k / (pack_factor * 4)) * expert_idx;
1850 int4* C_ptr = (int4*)C;
1851 const float* topk_weights_ptr = (const float*)topk_weights;
1852 const int* sorted_ids_ptr = (const int*)sorted_ids;
@@ -1636,19 +1868,22 @@ void marlin_mm_moe_f16i4(const void* A, const void* B, void* C,
1868 A_ptr = a_tmp_ptr;
1869 }
1870
1639 - int max_m_blocks = ceildiv(tot_m, 16);
1640 - for (int m_block = 0; m_block < max_m_blocks; m_block += 16) {
1641 - // Define kernel configurations
1642 -
1871 + int tot_m_blocks = ceildiv(tot_m, 16);
1872 + for (int m_block = 0; m_block < tot_m_blocks;
1873 + m_block += 4 * exec_cfg.max_m_blocks) {
1874 // make it max possible value
1644 - int thread_m_blocks = 4;
1875 + int thread_m_blocks = exec_cfg.max_m_blocks;
1876
1877 if (false) {
1878 }
1648 - CALL_IF_MOE(16, 4, 256)
1649 - CALL_IF_MOE(8, 8, 256)
1650 - CALL_IF_MOE(8, 4, 128)
1651 - CALL_IF_MOE(4, 8, 128)
1879 + CALL_IF_MOE(vllm::kU4B8, 16, 4, 256)
1880 + CALL_IF_MOE(vllm::kU4B8, 8, 8, 256)
1881 + CALL_IF_MOE(vllm::kU4B8, 8, 4, 128)
1882 + CALL_IF_MOE(vllm::kU4B8, 4, 8, 128)
1883 + CALL_IF_MOE(vllm::kU8B128, 16, 4, 256)
1884 + CALL_IF_MOE(vllm::kU8B128, 8, 8, 256)
1885 + CALL_IF_MOE(vllm::kU8B128, 8, 4, 128)
1886 + CALL_IF_MOE(vllm::kU8B128, 4, 8, 128)
1887 else {
1888 TORCH_CHECK(false, "Unsupported shapes: MNK = [" + str(prob_m) + ", " +
1889 str(prob_n) + ", " + str(prob_k) + "]" +
@@ -1670,9 +1905,15 @@ torch::Tensor marlin_gemm_moe(
1905 const torch::Tensor& sorted_ids, const torch::Tensor& topk_weights,
1906 const torch::Tensor& topk_ids, const torch::Tensor& b_scales,
1907 const torch::Tensor& g_idx, const torch::Tensor& perm,
1673 - torch::Tensor& workspace, int64_t size_m, int64_t size_n, int64_t size_k,
1674 - bool is_k_full, int64_t num_experts, int64_t topk, int64_t moe_block_size,
1908 + torch::Tensor& workspace, vllm::ScalarTypeTorchPtr const& b_q_type,
1909 + int64_t size_m, int64_t size_n, int64_t size_k, bool is_k_full,
1910 + int64_t num_experts, int64_t topk, int64_t moe_block_size,
1911 bool replicate_input, bool apply_weights) {
1912 + TORCH_CHECK(*b_q_type == vllm::kU4B8 || *b_q_type == vllm::kU8B128,
1913 + "b_q_type must be uint4b8 or uint8b128. Got = ", b_q_type->str());
1914 +
1915 + int pack_factor = 32 / b_q_type->size_bits();
1916 +
1917 int max_par = 4;
1918
1919 int dev = a.get_device();
@@ -1733,8 +1974,8 @@ torch::Tensor marlin_gemm_moe(
1974 topk_weights.data_ptr(), topk_ids.data_ptr(), b_scales.data_ptr(),
1975 g_idx.data_ptr(), perm.data_ptr(), a_tmp.data_ptr(),
1976 expert_offsets.data_ptr(), size_m, size_n, size_k, workspace.data_ptr(),
1736 - has_act_order, is_k_full, num_groups, group_size, num_experts, topk,
1737 - moe_block_size, dev, at::cuda::getCurrentCUDAStream(dev), thread_k,
1977 + *b_q_type, has_act_order, is_k_full, num_groups, group_size, num_experts,
1978 + topk, moe_block_size, dev, at::cuda::getCurrentCUDAStream(dev), thread_k,
1979 thread_n, sms, max_par, replicate_input, apply_weights);
1980 return c;
1981 }
csrc/moe/marlin_moe_ops.h
+5 -2
@@ -2,11 +2,14 @@
2
3 #include <torch/all.h>
4
5 +#include "core/scalar_type.hpp"
6 +
7 torch::Tensor marlin_gemm_moe(
8 const torch::Tensor& a, const torch::Tensor& b_q_weights,
9 const torch::Tensor& sorted_ids, const torch::Tensor& topk_weights,
10 const torch::Tensor& topk_ids, const torch::Tensor& b_scales,
11 const torch::Tensor& g_idx, const torch::Tensor& perm,
10 - torch::Tensor& workspace, int64_t size_m, int64_t size_n, int64_t size_k,
11 - bool is_k_full, int64_t num_experts, int64_t topk, int64_t moe_block_size,
12 + torch::Tensor& workspace, vllm::ScalarTypeTorchPtr const& b_q_type,
13 + int64_t size_m, int64_t size_n, int64_t size_k, bool is_k_full,
14 + int64_t num_experts, int64_t topk, int64_t moe_block_size,
15 bool replicate_input, bool apply_weights);
csrc/moe/torch_bindings.cpp
+5 -3
@@ -13,9 +13,11 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
13 m.def(
14 "marlin_gemm_moe(Tensor! a, Tensor! b_q_weights, Tensor! sorted_ids, "
15 "Tensor! topk_weights, Tensor! topk_ids, Tensor! b_scales, Tensor! "
16 - "g_idx, Tensor! perm, Tensor! workspace, int size_m, int size_n, int "
17 - "size_k, bool is_k_full, int num_experts, int topk, int moe_block_size, "
18 - "bool replicate_input, bool apply_weights) -> Tensor");
16 + "g_idx, Tensor! perm, Tensor! workspace, "
17 + "__torch__.torch.classes._core_C.ScalarType b_q_type, int size_m, "
18 + "int size_n, int size_k, bool is_k_full, int num_experts, int topk, "
19 + "int moe_block_size, bool replicate_input, bool apply_weights)"
20 + " -> Tensor");
21 m.impl("marlin_gemm_moe", torch::kCUDA, &marlin_gemm_moe);
22 #endif
23 }
tests/kernels/test_moe.py
+12 -6
@@ -140,6 +140,7 @@ def compute_max_diff(output, output_ref):
140 @pytest.mark.parametrize("topk", [2, 6])
141 @pytest.mark.parametrize("group_size", [-1, 32, 64, 128])
142 @pytest.mark.parametrize("act_order", [True, False])
143 +@pytest.mark.parametrize("num_bits", [4, 8])
144 def test_fused_marlin_moe(
145 m: int,
146 n: int,
@@ -148,6 +149,7 @@ def test_fused_marlin_moe(
149 topk: int,
150 group_size: int,
151 act_order: bool,
152 + num_bits: int,
153 ):
154 torch.manual_seed(7)
155
@@ -161,13 +163,12 @@ def test_fused_marlin_moe(
163 if group_size in (k, n):
164 return
165
164 - quant_type = scalar_types.uint4b8
166 + quant_type = (scalar_types.uint4b8
167 + if num_bits == 4 else scalar_types.uint8b128)
168 dtype = torch.float16
169 a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
170 w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10
171 w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10
169 - for i in range(w2.shape[0]):
170 - w2[0] = torch.eye(k, n, device="cuda", dtype=dtype)
172
173 w_ref1_l = []
174 qweight1_l = []
@@ -240,6 +241,7 @@ def test_fused_marlin_moe(
241 topk_ids,
242 w1_scale=scales1,
243 w2_scale=scales2,
244 + num_bits=num_bits,
245 )
246
247 assert compute_max_diff(marlin_output, triton_output) < 4e-2
@@ -254,7 +256,8 @@ def test_fused_marlin_moe(
256 @pytest.mark.parametrize("topk", [2, 6])
257 @pytest.mark.parametrize("group_size", [-1, 32, 64, 128])
258 @pytest.mark.parametrize("act_order", [True, False])
257 -def test_marlin_moe_mmm(
259 +@pytest.mark.parametrize("num_bits", [4, 8])
260 +def test_single_marlin_moe_multiply(
261 m: int,
262 n: int,
263 k: int,
@@ -262,6 +265,7 @@ def test_marlin_moe_mmm(
265 topk: int,
266 group_size: int,
267 act_order: bool,
268 + num_bits: int,
269 ):
270 if topk > e:
271 return
@@ -273,7 +277,8 @@ def test_marlin_moe_mmm(
277 if group_size == k:
278 return
279
276 - quant_type = scalar_types.uint4b8
280 + quant_type = (scalar_types.uint4b8
281 + if num_bits == 4 else scalar_types.uint8b128)
282 dtype = torch.float16
283 a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
284 w = torch.randn((e, n, k), device="cuda", dtype=dtype) / 10
@@ -308,7 +313,8 @@ def test_marlin_moe_mmm(
313 g_idx,
314 sort_indices,
315 topk,
311 - renormalize=False)
316 + renormalize=False,
317 + num_bits=num_bits)
318 torch_output = torch_moe_single(a, w_ref.transpose(1, 2), score, topk)
319
320 assert compute_max_diff(marlin_output, torch_output) < 1e-2
tests/weight_loading/models-large.txt
+2 -1
@@ -1,3 +1,4 @@
1 compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-quantized, main
2 compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-channel-quantized, main
3 -gptq_marlin, TheBloke/Mixtral-8x7B-v0.1-GPTQ, main
\ No newline at end of file
3 +compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W8A16-quantized, main
4 +gptq_marlin, TheBloke/Mixtral-8x7B-v0.1-GPTQ, main
tests/weight_loading/run_model_weight_loading_test.sh
vllm/_custom_ops.py
+1 -1
@@ -559,7 +559,7 @@ def gptq_marlin_moe_repack(b_q_weight: torch.Tensor, perm: torch.Tensor,
559 num_bits: int) -> torch.Tensor:
560 num_experts = b_q_weight.shape[0]
561 assert size_k % 16 == 0
562 - output = torch.empty((num_experts, size_k // 16, size_n * 2),
562 + output = torch.empty((num_experts, size_k // 16, size_n * (num_bits // 2)),
563 device=b_q_weight.device,
564 dtype=b_q_weight.dtype)
565 for e in range(num_experts):
vllm/model_executor/layers/fused_moe/fused_marlin_moe.py
+30 -14
@@ -7,18 +7,21 @@ import torch
7 from vllm import _custom_ops as ops
8 from vllm.model_executor.layers.fused_moe.fused_moe import (
9 fused_topk, moe_align_block_size, try_get_optimal_moe_config)
10 +from vllm.scalar_type import scalar_types
11
12
13 def single_marlin_moe(
13 - hidden_states: torch.Tensor,
14 - w: torch.Tensor,
15 - scales: torch.Tensor,
16 - gating_output: torch.Tensor,
17 - g_idx: torch.Tensor,
18 - perm: torch.Tensor,
19 - topk: int,
20 - renormalize: bool,
21 - override_config: Optional[Dict[str, Any]] = None) -> torch.Tensor:
14 + hidden_states: torch.Tensor,
15 + w: torch.Tensor,
16 + scales: torch.Tensor,
17 + gating_output: torch.Tensor,
18 + g_idx: torch.Tensor,
19 + perm: torch.Tensor,
20 + topk: int,
21 + renormalize: bool,
22 + override_config: Optional[Dict[str, Any]] = None,
23 + num_bits: int = 8,
24 +) -> torch.Tensor:
25 """
26 This function computes the multiplication of hidden_states with expert
27 weights used in Marlin MoE, using weights w and top-k gating mechanism.
@@ -36,6 +39,7 @@ def single_marlin_moe(
39 - renormalize (bool): If True, renormalize the top-k weights to sum to 1.
40 - override_config (Optional[Dict[str, Any]]): Optional override
41 for the kernel configuration.
42 + - num_bits (bool): The number of bits in expert weights quantization.
43
44 Returns:
45 - torch.Tensor: The output tensor after applying the MoE layer.
@@ -48,10 +52,11 @@ def single_marlin_moe(
52 assert hidden_states.is_contiguous(), "Hidden_states must be contiguous"
53 assert w.is_contiguous(), "Expert weights must be contiguous"
54 assert hidden_states.dtype == torch.float16
55 + assert num_bits in [4, 8]
56
57 M, K = hidden_states.shape
58 E = w.shape[0]
54 - N = w.shape[2] // 2
59 + N = w.shape[2] // (num_bits // 2)
60
61 topk_weights, topk_ids = fused_topk(hidden_states, gating_output, topk,
62 renormalize)
@@ -76,10 +81,13 @@ def single_marlin_moe(
81 device="cuda",
82 requires_grad=False)
83
84 + scalar_type = (scalar_types.uint4b8
85 + if num_bits == 4 else scalar_types.uint8b128)
86 +
87 intermediate_cache = torch.ops._moe_C.marlin_gemm_moe(
88 hidden_states, w, sorted_token_ids, topk_weights, topk_ids, scales,
81 - g_idx, perm, workspace, M, N, K, True, E, topk, block_size_m, True,
82 - False)
89 + g_idx, perm, workspace, scalar_type, M, N, K, True, E, topk,
90 + block_size_m, True, False)
91
92 return torch.sum(intermediate_cache.view(*intermediate_cache.shape), dim=1)
93
@@ -98,6 +106,7 @@ def fused_marlin_moe(
106 override_config: Optional[Dict[str, Any]] = None,
107 w1_scale: Optional[torch.Tensor] = None,
108 w2_scale: Optional[torch.Tensor] = None,
109 + num_bits: int = 8,
110 ) -> torch.Tensor:
111 """
112 This function computes a Mixture of Experts (MoE) layer using two sets of
@@ -122,6 +131,7 @@ def fused_marlin_moe(
131 w1.
132 - w2_scale (Optional[torch.Tensor]): Optional scale to be used for
133 w2.
134 + - num_bits (bool): The number of bits in expert weights quantization.
135
136 Returns:
137 - torch.Tensor: The output tensor after applying the MoE layer.
@@ -131,13 +141,14 @@ def fused_marlin_moe(
141 0], "Number of tokens mismatch"
142 assert hidden_states.shape[
143 1] == w1.shape[1] * 16, "Hidden size mismatch w1"
134 - assert hidden_states.shape[
135 - 1] == w2.shape[2] // 2, "Hidden size mismatch w2"
144 + assert hidden_states.shape[1] == w2.shape[2] // (
145 + num_bits // 2), "Hidden size mismatch w2"
146 assert gating_output.shape[1] == w1.shape[0], "Number of experts mismatch"
147 assert hidden_states.is_contiguous(), "Hidden_states must be contiguous"
148 assert w1.is_contiguous(), "Expert weights1 must be contiguous"
149 assert w2.is_contiguous(), "Expert weights2 must be contiguous"
150 assert hidden_states.dtype == torch.float16
151 + assert num_bits in [4, 8]
152
153 M, K = hidden_states.shape
154 E = w1.shape[0]
@@ -165,6 +176,9 @@ def fused_marlin_moe(
176 device="cuda",
177 requires_grad=False)
178
179 + scalar_type = (scalar_types.uint4b8
180 + if num_bits == 4 else scalar_types.uint8b128)
181 +
182 intermediate_cache2 = torch.empty(
183 (M * topk_ids.shape[1], N),
184 device=hidden_states.device,
@@ -181,6 +195,7 @@ def fused_marlin_moe(
195 g_idx1,
196 perm1,
197 workspace,
198 + scalar_type,
199 M,
200 2 * N,
201 K,
@@ -204,6 +219,7 @@ def fused_marlin_moe(
219 g_idx2,
220 perm2,
221 workspace,
222 + scalar_type,
223 M,
224 K,
225 N,
vllm/model_executor/layers/fused_moe/fused_moe.py
+1 -1
@@ -445,7 +445,7 @@ def grouped_topk(hidden_states: torch.Tensor,
445 if renormalize:
446 topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
447
448 - return topk_weights, topk_ids.to(torch.int32)
448 + return topk_weights.to(torch.float32), topk_ids.to(torch.int32)
449
450
451 def get_config_dtype_str(dtype: torch.dtype,
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+6 -2
@@ -6,6 +6,8 @@ import torch
6
7 from vllm import _custom_ops as ops
8 from vllm.model_executor.layers.fused_moe import FusedMoE, FusedMoEMethodBase
9 +from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
10 + WNA16_SUPPORTED_BITS)
11 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
12 CompressionFormat)
13 from vllm.model_executor.utils import set_weight_attrs
@@ -38,10 +40,11 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
40
41 if not (self.quant_config.quant_format
42 == CompressionFormat.pack_quantized.value
41 - and self.num_bits == 4):
43 + and self.num_bits in WNA16_SUPPORTED_BITS):
44 raise ValueError("For Fused MoE layers, only ",
45 f"{CompressionFormat.pack_quantized.value} ",
44 - "is supported for 4 bits")
46 + "is supported for the following bits: ",
47 + f"{WNA16_SUPPORTED_BITS}")
48
49 def create_weights(self, layer: torch.nn.Module, num_experts: int,
50 hidden_size: int, intermediate_size: int,
@@ -292,4 +295,5 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
295 topk_ids,
296 w1_scale=layer.w13_weight_scale,
297 w2_scale=layer.w2_weight_scale,
298 + num_bits=self.num_bits,
299 )
vllm/model_executor/layers/quantization/gptq_marlin.py
+1
@@ -611,4 +611,5 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
611 topk_ids,
612 w1_scale=layer.w13_scales,
613 w2_scale=layer.w2_scales,
614 + num_bits=self.quant_config.quant_type.size_bits,
615 ).to(orig_dtype)
vllm/model_executor/model_loader/utils.py
+1 -7
@@ -23,13 +23,7 @@ def get_model_architecture(
23 architectures = getattr(model_config.hf_config, "architectures", [])
24 # Special handling for quantized Mixtral.
25 # FIXME(woosuk): This is a temporary hack.
26 - mixtral_supported = ["fp8", "compressed-tensors"]
27 - # for gptq_marlin, only run fused MoE for int4
28 - if model_config.quantization == "gptq_marlin":
29 - hf_quant_config = getattr(model_config.hf_config,
30 - "quantization_config", None)
31 - if hf_quant_config and hf_quant_config.get("bits") == 4:
32 - mixtral_supported.append("gptq_marlin")
26 + mixtral_supported = ["fp8", "compressed-tensors", "gptq_marlin"]
27
28 if (model_config.quantization is not None
29 and model_config.quantization not in mixtral_supported