A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Kernel] Enable 8-bit weights in Fused Marlin MoE (#8032)
Co-authored-by: Dipika <dipikasikka1@gmail.com>
ElizaWszola committed
Sep 16, 2024 at 17:47 UTC
a091e2da3e3fcb4c63c8206839d7240a2a2a176a
12 files changed
+453
-185
csrc/moe/marlin_moe_ops.cu
+389
-148
@@ -25,6 +25,8 @@
25
26
#include <iostream>
27
28
+#include "core/scalar_type.hpp"
29
+
30
template <typename T>
31
inline std::string str(T x) {
32
return std::to_string(x);
@@ -131,11 +133,26 @@ __device__ inline int lop3(int a, int b, int c) {
133
return res;
134
}
135
134
-// Efficiently dequantize an int32 value into a full B-fragment of 4 fp16
135
-// values. We mostly follow the strategy in the link below, with some small
136
-// changes:
137
-// https://github.com/NVIDIA/FasterTransformer/blob/main/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h
138
-__device__ inline FragB dequant(int q) {
136
+// Constructs destination register by taking bytes from 2 sources (based on
137
+// mask)
138
+template <int start_byte, int mask>
139
+__device__ inline uint32_t prmt(uint32_t a) {
140
+ uint32_t res;
141
+ asm volatile("prmt.b32 %0, %1, %2, %3;\n"
142
+ : "=r"(res)
143
+ : "r"(a), "n"(start_byte), "n"(mask));
144
+ return res;
145
+}
146
+
147
+template <vllm::ScalarTypeId w_type_id>
148
+__device__ inline FragB dequant(int q);
149
+
150
+// Efficiently dequantize 4bit values packed in an int32 value into a full
151
+// B-fragment of 4 fp16 values. We mostly follow the strategy in the link below,
152
+// with some small changes:
153
+// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L215-L287
154
+template <>
155
+__device__ inline FragB dequant<vllm::kU4B8.id()>(int q) {
156
const int LO = 0x000f000f;
157
const int HI = 0x00f000f0;
158
const int EX = 0x64006400;
@@ -156,6 +173,28 @@ __device__ inline FragB dequant(int q) {
173
return frag_b;
174
}
175
176
+// Fast Int8ToFp16: Efficiently dequantize 8bit int values to fp16
177
+// Reference:
178
+// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L53-L85
179
+template <>
180
+__device__ inline FragB dequant<vllm::kU8B128.id()>(int q) {
181
+ static constexpr uint32_t mask_for_elt_01 = 0x5250;
182
+ static constexpr uint32_t mask_for_elt_23 = 0x5351;
183
+ static constexpr uint32_t start_byte_for_fp16 = 0x64646464;
184
+
185
+ uint32_t lo = prmt<start_byte_for_fp16, mask_for_elt_01>(q);
186
+ uint32_t hi = prmt<start_byte_for_fp16, mask_for_elt_23>(q);
187
+
188
+ static constexpr uint32_t I8s_TO_F16s_MAGIC_NUM = 0x64806480;
189
+
190
+ FragB frag_b;
191
+ frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
192
+ *reinterpret_cast<const half2*>(&I8s_TO_F16s_MAGIC_NUM));
193
+ frag_b[1] = __hsub2(*reinterpret_cast<half2*>(&hi),
194
+ *reinterpret_cast<const half2*>(&I8s_TO_F16s_MAGIC_NUM));
195
+ return frag_b;
196
+}
197
+
198
// Multiply dequantized values by the corresponding quantization scale; used
199
// only for grouped quantization.
200
__device__ inline void scale(FragB& frag_b, FragS& frag_s, int i) {
@@ -296,7 +335,8 @@ __global__ void compute_expert_offsets(int const* __restrict__ topk_ids,
335
__syncthreads();
336
}
337
299
-template <const int threads, // number of threads in a threadblock
338
+template <const vllm::ScalarTypeId w_type_id, // weight ScalarType id
339
+ const int threads, // number of threads in a threadblock
340
const int thread_m_blocks, // number of 16x16 blocks in the m
341
// dimension (batchsize) of the
342
// threadblock
@@ -331,6 +371,9 @@ __device__ inline void MarlinMoESingle(
371
bool apply_weights, // apply weights to output
372
int current_m_block // current m block to start kernel computation from
373
) {
374
+ static constexpr auto w_type = vllm::ScalarType::from_id(w_type_id);
375
+ constexpr int pack_factor = 32 / w_type.size_bits();
376
+
377
// For larger GEMMs we run multiple batchsize 64 versions in parallel for a
378
// better partitioning with less reductions
379
int parallel = 1;
@@ -423,21 +466,25 @@ __device__ inline void MarlinMoESingle(
466
constexpr int a_sh_wr_iters = ceildiv(a_sh_stage, a_sh_wr_delta);
467
468
// B sizes/strides
426
- int b_gl_stride = 16 * prob_n / 32;
427
- constexpr int b_sh_stride = 32 * thread_n_blocks / 4;
469
+ int b_gl_stride = 16 * prob_n / (pack_factor * 4);
470
+ constexpr int b_sh_stride = ((thread_n_blocks * 16) * 16 / pack_factor) / 4;
471
+ constexpr int b_thread_vecs = w_type.size_bits() == 4 ? 1 : 2;
472
+ constexpr int b_sh_stride_threads = b_sh_stride / b_thread_vecs;
473
+
474
int b_gl_rd_delta_o = b_gl_stride * thread_k_blocks;
429
- int b_gl_rd_delta_i = b_gl_stride * (threads / b_sh_stride);
430
- constexpr int b_sh_wr_delta = threads;
431
- constexpr int b_sh_rd_delta = threads;
475
+ int b_gl_rd_delta_i = b_gl_stride * (threads / b_sh_stride_threads);
476
+ constexpr int b_sh_wr_delta = threads * b_thread_vecs;
477
+ constexpr int b_sh_rd_delta = threads * b_thread_vecs;
478
constexpr int b_sh_stage = b_sh_stride * thread_k_blocks;
479
constexpr int b_sh_wr_iters = b_sh_stage / b_sh_wr_delta;
480
481
// Scale sizes/strides without act_order
482
int s_gl_stride = prob_n / 8;
483
constexpr int s_sh_stride = 16 * thread_n_blocks / 8;
438
- constexpr int s_tb_groups = !has_act_order && group_blocks < thread_k_blocks
439
- ? thread_k_blocks / group_blocks
440
- : 1;
484
+ constexpr int s_tb_groups =
485
+ !has_act_order && group_blocks != -1 && group_blocks < thread_k_blocks
486
+ ? thread_k_blocks / group_blocks
487
+ : 1;
488
constexpr int s_sh_stage = s_tb_groups * s_sh_stride;
489
int s_gl_rd_delta = s_gl_stride;
490
// Scale size/strides with act_order
@@ -465,12 +512,12 @@ __device__ inline void MarlinMoESingle(
512
a_sh_stride * ((threadIdx.x % 32) % 16) + (threadIdx.x % 32) / 16;
513
a_sh_rd += 2 * ((threadIdx.x / 32) / (thread_n_blocks / 4));
514
468
- int b_gl_rd =
469
- b_gl_stride * (threadIdx.x / b_sh_stride) + (threadIdx.x % b_sh_stride);
515
+ int b_gl_rd = b_gl_stride * (threadIdx.x / b_sh_stride_threads) +
516
+ (threadIdx.x % b_sh_stride_threads) * b_thread_vecs;
517
b_gl_rd += b_sh_stride * slice_col;
518
b_gl_rd += b_gl_rd_delta_o * slice_row;
472
- int b_sh_wr = threadIdx.x;
473
- int b_sh_rd = threadIdx.x;
519
+ int b_sh_wr = threadIdx.x * b_thread_vecs;
520
+ int b_sh_rd = threadIdx.x * b_thread_vecs;
521
522
// For act_order
523
constexpr int k_iter_size = tb_k / b_sh_wr_iters;
@@ -481,11 +528,13 @@ __device__ inline void MarlinMoESingle(
528
529
// No act_order
530
int s_gl_rd;
484
- if constexpr (group_blocks == -1 || group_blocks == 0) {
485
- s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
486
- } else {
487
- s_gl_rd = s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
488
- s_sh_stride * slice_col + threadIdx.x;
531
+ if constexpr (!has_act_order) {
532
+ if constexpr (group_blocks == -1) {
533
+ s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
534
+ } else {
535
+ s_gl_rd = s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
536
+ s_sh_stride * slice_col + threadIdx.x;
537
+ }
538
}
539
int s_sh_wr = threadIdx.x;
540
bool s_sh_wr_pred = threadIdx.x < s_sh_stride;
@@ -571,7 +620,7 @@ __device__ inline void MarlinMoESingle(
620
621
// Register storage for double buffer of shared memory reads.
622
FragA frag_a[2][thread_m_blocks];
574
- I4 frag_b_quant[2];
623
+ I4 frag_b_quant[2][b_thread_vecs];
624
FragC frag_c[thread_m_blocks][4][2];
625
FragS frag_s[2][4]; // No act-order
626
FragS act_frag_s[2][4][4]; // For act-order
@@ -637,7 +686,10 @@ __device__ inline void MarlinMoESingle(
686
int4* sh_b_stage = sh_b + b_sh_stage * pipe;
687
#pragma unroll
688
for (int i = 0; i < b_sh_wr_iters; i++) {
640
- cp_async4(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr], B_ptr[i]);
689
+ #pragma unroll
690
+ for (int j = 0; j < b_thread_vecs; j++) {
691
+ cp_async4(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr + j], B_ptr[i] + j);
692
+ }
693
B_ptr[i] += b_gl_rd_delta_o;
694
}
695
@@ -715,14 +767,24 @@ __device__ inline void MarlinMoESingle(
767
for (int i = 0; i < thread_m_blocks; i++)
768
ldsm4(frag_a[k % 2][i], &sh_a_stage[a_sh_rd_trans[k % b_sh_wr_iters][i]]);
769
int4* sh_b_stage = sh_b + b_sh_stage * pipe;
718
- frag_b_quant[k % 2] = *reinterpret_cast<I4*>(
719
- &sh_b_stage[b_sh_rd_delta * (k % b_sh_wr_iters) + b_sh_rd]);
770
+
771
+ #pragma unroll
772
+ for (int i = 0; i < b_thread_vecs; i++) {
773
+ frag_b_quant[k % 2][i] = *reinterpret_cast<I4*>(
774
+ &sh_b_stage[b_sh_rd_delta * (k % b_sh_wr_iters) + b_sh_rd + i]);
775
+ }
776
};
777
778
bool is_same_group[stages];
779
int same_group_id[stages];
780
781
auto init_same_group = [&](int pipe) {
782
+ if constexpr (!has_act_order) {
783
+ is_same_group[pipe] = false;
784
+ same_group_id[pipe] = 0;
785
+ return;
786
+ }
787
+
788
int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
789
int* sh_g_idx_int_ptr = reinterpret_cast<int*>(sh_g_idx_stage);
790
@@ -840,10 +902,19 @@ __device__ inline void MarlinMoESingle(
902
// dequantization and matmul operations.
903
#pragma unroll
904
for (int j = 0; j < 4; j++) {
843
- int b_quant = frag_b_quant[k % 2][j];
844
- int b_quant_shift = b_quant >> 8;
905
+ int b_quant_0, b_quant_1;
906
+ if constexpr (w_type.size_bits() == 4) {
907
+ b_quant_0 = frag_b_quant[k % 2][0][j];
908
+ b_quant_1 = b_quant_0 >> 8;
909
+ } else {
910
+ static_assert(w_type.size_bits() == 8);
911
+ int* frag_b_quant_ptr = reinterpret_cast<int*>(frag_b_quant[k % 2]);
912
+ b_quant_0 = frag_b_quant_ptr[j * 2 + 0];
913
+ b_quant_1 = frag_b_quant_ptr[j * 2 + 1];
914
+ }
915
846
- FragB frag_b0 = dequant(b_quant);
916
+ FragB frag_b0 = dequant<w_type_id>(b_quant_0);
917
+ FragB frag_b1 = dequant<w_type_id>(b_quant_1);
918
919
// Apply scale to frag_b0
920
if constexpr (has_act_order) {
@@ -855,8 +926,6 @@ __device__ inline void MarlinMoESingle(
926
}
927
}
928
858
- FragB frag_b1 = dequant(b_quant_shift);
859
-
929
// Apply scale to frag_b1
930
if constexpr (has_act_order) {
931
scale4(frag_b1, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
@@ -881,13 +950,13 @@ __device__ inline void MarlinMoESingle(
950
// multiple warps that accumulate their partial sums of the same output
951
// location; which we have to reduce over in the end. We do in shared memory.
952
auto thread_block_reduce = [&]() {
884
- constexpr int red_off = threads / b_sh_stride / 2;
953
+ constexpr int red_off = threads / b_sh_stride_threads / 2;
954
if (red_off >= 1) {
886
- int red_idx = threadIdx.x / b_sh_stride;
887
- constexpr int red_sh_stride = b_sh_stride * 4 * 2;
888
- constexpr int red_sh_delta = b_sh_stride;
889
- int red_sh_rd = red_sh_stride * (threadIdx.x / b_sh_stride) +
890
- (threadIdx.x % b_sh_stride);
955
+ int red_idx = threadIdx.x / b_sh_stride_threads;
956
+ constexpr int red_sh_stride = b_sh_stride_threads * 4 * 2;
957
+ constexpr int red_sh_delta = b_sh_stride_threads;
958
+ int red_sh_rd = red_sh_stride * (threadIdx.x / b_sh_stride_threads) +
959
+ (threadIdx.x % b_sh_stride_threads);
960
961
// Parallel logarithmic shared memory reduction. We make sure to avoid any
962
// unnecessary read or write iterations, e.g., for two warps we write only
@@ -1035,8 +1104,10 @@ __device__ inline void MarlinMoESingle(
1104
auto write = [&](int idx, float c0, float c1, FragS& s) {
1105
half2 res = __halves2half2(__float2half(c0), __float2half(c1));
1106
1038
- // For per-column quantization we finally apply the scale here
1039
- if constexpr (!has_act_order && group_blocks == -1) {
1107
+ // For per-column quantization we finally apply the scale here (only for
1108
+ // 4-bit)
1109
+ if constexpr (!has_act_order && group_blocks == -1 &&
1110
+ w_type.size_bits() == 4) {
1111
res = __hmul2(res, s[0]);
1112
}
1113
@@ -1088,9 +1159,9 @@ __device__ inline void MarlinMoESingle(
1159
1160
// Start global fetch and register load pipelines.
1161
auto start_pipes = [&]() {
1091
- // TODO re-enable after fixing this function
1092
- // fetch_sorted_ids_to_shared();
1093
- __syncthreads();
1162
+ // TODO re-enable after fixing this function
1163
+ // fetch_sorted_ids_to_shared();
1164
+ // __syncthreads();
1165
1166
#pragma unroll
1167
for (int i = 0; i < stages - 1; i++) {
@@ -1166,28 +1237,70 @@ __device__ inline void MarlinMoESingle(
1237
if (slice_iters == 0) {
1238
cp_async_wait<0>();
1239
bool last = slice_idx == slice_count - 1;
1169
- // For per-column scales, we only fetch them here in the final step before
1170
- // write-out
1240
if constexpr (!has_act_order && group_blocks == -1) {
1172
- if (last) {
1241
+ if constexpr (w_type.size_bits() == 8) {
1242
if (s_sh_wr_pred) {
1243
cp_async4(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]);
1244
}
1245
cp_async_fence();
1246
+ } else {
1247
+ // For 4-bit per-column scales, we only fetch them here in the
1248
+ // final step before write-out
1249
+ if (last) {
1250
+ if (s_sh_wr_pred) {
1251
+ cp_async4(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]);
1252
+ }
1253
+ cp_async_fence();
1254
+ }
1255
}
1256
}
1257
1258
thread_block_reduce();
1259
if constexpr (!has_act_order && group_blocks == -1) {
1182
- if (last) {
1260
+ if constexpr (w_type.size_bits() == 8) {
1261
cp_async_wait<0>();
1262
__syncthreads();
1263
if (threadIdx.x / 32 < thread_n_blocks / 4) {
1264
reinterpret_cast<int4*>(&frag_s)[0] = sh_s[s_sh_rd + 0];
1265
reinterpret_cast<int4*>(&frag_s)[1] = sh_s[s_sh_rd + 4];
1266
}
1267
+
1268
+ } else {
1269
+ if (last) {
1270
+ cp_async_wait<0>();
1271
+ __syncthreads();
1272
+ if (threadIdx.x / 32 < thread_n_blocks / 4) {
1273
+ reinterpret_cast<int4*>(&frag_s)[0] = sh_s[s_sh_rd + 0];
1274
+ reinterpret_cast<int4*>(&frag_s)[1] = sh_s[s_sh_rd + 4];
1275
+ }
1276
+ }
1277
}
1278
}
1279
+
1280
+ // For 8-bit channelwise, we apply the scale before the global reduction
1281
+ // that converts the fp32 results to fp16 (so that we avoid possible
1282
+ // overflow in fp16)
1283
+ if constexpr (!has_act_order && group_blocks == -1 &&
1284
+ w_type.size_bits() == 8) {
1285
+ if (threadIdx.x / 32 < thread_n_blocks / 4) {
1286
+ #pragma unroll
1287
+ for (int i = 0; i < thread_m_blocks; i++) {
1288
+ #pragma unroll
1289
+ for (int j = 0; j < 4; j++) {
1290
+ scale_float(reinterpret_cast<float*>(&frag_c[i][j][0][0]),
1291
+ frag_s[j / 2][2 * (j % 2) + 0]);
1292
+ scale_float(reinterpret_cast<float*>(&frag_c[i][j][0][2]),
1293
+ frag_s[j / 2][2 * (j % 2) + 0]);
1294
+
1295
+ scale_float(reinterpret_cast<float*>(&frag_c[i][j][1][0]),
1296
+ frag_s[j / 2][2 * (j % 2) + 1]);
1297
+ scale_float(reinterpret_cast<float*>(&frag_c[i][j][1][2]),
1298
+ frag_s[j / 2][2 * (j % 2) + 1]);
1299
+ }
1300
+ }
1301
+ }
1302
+ }
1303
+
1304
if (slice_count > 1) { // only globally reduce if there is more than one
1305
// block in a slice
1306
barrier_acquire(&locks[slice_col], slice_idx);
@@ -1227,7 +1340,8 @@ __device__ inline void MarlinMoESingle(
1340
}
1341
}
1342
1230
-template <const int threads, // number of threads in a threadblock
1343
+template <const vllm::ScalarTypeId w_type_id, // weight ScalarType id
1344
+ const int threads, // number of threads in a threadblock
1345
const int thread_m_blocks, // number of 16x16 blocks in the m
1346
// dimension (batchsize) of the
1347
// threadblock
@@ -1261,7 +1375,8 @@ __global__ void MarlinMoE(
1375
bool replicate_input, // do we use the same input for each expert?
1376
bool apply_weights, // apply weights to output
1377
int current_m_block, // current m block to start kernel computation from
1264
- int max_par // maximum parallelism
1378
+ int max_par, // maximum parallelism
1379
+ int cfg_max_m_blocks // upper bound on m blocks
1380
) {
1381
int m_block_ctr = current_m_block;
1382
@@ -1282,40 +1397,40 @@ __global__ void MarlinMoE(
1397
prob_m = tot_its - 16 * m_block_ctr;
1398
1399
int par = 1;
1285
- if (max_block > 4) {
1400
+ if (max_block > cfg_max_m_blocks) {
1401
// Note that parallel > 1 currently only works for inputs without any
1402
// padding
1288
- par = (16 * max_block - pad) / 64;
1289
- par = min((16 * max_block - pad) / 64, max_par);
1290
- prob_m = 64 * par;
1291
- m_block_ctr += 4 * (par - 1);
1292
- max_block = 4;
1403
+ par = (16 * max_block - pad) / (16 * cfg_max_m_blocks);
1404
+ if (par > max_par) par = max_par;
1405
+ prob_m = (16 * cfg_max_m_blocks) * par;
1406
+ m_block_ctr += cfg_max_m_blocks * (par - 1);
1407
+ max_block = cfg_max_m_blocks;
1408
}
1409
1410
if (max_block == 1) {
1296
- MarlinMoESingle<threads, 1, thread_n_blocks, thread_k_blocks, stages,
1297
- has_act_order, group_blocks>(
1411
+ MarlinMoESingle<w_type_id, threads, 1, thread_n_blocks, thread_k_blocks,
1412
+ stages, has_act_order, group_blocks>(
1413
A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1414
expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1415
prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1416
current_m_block);
1417
} else if (max_block == 2) {
1303
- MarlinMoESingle<threads, 2, thread_n_blocks, thread_k_blocks, stages,
1304
- has_act_order, group_blocks>(
1418
+ MarlinMoESingle<w_type_id, threads, 2, thread_n_blocks, thread_k_blocks,
1419
+ stages, has_act_order, group_blocks>(
1420
A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1421
expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1422
prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1423
current_m_block);
1424
} else if (max_block == 3) {
1310
- MarlinMoESingle<threads, 3, thread_n_blocks, thread_k_blocks, stages,
1311
- has_act_order, group_blocks>(
1425
+ MarlinMoESingle<w_type_id, threads, 3, thread_n_blocks, thread_k_blocks,
1426
+ stages, has_act_order, group_blocks>(
1427
A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1428
expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1429
prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1430
current_m_block);
1431
} else {
1317
- MarlinMoESingle<threads, 4, thread_n_blocks, thread_k_blocks, stages,
1318
- has_act_order, group_blocks>(
1432
+ MarlinMoESingle<w_type_id, threads, 4, thread_n_blocks, thread_k_blocks,
1433
+ stages, has_act_order, group_blocks>(
1434
A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1435
expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1436
prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
@@ -1342,7 +1457,8 @@ __global__ void compute_expert_offsets(int const* __restrict__ topk_ids,
1457
return;
1458
}
1459
1345
-template <const int threads, // number of threads in a threadblock
1460
+template <const vllm::ScalarTypeId w_type_id, // weight ScalarType id
1461
+ const int threads, // number of threads in a threadblock
1462
const int thread_m_blocks, // number of 16x16 blocks in the m
1463
// dimension (batchsize) of the
1464
// threadblock
@@ -1376,7 +1492,9 @@ __global__ void MarlinMoE(
1492
bool replicate_input, // do we use the same input for each expert?
1493
bool apply_weights, // apply weights to output
1494
int current_m_block, // current m block to start kernel computation from
1379
- int max_par // maximum parallelism
1495
+ int max_par, // maximum parallelism
1496
+ int cfg_max_m_blocks // upper bound on m blocks
1497
+
1498
) {
1499
// Marlin is not implemented yet for SM < 8.0
1500
assert(false);
@@ -1397,24 +1515,26 @@ const int STAGES = 4; // 4 pipeline stages fit into shared memory
1515
static constexpr int min_thread_n = 64;
1516
static constexpr int min_thread_k = 64;
1517
1400
-#define __CALL_IF_MOE(THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1401
- HAS_ACT_ORDER, GROUP_BLOCKS, NUM_THREADS) \
1402
- else if (thread_m_blocks == THREAD_M_BLOCKS && \
1518
+#define __CALL_IF_MOE(W_TYPE, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1519
+ THREAD_K_BLOCKS, HAS_ACT_ORDER, GROUP_BLOCKS, \
1520
+ NUM_THREADS) \
1521
+ else if (q_type == W_TYPE && thread_m_blocks == THREAD_M_BLOCKS && \
1522
thread_n_blocks == THREAD_N_BLOCKS && \
1523
thread_k_blocks == THREAD_K_BLOCKS && \
1524
has_act_order == HAS_ACT_ORDER && group_blocks == GROUP_BLOCKS && \
1525
num_threads == NUM_THREADS) { \
1526
cudaFuncSetAttribute( \
1408
- MarlinMoE<NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1527
+ MarlinMoE<W_TYPE.id(), NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1528
THREAD_K_BLOCKS, STAGES, HAS_ACT_ORDER, GROUP_BLOCKS>, \
1529
cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
1411
- MarlinMoE<NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1412
- STAGES, HAS_ACT_ORDER, GROUP_BLOCKS> \
1530
+ MarlinMoE<W_TYPE.id(), NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1531
+ THREAD_K_BLOCKS, STAGES, HAS_ACT_ORDER, GROUP_BLOCKS> \
1532
<<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
1533
A_ptr, B_ptr, C_ptr, sorted_ids_ptr, topk_weights_ptr, s_ptr, \
1534
g_idx_ptr, expert_offsets_ptr, num_groups, expert_idx, \
1535
num_experts, topk, prob_m, prob_n, prob_k, tot_m, locks, \
1417
- replicate_input, apply_weights, m_block, max_par); \
1536
+ replicate_input, apply_weights, m_block, max_par, \
1537
+ exec_cfg.max_m_blocks); \
1538
}
1539
1540
typedef struct {
@@ -1423,6 +1543,11 @@ typedef struct {
1543
int num_threads;
1544
} thread_config_t;
1545
1546
+typedef struct {
1547
+ int max_m_blocks;
1548
+ thread_config_t tb_cfg;
1549
+} exec_config_t;
1550
+
1551
thread_config_t small_batch_thread_configs[] = {
1552
// Ordered by priority
1553
@@ -1443,8 +1568,77 @@ thread_config_t large_batch_thread_configs[] = {
1568
{128, 64, 128}, // Reduce N 4X, increase K 2X
1569
};
1570
1446
-bool is_valid_config(thread_config_t const& th_config, int prob_m, int prob_n,
1447
- int prob_k) {
1571
+int get_scales_cache_size(thread_config_t const& th_config, int prob_m,
1572
+ int prob_n, int prob_k, int num_bits, int group_size,
1573
+ bool has_act_order, bool is_k_full) {
1574
+ bool cache_scales_chunk = has_act_order && !is_k_full;
1575
+
1576
+ int tb_n = th_config.thread_n;
1577
+ int tb_k = th_config.thread_k;
1578
+
1579
+ // Get max scale groups per thread-block
1580
+ int tb_groups;
1581
+ if (group_size == -1) {
1582
+ tb_groups = 1;
1583
+ } else if (group_size == 0) {
1584
+ tb_groups = ceildiv(tb_k, 32); // Worst case is 32 group size
1585
+ } else {
1586
+ tb_groups = ceildiv(tb_k, group_size);
1587
+ }
1588
+
1589
+ if (cache_scales_chunk) {
1590
+ int load_groups =
1591
+ tb_groups * STAGES * 2; // Chunk size is 2x pipeline over dim K
1592
+ load_groups = max(load_groups, 32); // We load at least 32 scale groups
1593
+ return load_groups * tb_n * 2;
1594
+
1595
+ } else {
1596
+ int tb_scales = tb_groups * tb_n * 2;
1597
+
1598
+ return tb_scales * STAGES;
1599
+ }
1600
+}
1601
+
1602
+bool is_valid_cache_size(thread_config_t const& th_config, int max_m_blocks,
1603
+ int prob_m, int prob_n, int prob_k, int num_bits,
1604
+ int scales_cache_size, int max_shared_mem) {
1605
+ int pack_factor = 32 / num_bits;
1606
+
1607
+ // Get B size
1608
+ int tb_k = th_config.thread_k;
1609
+ int tb_n = th_config.thread_n;
1610
+
1611
+ int b_size = (tb_k * tb_n / pack_factor) * 4;
1612
+
1613
+ // Get A size
1614
+ int m_blocks = ceildiv(prob_m, 16);
1615
+ int tb_max_m = 16;
1616
+
1617
+ while (true) {
1618
+ if (m_blocks >= max_m_blocks) {
1619
+ tb_max_m *= max_m_blocks;
1620
+ break;
1621
+ }
1622
+
1623
+ max_m_blocks--;
1624
+ if (max_m_blocks == 0) {
1625
+ TORCH_CHECK(false, "Unexpected m_blocks = ", m_blocks);
1626
+ }
1627
+ }
1628
+
1629
+ int a_size = (tb_max_m * tb_k) * 2;
1630
+
1631
+ float pipe_size = (a_size + b_size) * STAGES;
1632
+
1633
+ TORCH_CHECK(max_shared_mem / 2 > scales_cache_size); // Sanity
1634
+
1635
+ return pipe_size < 0.95f * (max_shared_mem - scales_cache_size);
1636
+}
1637
+
1638
+bool is_valid_config(thread_config_t const& th_config, int max_m_blocks,
1639
+ int prob_m, int prob_n, int prob_k, int num_bits,
1640
+ int group_size, bool has_act_order, bool is_k_full,
1641
+ int max_shared_mem) {
1642
// Sanity
1643
if (th_config.thread_k == -1 || th_config.thread_n == -1 ||
1644
th_config.num_threads == -1) {
@@ -1472,64 +1666,88 @@ bool is_valid_config(thread_config_t const& th_config, int prob_m, int prob_n,
1666
return false;
1667
}
1668
1669
+ // Determine cache for scales
1670
+ int scales_cache_size =
1671
+ get_scales_cache_size(th_config, prob_m, prob_n, prob_k, num_bits,
1672
+ group_size, has_act_order, is_k_full);
1673
+
1674
+ // Check that pipeline fits into cache
1675
+ if (!is_valid_cache_size(th_config, max_m_blocks, prob_m, prob_n, prob_k,
1676
+ num_bits, scales_cache_size, max_shared_mem)) {
1677
+ return false;
1678
+ }
1679
+
1680
return true;
1681
}
1682
1478
-thread_config_t determine_thread_config(int prob_m, int prob_n, int prob_k) {
1479
- if (prob_m <= 16) {
1480
- for (auto th_config : small_batch_thread_configs) {
1481
- if (is_valid_config(th_config, prob_m, prob_n, prob_k)) {
1482
- return th_config;
1683
+exec_config_t determine_thread_config(int prob_m, int prob_n, int prob_k,
1684
+ int num_bits, int group_size,
1685
+ bool has_act_order, bool is_k_full,
1686
+ int max_shared_mem) {
1687
+ int max_m_blocks = 4;
1688
+ while (max_m_blocks > 0) {
1689
+ if (prob_m <= 16) {
1690
+ for (auto th_config : small_batch_thread_configs) {
1691
+ if (is_valid_config(th_config, max_m_blocks, prob_m, prob_n, prob_k,
1692
+ num_bits, group_size, has_act_order, is_k_full,
1693
+ max_shared_mem)) {
1694
+ return exec_config_t{max_m_blocks, th_config};
1695
+ }
1696
}
1484
- }
1485
-
1486
- } else {
1487
- for (auto th_config : large_batch_thread_configs) {
1488
- if (is_valid_config(th_config, prob_m, prob_n, prob_k)) {
1489
- return th_config;
1697
+ } else {
1698
+ for (auto th_config : large_batch_thread_configs) {
1699
+ if (is_valid_config(th_config, max_m_blocks, prob_m, prob_n, prob_k,
1700
+ num_bits, group_size, has_act_order, is_k_full,
1701
+ max_shared_mem)) {
1702
+ return exec_config_t{max_m_blocks, th_config};
1703
+ }
1704
}
1705
}
1706
+
1707
+ max_m_blocks--; // Process less M blocks per invocation to reduce cache
1708
+ // usage
1709
}
1710
1494
- return thread_config_t{-1, -1, -1};
1711
+ return exec_config_t{0, {-1, -1, -1}};
1712
}
1713
1497
-#define CALL_IF_MOE(N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1498
- __CALL_IF_MOE(1, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1499
- __CALL_IF_MOE(2, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1500
- __CALL_IF_MOE(3, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1501
- __CALL_IF_MOE(4, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1502
- \
1503
- __CALL_IF_MOE(1, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1504
- __CALL_IF_MOE(1, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1505
- __CALL_IF_MOE(1, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1506
- __CALL_IF_MOE(1, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1507
- \
1508
- __CALL_IF_MOE(2, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1509
- __CALL_IF_MOE(2, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1510
- __CALL_IF_MOE(2, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1511
- __CALL_IF_MOE(2, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1512
- \
1513
- __CALL_IF_MOE(3, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1514
- __CALL_IF_MOE(3, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1515
- __CALL_IF_MOE(3, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1516
- __CALL_IF_MOE(3, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1517
- \
1518
- __CALL_IF_MOE(4, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1519
- __CALL_IF_MOE(4, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1520
- __CALL_IF_MOE(4, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1521
- __CALL_IF_MOE(4, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS)
1714
+#define CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1715
+ __CALL_IF_MOE(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1716
+ __CALL_IF_MOE(W_TYPE, 2, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1717
+ __CALL_IF_MOE(W_TYPE, 3, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1718
+ __CALL_IF_MOE(W_TYPE, 4, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1719
+ \
1720
+ __CALL_IF_MOE(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1721
+ __CALL_IF_MOE(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1722
+ __CALL_IF_MOE(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1723
+ __CALL_IF_MOE(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1724
+ \
1725
+ __CALL_IF_MOE(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1726
+ __CALL_IF_MOE(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1727
+ __CALL_IF_MOE(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1728
+ __CALL_IF_MOE(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1729
+ \
1730
+ __CALL_IF_MOE(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1731
+ __CALL_IF_MOE(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1732
+ __CALL_IF_MOE(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1733
+ __CALL_IF_MOE(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1734
+ \
1735
+ __CALL_IF_MOE(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1736
+ __CALL_IF_MOE(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1737
+ __CALL_IF_MOE(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1738
+ __CALL_IF_MOE(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS)
1739
1740
void marlin_mm_moe_f16i4(const void* A, const void* B, void* C,
1741
const void* sorted_ids, const void* topk_weights,
1742
const void* topk_ids, const void* s, const void* g_idx,
1743
const void* perm, void* a_tmp, void* expert_offsets,
1744
int prob_m, int prob_n, int prob_k, void* workspace,
1528
- bool has_act_order, bool is_k_full, int num_groups,
1529
- int group_size, int num_experts, int topk,
1530
- int moe_block_size, int dev, cudaStream_t stream,
1531
- int thread_k, int thread_n, int sms, int max_par,
1532
- bool replicate_input, bool apply_weights) {
1745
+ vllm::ScalarType const& q_type, bool has_act_order,
1746
+ bool is_k_full, int num_groups, int group_size,
1747
+ int num_experts, int topk, int moe_block_size, int dev,
1748
+ cudaStream_t stream, int thread_k, int thread_n,
1749
+ int sms, int max_par, bool replicate_input,
1750
+ bool apply_weights) {
1751
TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m,
1752
", ", prob_n, ", ", prob_k, "]");
1753
@@ -1537,26 +1755,42 @@ void marlin_mm_moe_f16i4(const void* A, const void* B, void* C,
1755
cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev);
1756
}
1757
1758
+ int max_shared_mem = 0;
1759
+ cudaDeviceGetAttribute(&max_shared_mem,
1760
+ cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
1761
+ TORCH_CHECK(max_shared_mem > 0);
1762
+
1763
+ int num_bits = q_type.size_bits();
1764
+
1765
// Set thread config
1541
- thread_config_t th_config;
1766
+ exec_config_t exec_cfg;
1767
if (thread_k != -1 && thread_n != -1) {
1768
// User-defined config
1544
- th_config = thread_config_t{thread_k, thread_n, USER_THREADS};
1769
+ exec_cfg =
1770
+ exec_config_t{4, thread_config_t{thread_k, thread_n, USER_THREADS}};
1771
} else {
1772
// Auto config
1547
- th_config = determine_thread_config(prob_m, prob_n, prob_k);
1773
+ exec_cfg =
1774
+ determine_thread_config(prob_m, prob_n, prob_k, num_bits, group_size,
1775
+ has_act_order, is_k_full, max_shared_mem);
1776
}
1777
1550
- TORCH_CHECK(is_valid_config(th_config, prob_m, prob_n, prob_k),
1551
- "Invalid thread config: thread_k = " + str(th_config.thread_k) +
1552
- ", thread_n = " + str(th_config.thread_n) +
1553
- ", num_threads = " + str(th_config.num_threads) +
1554
- " for MKN = [" + str(prob_m) + ", " + str(prob_k) + ", " +
1555
- str(prob_n) + "]");
1556
-
1557
- int num_threads = th_config.num_threads;
1558
- thread_k = th_config.thread_k;
1559
- thread_n = th_config.thread_n;
1778
+ TORCH_CHECK(exec_cfg.max_m_blocks > 0 &&
1779
+ is_valid_config(exec_cfg.tb_cfg, exec_cfg.max_m_blocks,
1780
+ prob_m, prob_n, prob_k, num_bits, group_size,
1781
+ has_act_order, is_k_full, max_shared_mem),
1782
+ "Invalid thread config: max_m_blocks = ", exec_cfg.max_m_blocks,
1783
+ ", thread_k = ", exec_cfg.tb_cfg.thread_k,
1784
+ ", thread_n = ", exec_cfg.tb_cfg.thread_n,
1785
+ ", num_threads = ", exec_cfg.tb_cfg.num_threads, " for MKN = [",
1786
+ prob_m, ", ", prob_k, ", ", prob_n, "] and num_bits = ", num_bits,
1787
+ ", group_size = ", group_size,
1788
+ ", has_act_order = ", has_act_order, ", is_k_full = ", is_k_full,
1789
+ ", max_shared_mem = ", max_shared_mem);
1790
+
1791
+ int num_threads = exec_cfg.tb_cfg.num_threads;
1792
+ thread_k = exec_cfg.tb_cfg.thread_k;
1793
+ thread_n = exec_cfg.tb_cfg.thread_n;
1794
1795
int thread_k_blocks = thread_k / 16;
1796
int thread_n_blocks = thread_n / 16;
@@ -1590,11 +1824,6 @@ void marlin_mm_moe_f16i4(const void* A, const void* B, void* C,
1824
}
1825
}
1826
1593
- int max_shared_mem = 0;
1594
- cudaDeviceGetAttribute(&max_shared_mem,
1595
- cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
1596
- TORCH_CHECK(max_shared_mem > 0);
1597
-
1827
int tot_m = prob_m;
1828
1829
const int* topk_ids_ptr = (const int*)topk_ids;
@@ -1611,10 +1840,13 @@ void marlin_mm_moe_f16i4(const void* A, const void* B, void* C,
1840
has_act_order = false;
1841
}
1842
1843
+ int pack_factor = 32 / q_type.size_bits();
1844
+
1845
for (int expert_idx = 0; expert_idx < num_experts; ++expert_idx) {
1846
const int4* A_ptr = (const int4*)A;
1847
int4* a_tmp_ptr = (int4*)a_tmp;
1617
- const int4* B_ptr = (const int4*)B + (prob_n * prob_k / 32) * expert_idx;
1848
+ const int4* B_ptr =
1849
+ (const int4*)B + (prob_n * prob_k / (pack_factor * 4)) * expert_idx;
1850
int4* C_ptr = (int4*)C;
1851
const float* topk_weights_ptr = (const float*)topk_weights;
1852
const int* sorted_ids_ptr = (const int*)sorted_ids;
@@ -1636,19 +1868,22 @@ void marlin_mm_moe_f16i4(const void* A, const void* B, void* C,
1868
A_ptr = a_tmp_ptr;
1869
}
1870
1639
- int max_m_blocks = ceildiv(tot_m, 16);
1640
- for (int m_block = 0; m_block < max_m_blocks; m_block += 16) {
1641
- // Define kernel configurations
1642
-
1871
+ int tot_m_blocks = ceildiv(tot_m, 16);
1872
+ for (int m_block = 0; m_block < tot_m_blocks;
1873
+ m_block += 4 * exec_cfg.max_m_blocks) {
1874
// make it max possible value
1644
- int thread_m_blocks = 4;
1875
+ int thread_m_blocks = exec_cfg.max_m_blocks;
1876
1877
if (false) {
1878
}
1648
- CALL_IF_MOE(16, 4, 256)
1649
- CALL_IF_MOE(8, 8, 256)
1650
- CALL_IF_MOE(8, 4, 128)
1651
- CALL_IF_MOE(4, 8, 128)
1879
+ CALL_IF_MOE(vllm::kU4B8, 16, 4, 256)
1880
+ CALL_IF_MOE(vllm::kU4B8, 8, 8, 256)
1881
+ CALL_IF_MOE(vllm::kU4B8, 8, 4, 128)
1882
+ CALL_IF_MOE(vllm::kU4B8, 4, 8, 128)
1883
+ CALL_IF_MOE(vllm::kU8B128, 16, 4, 256)
1884
+ CALL_IF_MOE(vllm::kU8B128, 8, 8, 256)
1885
+ CALL_IF_MOE(vllm::kU8B128, 8, 4, 128)
1886
+ CALL_IF_MOE(vllm::kU8B128, 4, 8, 128)
1887
else {
1888
TORCH_CHECK(false, "Unsupported shapes: MNK = [" + str(prob_m) + ", " +
1889
str(prob_n) + ", " + str(prob_k) + "]" +
@@ -1670,9 +1905,15 @@ torch::Tensor marlin_gemm_moe(
1905
const torch::Tensor& sorted_ids, const torch::Tensor& topk_weights,
1906
const torch::Tensor& topk_ids, const torch::Tensor& b_scales,
1907
const torch::Tensor& g_idx, const torch::Tensor& perm,
1673
- torch::Tensor& workspace, int64_t size_m, int64_t size_n, int64_t size_k,
1674
- bool is_k_full, int64_t num_experts, int64_t topk, int64_t moe_block_size,
1908
+ torch::Tensor& workspace, vllm::ScalarTypeTorchPtr const& b_q_type,
1909
+ int64_t size_m, int64_t size_n, int64_t size_k, bool is_k_full,
1910
+ int64_t num_experts, int64_t topk, int64_t moe_block_size,
1911
bool replicate_input, bool apply_weights) {
1912
+ TORCH_CHECK(*b_q_type == vllm::kU4B8 || *b_q_type == vllm::kU8B128,
1913
+ "b_q_type must be uint4b8 or uint8b128. Got = ", b_q_type->str());
1914
+
1915
+ int pack_factor = 32 / b_q_type->size_bits();
1916
+
1917
int max_par = 4;
1918
1919
int dev = a.get_device();
@@ -1733,8 +1974,8 @@ torch::Tensor marlin_gemm_moe(
1974
topk_weights.data_ptr(), topk_ids.data_ptr(), b_scales.data_ptr(),
1975
g_idx.data_ptr(), perm.data_ptr(), a_tmp.data_ptr(),
1976
expert_offsets.data_ptr(), size_m, size_n, size_k, workspace.data_ptr(),
1736
- has_act_order, is_k_full, num_groups, group_size, num_experts, topk,
1737
- moe_block_size, dev, at::cuda::getCurrentCUDAStream(dev), thread_k,
1977
+ *b_q_type, has_act_order, is_k_full, num_groups, group_size, num_experts,
1978
+ topk, moe_block_size, dev, at::cuda::getCurrentCUDAStream(dev), thread_k,
1979
thread_n, sms, max_par, replicate_input, apply_weights);
1980
return c;
1981
}
csrc/moe/marlin_moe_ops.h
+5
-2
@@ -2,11 +2,14 @@
2
3
#include <torch/all.h>
4
5
+#include "core/scalar_type.hpp"
6
+
7
torch::Tensor marlin_gemm_moe(
8
const torch::Tensor& a, const torch::Tensor& b_q_weights,
9
const torch::Tensor& sorted_ids, const torch::Tensor& topk_weights,
10
const torch::Tensor& topk_ids, const torch::Tensor& b_scales,
11
const torch::Tensor& g_idx, const torch::Tensor& perm,
10
- torch::Tensor& workspace, int64_t size_m, int64_t size_n, int64_t size_k,
11
- bool is_k_full, int64_t num_experts, int64_t topk, int64_t moe_block_size,
12
+ torch::Tensor& workspace, vllm::ScalarTypeTorchPtr const& b_q_type,
13
+ int64_t size_m, int64_t size_n, int64_t size_k, bool is_k_full,
14
+ int64_t num_experts, int64_t topk, int64_t moe_block_size,
15
bool replicate_input, bool apply_weights);
csrc/moe/torch_bindings.cpp
+5
-3
@@ -13,9 +13,11 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
13
m.def(
14
"marlin_gemm_moe(Tensor! a, Tensor! b_q_weights, Tensor! sorted_ids, "
15
"Tensor! topk_weights, Tensor! topk_ids, Tensor! b_scales, Tensor! "
16
- "g_idx, Tensor! perm, Tensor! workspace, int size_m, int size_n, int "
17
- "size_k, bool is_k_full, int num_experts, int topk, int moe_block_size, "
18
- "bool replicate_input, bool apply_weights) -> Tensor");
16
+ "g_idx, Tensor! perm, Tensor! workspace, "
17
+ "__torch__.torch.classes._core_C.ScalarType b_q_type, int size_m, "
18
+ "int size_n, int size_k, bool is_k_full, int num_experts, int topk, "
19
+ "int moe_block_size, bool replicate_input, bool apply_weights)"
20
+ " -> Tensor");
21
m.impl("marlin_gemm_moe", torch::kCUDA, &marlin_gemm_moe);
22
#endif
23
}
tests/kernels/test_moe.py
+12
-6
@@ -140,6 +140,7 @@ def compute_max_diff(output, output_ref):
140
@pytest.mark.parametrize("topk", [2, 6])
141
@pytest.mark.parametrize("group_size", [-1, 32, 64, 128])
142
@pytest.mark.parametrize("act_order", [True, False])
143
+@pytest.mark.parametrize("num_bits", [4, 8])
144
def test_fused_marlin_moe(
145
m: int,
146
n: int,
@@ -148,6 +149,7 @@ def test_fused_marlin_moe(
149
topk: int,
150
group_size: int,
151
act_order: bool,
152
+ num_bits: int,
153
):
154
torch.manual_seed(7)
155
@@ -161,13 +163,12 @@ def test_fused_marlin_moe(
163
if group_size in (k, n):
164
return
165
164
- quant_type = scalar_types.uint4b8
166
+ quant_type = (scalar_types.uint4b8
167
+ if num_bits == 4 else scalar_types.uint8b128)
168
dtype = torch.float16
169
a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
170
w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10
171
w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10
169
- for i in range(w2.shape[0]):
170
- w2[0] = torch.eye(k, n, device="cuda", dtype=dtype)
172
173
w_ref1_l = []
174
qweight1_l = []
@@ -240,6 +241,7 @@ def test_fused_marlin_moe(
241
topk_ids,
242
w1_scale=scales1,
243
w2_scale=scales2,
244
+ num_bits=num_bits,
245
)
246
247
assert compute_max_diff(marlin_output, triton_output) < 4e-2
@@ -254,7 +256,8 @@ def test_fused_marlin_moe(
256
@pytest.mark.parametrize("topk", [2, 6])
257
@pytest.mark.parametrize("group_size", [-1, 32, 64, 128])
258
@pytest.mark.parametrize("act_order", [True, False])
257
-def test_marlin_moe_mmm(
259
+@pytest.mark.parametrize("num_bits", [4, 8])
260
+def test_single_marlin_moe_multiply(
261
m: int,
262
n: int,
263
k: int,
@@ -262,6 +265,7 @@ def test_marlin_moe_mmm(
265
topk: int,
266
group_size: int,
267
act_order: bool,
268
+ num_bits: int,
269
):
270
if topk > e:
271
return
@@ -273,7 +277,8 @@ def test_marlin_moe_mmm(
277
if group_size == k:
278
return
279
276
- quant_type = scalar_types.uint4b8
280
+ quant_type = (scalar_types.uint4b8
281
+ if num_bits == 4 else scalar_types.uint8b128)
282
dtype = torch.float16
283
a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
284
w = torch.randn((e, n, k), device="cuda", dtype=dtype) / 10
@@ -308,7 +313,8 @@ def test_marlin_moe_mmm(
313
g_idx,
314
sort_indices,
315
topk,
311
- renormalize=False)
316
+ renormalize=False,
317
+ num_bits=num_bits)
318
torch_output = torch_moe_single(a, w_ref.transpose(1, 2), score, topk)
319
320
assert compute_max_diff(marlin_output, torch_output) < 1e-2
tests/weight_loading/models-large.txt
+2
-1
@@ -1,3 +1,4 @@
1
compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-quantized, main
2
compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-channel-quantized, main
3
-gptq_marlin, TheBloke/Mixtral-8x7B-v0.1-GPTQ, main
\ No newline at end of file
3
+compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W8A16-quantized, main
4
+gptq_marlin, TheBloke/Mixtral-8x7B-v0.1-GPTQ, main
tests/weight_loading/run_model_weight_loading_test.sh
vllm/_custom_ops.py
+1
-1
@@ -559,7 +559,7 @@ def gptq_marlin_moe_repack(b_q_weight: torch.Tensor, perm: torch.Tensor,
559
num_bits: int) -> torch.Tensor:
560
num_experts = b_q_weight.shape[0]
561
assert size_k % 16 == 0
562
- output = torch.empty((num_experts, size_k // 16, size_n * 2),
562
+ output = torch.empty((num_experts, size_k // 16, size_n * (num_bits // 2)),
563
device=b_q_weight.device,
564
dtype=b_q_weight.dtype)
565
for e in range(num_experts):
vllm/model_executor/layers/fused_moe/fused_marlin_moe.py
+30
-14
@@ -7,18 +7,21 @@ import torch
7
from vllm import _custom_ops as ops
8
from vllm.model_executor.layers.fused_moe.fused_moe import (
9
fused_topk, moe_align_block_size, try_get_optimal_moe_config)
10
+from vllm.scalar_type import scalar_types
11
12
13
def single_marlin_moe(
13
- hidden_states: torch.Tensor,
14
- w: torch.Tensor,
15
- scales: torch.Tensor,
16
- gating_output: torch.Tensor,
17
- g_idx: torch.Tensor,
18
- perm: torch.Tensor,
19
- topk: int,
20
- renormalize: bool,
21
- override_config: Optional[Dict[str, Any]] = None) -> torch.Tensor:
14
+ hidden_states: torch.Tensor,
15
+ w: torch.Tensor,
16
+ scales: torch.Tensor,
17
+ gating_output: torch.Tensor,
18
+ g_idx: torch.Tensor,
19
+ perm: torch.Tensor,
20
+ topk: int,
21
+ renormalize: bool,
22
+ override_config: Optional[Dict[str, Any]] = None,
23
+ num_bits: int = 8,
24
+) -> torch.Tensor:
25
"""
26
This function computes the multiplication of hidden_states with expert
27
weights used in Marlin MoE, using weights w and top-k gating mechanism.
@@ -36,6 +39,7 @@ def single_marlin_moe(
39
- renormalize (bool): If True, renormalize the top-k weights to sum to 1.
40
- override_config (Optional[Dict[str, Any]]): Optional override
41
for the kernel configuration.
42
+ - num_bits (bool): The number of bits in expert weights quantization.
43
44
Returns:
45
- torch.Tensor: The output tensor after applying the MoE layer.
@@ -48,10 +52,11 @@ def single_marlin_moe(
52
assert hidden_states.is_contiguous(), "Hidden_states must be contiguous"
53
assert w.is_contiguous(), "Expert weights must be contiguous"
54
assert hidden_states.dtype == torch.float16
55
+ assert num_bits in [4, 8]
56
57
M, K = hidden_states.shape
58
E = w.shape[0]
54
- N = w.shape[2] // 2
59
+ N = w.shape[2] // (num_bits // 2)
60
61
topk_weights, topk_ids = fused_topk(hidden_states, gating_output, topk,
62
renormalize)
@@ -76,10 +81,13 @@ def single_marlin_moe(
81
device="cuda",
82
requires_grad=False)
83
84
+ scalar_type = (scalar_types.uint4b8
85
+ if num_bits == 4 else scalar_types.uint8b128)
86
+
87
intermediate_cache = torch.ops._moe_C.marlin_gemm_moe(
88
hidden_states, w, sorted_token_ids, topk_weights, topk_ids, scales,
81
- g_idx, perm, workspace, M, N, K, True, E, topk, block_size_m, True,
82
- False)
89
+ g_idx, perm, workspace, scalar_type, M, N, K, True, E, topk,
90
+ block_size_m, True, False)
91
92
return torch.sum(intermediate_cache.view(*intermediate_cache.shape), dim=1)
93
@@ -98,6 +106,7 @@ def fused_marlin_moe(
106
override_config: Optional[Dict[str, Any]] = None,
107
w1_scale: Optional[torch.Tensor] = None,
108
w2_scale: Optional[torch.Tensor] = None,
109
+ num_bits: int = 8,
110
) -> torch.Tensor:
111
"""
112
This function computes a Mixture of Experts (MoE) layer using two sets of
@@ -122,6 +131,7 @@ def fused_marlin_moe(
131
w1.
132
- w2_scale (Optional[torch.Tensor]): Optional scale to be used for
133
w2.
134
+ - num_bits (bool): The number of bits in expert weights quantization.
135
136
Returns:
137
- torch.Tensor: The output tensor after applying the MoE layer.
@@ -131,13 +141,14 @@ def fused_marlin_moe(
141
0], "Number of tokens mismatch"
142
assert hidden_states.shape[
143
1] == w1.shape[1] * 16, "Hidden size mismatch w1"
134
- assert hidden_states.shape[
135
- 1] == w2.shape[2] // 2, "Hidden size mismatch w2"
144
+ assert hidden_states.shape[1] == w2.shape[2] // (
145
+ num_bits // 2), "Hidden size mismatch w2"
146
assert gating_output.shape[1] == w1.shape[0], "Number of experts mismatch"
147
assert hidden_states.is_contiguous(), "Hidden_states must be contiguous"
148
assert w1.is_contiguous(), "Expert weights1 must be contiguous"
149
assert w2.is_contiguous(), "Expert weights2 must be contiguous"
150
assert hidden_states.dtype == torch.float16
151
+ assert num_bits in [4, 8]
152
153
M, K = hidden_states.shape
154
E = w1.shape[0]
@@ -165,6 +176,9 @@ def fused_marlin_moe(
176
device="cuda",
177
requires_grad=False)
178
179
+ scalar_type = (scalar_types.uint4b8
180
+ if num_bits == 4 else scalar_types.uint8b128)
181
+
182
intermediate_cache2 = torch.empty(
183
(M * topk_ids.shape[1], N),
184
device=hidden_states.device,
@@ -181,6 +195,7 @@ def fused_marlin_moe(
195
g_idx1,
196
perm1,
197
workspace,
198
+ scalar_type,
199
M,
200
2 * N,
201
K,
@@ -204,6 +219,7 @@ def fused_marlin_moe(
219
g_idx2,
220
perm2,
221
workspace,
222
+ scalar_type,
223
M,
224
K,
225
N,
vllm/model_executor/layers/fused_moe/fused_moe.py
+1
-1
@@ -445,7 +445,7 @@ def grouped_topk(hidden_states: torch.Tensor,
445
if renormalize:
446
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
447
448
- return topk_weights, topk_ids.to(torch.int32)
448
+ return topk_weights.to(torch.float32), topk_ids.to(torch.int32)
449
450
451
def get_config_dtype_str(dtype: torch.dtype,
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+6
-2
@@ -6,6 +6,8 @@ import torch
6
7
from vllm import _custom_ops as ops
8
from vllm.model_executor.layers.fused_moe import FusedMoE, FusedMoEMethodBase
9
+from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
10
+ WNA16_SUPPORTED_BITS)
11
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
12
CompressionFormat)
13
from vllm.model_executor.utils import set_weight_attrs
@@ -38,10 +40,11 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
40
41
if not (self.quant_config.quant_format
42
== CompressionFormat.pack_quantized.value
41
- and self.num_bits == 4):
43
+ and self.num_bits in WNA16_SUPPORTED_BITS):
44
raise ValueError("For Fused MoE layers, only ",
45
f"{CompressionFormat.pack_quantized.value} ",
44
- "is supported for 4 bits")
46
+ "is supported for the following bits: ",
47
+ f"{WNA16_SUPPORTED_BITS}")
48
49
def create_weights(self, layer: torch.nn.Module, num_experts: int,
50
hidden_size: int, intermediate_size: int,
@@ -292,4 +295,5 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
295
topk_ids,
296
w1_scale=layer.w13_weight_scale,
297
w2_scale=layer.w2_weight_scale,
298
+ num_bits=self.num_bits,
299
)
vllm/model_executor/layers/quantization/gptq_marlin.py
+1
@@ -611,4 +611,5 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
611
topk_ids,
612
w1_scale=layer.w13_scales,
613
w2_scale=layer.w2_scales,
614
+ num_bits=self.quant_config.quant_type.size_bits,
615
).to(orig_dtype)
vllm/model_executor/model_loader/utils.py
+1
-7
@@ -23,13 +23,7 @@ def get_model_architecture(
23
architectures = getattr(model_config.hf_config, "architectures", [])
24
# Special handling for quantized Mixtral.
25
# FIXME(woosuk): This is a temporary hack.
26
- mixtral_supported = ["fp8", "compressed-tensors"]
27
- # for gptq_marlin, only run fused MoE for int4
28
- if model_config.quantization == "gptq_marlin":
29
- hf_quant_config = getattr(model_config.hf_config,
30
- "quantization_config", None)
31
- if hf_quant_config and hf_quant_config.get("bits") == 4:
32
- mixtral_supported.append("gptq_marlin")
26
+ mixtral_supported = ["fp8", "compressed-tensors", "gptq_marlin"]
27
28
if (model_config.quantization is not None
29
and model_config.quantization not in mixtral_supported