A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Kernel] port sgl moe_align_block_size kernels (#12574)
sgl_moe_align_block_size is based on: https://github.com/sgl-project/sglang/commit/ded9fcd09a43d5e7d5bb31a2bc3e9fc21bf65d2a moe_align_block_size is based on: https://github.com/sgl-project/sglang/commit/ba5112ff691d791a9e38c6c71f59324a5fcb49d0 Signed-off-by: Yang Chen <yangche@fb.com>
Yang Chen committed
Feb 2, 2025 at 21:09 UTC
95460fc51318702a33226b87152afe810187e01e
6 files changed
+284
-3
csrc/moe/moe_align_sum_kernels.cu
+92
@@ -197,6 +197,72 @@ __global__ void moe_align_block_size_global_mem_kernel(
197
}
198
}
199
200
+// taken from
201
+// https://github.com/sgl-project/sglang/commit/ded9fcd09a43d5e7d5bb31a2bc3e9fc21bf65d2a
202
+template <typename scalar_t>
203
+__global__ void sgl_moe_align_block_size_kernel(
204
+ scalar_t* __restrict__ topk_ids, int32_t* sorted_token_ids,
205
+ int32_t* expert_ids, int32_t* total_tokens_post_pad, int32_t num_experts,
206
+ int32_t block_size, size_t numel, int32_t* cumsum) {
207
+ __shared__ int32_t shared_counts[32][8];
208
+ __shared__ int32_t local_offsets[256];
209
+
210
+ const int warp_id = threadIdx.x / WARP_SIZE;
211
+ const int lane_id = threadIdx.x % WARP_SIZE;
212
+ const int experts_per_warp = 8;
213
+ const int my_expert_start = warp_id * experts_per_warp;
214
+
215
+ for (int i = 0; i < experts_per_warp; ++i) {
216
+ if (my_expert_start + i < num_experts) {
217
+ shared_counts[warp_id][i] = 0;
218
+ }
219
+ }
220
+
221
+ const size_t tokens_per_thread = CEILDIV(numel, blockDim.x);
222
+ const size_t start_idx = threadIdx.x * tokens_per_thread;
223
+
224
+ for (int i = start_idx; i < numel && i < start_idx + tokens_per_thread; ++i) {
225
+ int expert_id = topk_ids[i];
226
+ int warp_idx = expert_id / experts_per_warp;
227
+ int expert_offset = expert_id % experts_per_warp;
228
+ atomicAdd(&shared_counts[warp_idx][expert_offset], 1);
229
+ }
230
+
231
+ __syncthreads();
232
+
233
+ if (threadIdx.x == 0) {
234
+ cumsum[0] = 0;
235
+ for (int i = 1; i <= num_experts; ++i) {
236
+ int expert_count = 0;
237
+ int warp_idx = (i - 1) / experts_per_warp;
238
+ int expert_offset = (i - 1) % experts_per_warp;
239
+ expert_count = shared_counts[warp_idx][expert_offset];
240
+
241
+ cumsum[i] =
242
+ cumsum[i - 1] + CEILDIV(expert_count, block_size) * block_size;
243
+ }
244
+ *total_tokens_post_pad = cumsum[num_experts];
245
+ }
246
+
247
+ __syncthreads();
248
+
249
+ if (threadIdx.x < num_experts) {
250
+ for (int i = cumsum[threadIdx.x]; i < cumsum[threadIdx.x + 1];
251
+ i += block_size) {
252
+ expert_ids[i / block_size] = threadIdx.x;
253
+ }
254
+ local_offsets[threadIdx.x] = cumsum[threadIdx.x];
255
+ }
256
+
257
+ __syncthreads();
258
+
259
+ for (int i = start_idx; i < numel && i < start_idx + tokens_per_thread; ++i) {
260
+ int32_t expert_id = topk_ids[i];
261
+ int32_t rank_post_pad = atomicAdd(&local_offsets[expert_id], 1);
262
+ sorted_token_ids[rank_post_pad] = i;
263
+ }
264
+}
265
+
266
template <typename scalar_t, int TOPK>
267
__global__ void moe_sum_kernel(
268
scalar_t* __restrict__ out, // [..., d]
@@ -305,6 +371,32 @@ void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
371
}
372
}
373
374
+void sgl_moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
375
+ int64_t block_size,
376
+ torch::Tensor sorted_token_ids,
377
+ torch::Tensor experts_ids,
378
+ torch::Tensor num_tokens_post_pad) {
379
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
380
+ VLLM_DISPATCH_INTEGRAL_TYPES(
381
+ topk_ids.scalar_type(), "sgl_moe_align_block_size_kernel", [&] {
382
+ // calc needed amount of shared mem for `tokens_cnts` and `cumsum`
383
+ // tensors
384
+ auto options_int =
385
+ torch::TensorOptions().dtype(torch::kInt).device(topk_ids.device());
386
+ // torch::Tensor token_cnts_buffer =
387
+ // torch::empty({(num_experts + 1) * num_experts}, options_int);
388
+ torch::Tensor cumsum_buffer =
389
+ torch::empty({num_experts + 1}, options_int);
390
+
391
+ auto kernel = vllm::moe::sgl_moe_align_block_size_kernel<scalar_t>;
392
+ kernel<<<1, 1024, 0, stream>>>(
393
+ topk_ids.data_ptr<scalar_t>(), sorted_token_ids.data_ptr<int32_t>(),
394
+ experts_ids.data_ptr<int32_t>(),
395
+ num_tokens_post_pad.data_ptr<int32_t>(), num_experts, block_size,
396
+ topk_ids.numel(), cumsum_buffer.data_ptr<int32_t>());
397
+ });
398
+}
399
+
400
void moe_sum(torch::Tensor& input, // [num_tokens, topk, hidden_size]
401
torch::Tensor& output) // [num_tokens, hidden_size]
402
{
csrc/moe/moe_ops.h
+6
@@ -12,3 +12,9 @@ void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
12
int64_t block_size, torch::Tensor sorted_token_ids,
13
torch::Tensor experts_ids,
14
torch::Tensor num_tokens_post_pad);
15
+
16
+void sgl_moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
17
+ int64_t block_size,
18
+ torch::Tensor sorted_token_ids,
19
+ torch::Tensor experts_ids,
20
+ torch::Tensor num_tokens_post_pad);
csrc/moe/torch_bindings.cpp
+9
@@ -22,6 +22,15 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
22
" Tensor! num_tokens_post_pad) -> ()");
23
m.impl("moe_align_block_size", torch::kCUDA, &moe_align_block_size);
24
25
+ // temporarily adapted from
26
+ // https://github.com/sgl-project/sglang/commit/ded9fcd09a43d5e7d5bb31a2bc3e9fc21bf65d2a
27
+ m.def(
28
+ "sgl_moe_align_block_size(Tensor topk_ids, int num_experts,"
29
+ " int block_size, Tensor! sorted_token_ids,"
30
+ " Tensor! experts_ids,"
31
+ " Tensor! num_tokens_post_pad) -> ()");
32
+ m.impl("sgl_moe_align_block_size", torch::kCUDA, &sgl_moe_align_block_size);
33
+
34
#ifndef USE_ROCM
35
m.def(
36
"marlin_gemm_moe(Tensor! a, Tensor! b_q_weights, Tensor! sorted_ids, "
vllm/_custom_ops.py
+9
@@ -952,6 +952,15 @@ def moe_align_block_size(topk_ids: torch.Tensor, num_experts: int,
952
num_tokens_post_pad)
953
954
955
+def sgl_moe_align_block_size(topk_ids: torch.Tensor, num_experts: int,
956
+ block_size: int, sorted_token_ids: torch.Tensor,
957
+ experts_ids: torch.Tensor,
958
+ num_tokens_post_pad: torch.Tensor) -> None:
959
+ torch.ops._moe_C.sgl_moe_align_block_size(topk_ids, num_experts,
960
+ block_size, sorted_token_ids,
961
+ experts_ids, num_tokens_post_pad)
962
+
963
+
964
def topk_softmax(topk_weights: torch.Tensor, topk_ids: torch.Tensor,
965
token_expert_indicies: torch.Tensor,
966
gating_output: float) -> None:
vllm/envs.py
+8
-1
@@ -82,6 +82,7 @@ if TYPE_CHECKING:
82
VLLM_MLA_DISABLE: bool = False
83
VLLM_MLA_PERFORM_MATRIX_ABSORPTION: bool = True
84
VLLM_MLA_DISABLE_REQUANTIZATION: bool = False
85
+ VLLM_ENABLE_MOE_ALIGN_BLOCK_SIZE_TRITON: bool = False
86
87
88
def get_default_cache_root():
@@ -531,7 +532,13 @@ environment_variables: Dict[str, Callable[[], Any]] = {
532
# matrices to match the activation type. This can lead to higher memory and
533
# compute usage but better preserves the accuracy of the original model.
534
"VLLM_MLA_DISABLE_REQUANTIZATION":
534
- lambda: bool(int(os.getenv("VLLM_MLA_DISABLE_REQUANTIZATION", "0")))
535
+ lambda: bool(int(os.getenv("VLLM_MLA_DISABLE_REQUANTIZATION", "0"))),
536
+
537
+ # If set, vLLM will use the Triton implementation of moe_align_block_size,
538
+ # i.e. moe_align_block_size_triton in fused_moe.py.
539
+ "VLLM_ENABLE_MOE_ALIGN_BLOCK_SIZE_TRITON":
540
+ lambda: bool(int(os.getenv("VLLM_ENABLE_MOE_ALIGN_BLOCK_SIZE_TRITON", "0"))
541
+ ),
542
}
543
544
# end-env-vars-definition
vllm/model_executor/layers/fused_moe/fused_moe.py
+160
-2
@@ -405,6 +405,144 @@ def fused_moe_kernel(
405
tl.store(c_ptrs, accumulator, mask=c_mask)
406
407
408
+def ceil_div(a, b):
409
+ return (a + b - 1) // b
410
+
411
+
412
+@triton.jit
413
+def moe_align_block_size_stage1(
414
+ topk_ids_ptr,
415
+ tokens_cnts_ptr,
416
+ num_experts: tl.constexpr,
417
+ numel: tl.constexpr,
418
+ tokens_per_thread: tl.constexpr,
419
+):
420
+ pid = tl.program_id(0)
421
+
422
+ start_idx = pid * tokens_per_thread
423
+
424
+ off_c = (pid + 1) * num_experts
425
+
426
+ for i in range(tokens_per_thread):
427
+ if start_idx + i < numel:
428
+ idx = tl.load(topk_ids_ptr + start_idx + i)
429
+ token_cnt = tl.load(tokens_cnts_ptr + off_c + idx)
430
+ tl.store(tokens_cnts_ptr + off_c + idx, token_cnt + 1)
431
+
432
+
433
+@triton.jit
434
+def moe_align_block_size_stage2(
435
+ tokens_cnts_ptr,
436
+ num_experts: tl.constexpr,
437
+):
438
+ pid = tl.program_id(0)
439
+
440
+ last_cnt = 0
441
+ for i in range(1, num_experts + 1):
442
+ token_cnt = tl.load(tokens_cnts_ptr + i * num_experts + pid)
443
+ last_cnt = last_cnt + token_cnt
444
+ tl.store(tokens_cnts_ptr + i * num_experts + pid, last_cnt)
445
+
446
+
447
+@triton.jit
448
+def moe_align_block_size_stage3(
449
+ total_tokens_post_pad_ptr,
450
+ tokens_cnts_ptr,
451
+ cumsum_ptr,
452
+ num_experts: tl.constexpr,
453
+ block_size: tl.constexpr,
454
+):
455
+ last_cumsum = 0
456
+ off_cnt = num_experts * num_experts
457
+ for i in range(1, num_experts + 1):
458
+ token_cnt = tl.load(tokens_cnts_ptr + off_cnt + i - 1)
459
+ last_cumsum = last_cumsum + tl.cdiv(token_cnt, block_size) * block_size
460
+ tl.store(cumsum_ptr + i, last_cumsum)
461
+ tl.store(total_tokens_post_pad_ptr, last_cumsum)
462
+
463
+
464
+@triton.jit
465
+def moe_align_block_size_stage4(
466
+ topk_ids_ptr,
467
+ sorted_token_ids_ptr,
468
+ expert_ids_ptr,
469
+ tokens_cnts_ptr,
470
+ cumsum_ptr,
471
+ num_experts: tl.constexpr,
472
+ block_size: tl.constexpr,
473
+ numel: tl.constexpr,
474
+ tokens_per_thread: tl.constexpr,
475
+):
476
+ pid = tl.program_id(0)
477
+ start_idx = tl.load(cumsum_ptr + pid)
478
+ end_idx = tl.load(cumsum_ptr + pid + 1)
479
+
480
+ for i in range(start_idx, end_idx, block_size):
481
+ tl.store(expert_ids_ptr + i // block_size, pid)
482
+
483
+ start_idx = pid * tokens_per_thread
484
+ off_t = pid * num_experts
485
+
486
+ for i in range(start_idx, tl.minimum(start_idx + tokens_per_thread,
487
+ numel)):
488
+ expert_id = tl.load(topk_ids_ptr + i)
489
+ token_cnt = tl.load(tokens_cnts_ptr + off_t + expert_id)
490
+ rank_post_pad = token_cnt + tl.load(cumsum_ptr + expert_id)
491
+ tl.store(sorted_token_ids_ptr + rank_post_pad, i)
492
+ tl.store(tokens_cnts_ptr + off_t + expert_id, token_cnt + 1)
493
+
494
+
495
+# Triton implementation based on:
496
+# https://github.com/sgl-project/sglang/commit/ba5112ff691d791a9e38c6c71f59324a5fcb49d0
497
+def moe_align_block_size_triton(
498
+ topk_ids: torch.Tensor,
499
+ num_experts: int,
500
+ block_size: int,
501
+ sorted_token_ids: torch.Tensor,
502
+ expert_ids: torch.Tensor,
503
+ num_tokens_post_pad: torch.Tensor,
504
+) -> None:
505
+ numel = topk_ids.numel()
506
+ grid = (num_experts, )
507
+ tokens_cnts = torch.zeros((num_experts + 1, num_experts),
508
+ dtype=torch.int32,
509
+ device=topk_ids.device)
510
+ cumsum = torch.zeros((num_experts + 1, ),
511
+ dtype=torch.int32,
512
+ device=topk_ids.device)
513
+ tokens_per_thread = ceil_div(numel, num_experts)
514
+
515
+ moe_align_block_size_stage1[grid](
516
+ topk_ids,
517
+ tokens_cnts,
518
+ num_experts,
519
+ numel,
520
+ tokens_per_thread,
521
+ )
522
+ moe_align_block_size_stage2[grid](
523
+ tokens_cnts,
524
+ num_experts,
525
+ )
526
+ moe_align_block_size_stage3[(1, )](
527
+ num_tokens_post_pad,
528
+ tokens_cnts,
529
+ cumsum,
530
+ num_experts,
531
+ block_size,
532
+ )
533
+ moe_align_block_size_stage4[grid](
534
+ topk_ids,
535
+ sorted_token_ids,
536
+ expert_ids,
537
+ tokens_cnts,
538
+ cumsum,
539
+ num_experts,
540
+ block_size,
541
+ numel,
542
+ tokens_per_thread,
543
+ )
544
+
545
+
546
def moe_align_block_size(
547
topk_ids: torch.Tensor, block_size: int,
548
num_experts: int) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
@@ -457,8 +595,28 @@ def moe_align_block_size(
595
num_tokens_post_pad = torch.empty((1),
596
dtype=torch.int32,
597
device=topk_ids.device)
460
- ops.moe_align_block_size(topk_ids, num_experts, block_size, sorted_ids,
461
- expert_ids, num_tokens_post_pad)
598
+ if num_experts >= 224:
599
+ if envs.VLLM_ENABLE_MOE_ALIGN_BLOCK_SIZE_TRITON:
600
+ moe_align_block_size_triton(
601
+ topk_ids,
602
+ num_experts,
603
+ block_size,
604
+ sorted_ids,
605
+ expert_ids,
606
+ num_tokens_post_pad,
607
+ )
608
+ else:
609
+ ops.sgl_moe_align_block_size(
610
+ topk_ids,
611
+ num_experts,
612
+ block_size,
613
+ sorted_ids,
614
+ expert_ids,
615
+ num_tokens_post_pad,
616
+ )
617
+ else:
618
+ ops.moe_align_block_size(topk_ids, num_experts, block_size, sorted_ids,
619
+ expert_ids, num_tokens_post_pad)
620
return sorted_ids, expert_ids, num_tokens_post_pad
621
622