@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 95460fc513
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Kernel] port sgl moe_align_block_size kernels (#12574)

sgl_moe_align_block_size is based on: https://github.com/sgl-project/sglang/commit/ded9fcd09a43d5e7d5bb31a2bc3e9fc21bf65d2a moe_align_block_size is based on: https://github.com/sgl-project/sglang/commit/ba5112ff691d791a9e38c6c71f59324a5fcb49d0 Signed-off-by: Yang Chen <yangche@fb.com>

Yang Chen committed Feb 2, 2025 at 21:09 UTC 95460fc51318702a33226b87152afe810187e01e
6 files changed +284 -3
csrc/moe/moe_align_sum_kernels.cu
+92
@@ -197,6 +197,72 @@ __global__ void moe_align_block_size_global_mem_kernel(
197 }
198 }
199
200 +// taken from
201 +// https://github.com/sgl-project/sglang/commit/ded9fcd09a43d5e7d5bb31a2bc3e9fc21bf65d2a
202 +template <typename scalar_t>
203 +__global__ void sgl_moe_align_block_size_kernel(
204 + scalar_t* __restrict__ topk_ids, int32_t* sorted_token_ids,
205 + int32_t* expert_ids, int32_t* total_tokens_post_pad, int32_t num_experts,
206 + int32_t block_size, size_t numel, int32_t* cumsum) {
207 + __shared__ int32_t shared_counts[32][8];
208 + __shared__ int32_t local_offsets[256];
209 +
210 + const int warp_id = threadIdx.x / WARP_SIZE;
211 + const int lane_id = threadIdx.x % WARP_SIZE;
212 + const int experts_per_warp = 8;
213 + const int my_expert_start = warp_id * experts_per_warp;
214 +
215 + for (int i = 0; i < experts_per_warp; ++i) {
216 + if (my_expert_start + i < num_experts) {
217 + shared_counts[warp_id][i] = 0;
218 + }
219 + }
220 +
221 + const size_t tokens_per_thread = CEILDIV(numel, blockDim.x);
222 + const size_t start_idx = threadIdx.x * tokens_per_thread;
223 +
224 + for (int i = start_idx; i < numel && i < start_idx + tokens_per_thread; ++i) {
225 + int expert_id = topk_ids[i];
226 + int warp_idx = expert_id / experts_per_warp;
227 + int expert_offset = expert_id % experts_per_warp;
228 + atomicAdd(&shared_counts[warp_idx][expert_offset], 1);
229 + }
230 +
231 + __syncthreads();
232 +
233 + if (threadIdx.x == 0) {
234 + cumsum[0] = 0;
235 + for (int i = 1; i <= num_experts; ++i) {
236 + int expert_count = 0;
237 + int warp_idx = (i - 1) / experts_per_warp;
238 + int expert_offset = (i - 1) % experts_per_warp;
239 + expert_count = shared_counts[warp_idx][expert_offset];
240 +
241 + cumsum[i] =
242 + cumsum[i - 1] + CEILDIV(expert_count, block_size) * block_size;
243 + }
244 + *total_tokens_post_pad = cumsum[num_experts];
245 + }
246 +
247 + __syncthreads();
248 +
249 + if (threadIdx.x < num_experts) {
250 + for (int i = cumsum[threadIdx.x]; i < cumsum[threadIdx.x + 1];
251 + i += block_size) {
252 + expert_ids[i / block_size] = threadIdx.x;
253 + }
254 + local_offsets[threadIdx.x] = cumsum[threadIdx.x];
255 + }
256 +
257 + __syncthreads();
258 +
259 + for (int i = start_idx; i < numel && i < start_idx + tokens_per_thread; ++i) {
260 + int32_t expert_id = topk_ids[i];
261 + int32_t rank_post_pad = atomicAdd(&local_offsets[expert_id], 1);
262 + sorted_token_ids[rank_post_pad] = i;
263 + }
264 +}
265 +
266 template <typename scalar_t, int TOPK>
267 __global__ void moe_sum_kernel(
268 scalar_t* __restrict__ out, // [..., d]
@@ -305,6 +371,32 @@ void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
371 }
372 }
373
374 +void sgl_moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
375 + int64_t block_size,
376 + torch::Tensor sorted_token_ids,
377 + torch::Tensor experts_ids,
378 + torch::Tensor num_tokens_post_pad) {
379 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
380 + VLLM_DISPATCH_INTEGRAL_TYPES(
381 + topk_ids.scalar_type(), "sgl_moe_align_block_size_kernel", [&] {
382 + // calc needed amount of shared mem for `tokens_cnts` and `cumsum`
383 + // tensors
384 + auto options_int =
385 + torch::TensorOptions().dtype(torch::kInt).device(topk_ids.device());
386 + // torch::Tensor token_cnts_buffer =
387 + // torch::empty({(num_experts + 1) * num_experts}, options_int);
388 + torch::Tensor cumsum_buffer =
389 + torch::empty({num_experts + 1}, options_int);
390 +
391 + auto kernel = vllm::moe::sgl_moe_align_block_size_kernel<scalar_t>;
392 + kernel<<<1, 1024, 0, stream>>>(
393 + topk_ids.data_ptr<scalar_t>(), sorted_token_ids.data_ptr<int32_t>(),
394 + experts_ids.data_ptr<int32_t>(),
395 + num_tokens_post_pad.data_ptr<int32_t>(), num_experts, block_size,
396 + topk_ids.numel(), cumsum_buffer.data_ptr<int32_t>());
397 + });
398 +}
399 +
400 void moe_sum(torch::Tensor& input, // [num_tokens, topk, hidden_size]
401 torch::Tensor& output) // [num_tokens, hidden_size]
402 {
csrc/moe/moe_ops.h
+6
@@ -12,3 +12,9 @@ void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
12 int64_t block_size, torch::Tensor sorted_token_ids,
13 torch::Tensor experts_ids,
14 torch::Tensor num_tokens_post_pad);
15 +
16 +void sgl_moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
17 + int64_t block_size,
18 + torch::Tensor sorted_token_ids,
19 + torch::Tensor experts_ids,
20 + torch::Tensor num_tokens_post_pad);
csrc/moe/torch_bindings.cpp
+9
@@ -22,6 +22,15 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
22 " Tensor! num_tokens_post_pad) -> ()");
23 m.impl("moe_align_block_size", torch::kCUDA, &moe_align_block_size);
24
25 + // temporarily adapted from
26 + // https://github.com/sgl-project/sglang/commit/ded9fcd09a43d5e7d5bb31a2bc3e9fc21bf65d2a
27 + m.def(
28 + "sgl_moe_align_block_size(Tensor topk_ids, int num_experts,"
29 + " int block_size, Tensor! sorted_token_ids,"
30 + " Tensor! experts_ids,"
31 + " Tensor! num_tokens_post_pad) -> ()");
32 + m.impl("sgl_moe_align_block_size", torch::kCUDA, &sgl_moe_align_block_size);
33 +
34 #ifndef USE_ROCM
35 m.def(
36 "marlin_gemm_moe(Tensor! a, Tensor! b_q_weights, Tensor! sorted_ids, "
vllm/_custom_ops.py
+9
@@ -952,6 +952,15 @@ def moe_align_block_size(topk_ids: torch.Tensor, num_experts: int,
952 num_tokens_post_pad)
953
954
955 +def sgl_moe_align_block_size(topk_ids: torch.Tensor, num_experts: int,
956 + block_size: int, sorted_token_ids: torch.Tensor,
957 + experts_ids: torch.Tensor,
958 + num_tokens_post_pad: torch.Tensor) -> None:
959 + torch.ops._moe_C.sgl_moe_align_block_size(topk_ids, num_experts,
960 + block_size, sorted_token_ids,
961 + experts_ids, num_tokens_post_pad)
962 +
963 +
964 def topk_softmax(topk_weights: torch.Tensor, topk_ids: torch.Tensor,
965 token_expert_indicies: torch.Tensor,
966 gating_output: float) -> None:
vllm/envs.py
+8 -1
@@ -82,6 +82,7 @@ if TYPE_CHECKING:
82 VLLM_MLA_DISABLE: bool = False
83 VLLM_MLA_PERFORM_MATRIX_ABSORPTION: bool = True
84 VLLM_MLA_DISABLE_REQUANTIZATION: bool = False
85 + VLLM_ENABLE_MOE_ALIGN_BLOCK_SIZE_TRITON: bool = False
86
87
88 def get_default_cache_root():
@@ -531,7 +532,13 @@ environment_variables: Dict[str, Callable[[], Any]] = {
532 # matrices to match the activation type. This can lead to higher memory and
533 # compute usage but better preserves the accuracy of the original model.
534 "VLLM_MLA_DISABLE_REQUANTIZATION":
534 - lambda: bool(int(os.getenv("VLLM_MLA_DISABLE_REQUANTIZATION", "0")))
535 + lambda: bool(int(os.getenv("VLLM_MLA_DISABLE_REQUANTIZATION", "0"))),
536 +
537 + # If set, vLLM will use the Triton implementation of moe_align_block_size,
538 + # i.e. moe_align_block_size_triton in fused_moe.py.
539 + "VLLM_ENABLE_MOE_ALIGN_BLOCK_SIZE_TRITON":
540 + lambda: bool(int(os.getenv("VLLM_ENABLE_MOE_ALIGN_BLOCK_SIZE_TRITON", "0"))
541 + ),
542 }
543
544 # end-env-vars-definition
vllm/model_executor/layers/fused_moe/fused_moe.py
+160 -2
@@ -405,6 +405,144 @@ def fused_moe_kernel(
405 tl.store(c_ptrs, accumulator, mask=c_mask)
406
407
408 +def ceil_div(a, b):
409 + return (a + b - 1) // b
410 +
411 +
412 +@triton.jit
413 +def moe_align_block_size_stage1(
414 + topk_ids_ptr,
415 + tokens_cnts_ptr,
416 + num_experts: tl.constexpr,
417 + numel: tl.constexpr,
418 + tokens_per_thread: tl.constexpr,
419 +):
420 + pid = tl.program_id(0)
421 +
422 + start_idx = pid * tokens_per_thread
423 +
424 + off_c = (pid + 1) * num_experts
425 +
426 + for i in range(tokens_per_thread):
427 + if start_idx + i < numel:
428 + idx = tl.load(topk_ids_ptr + start_idx + i)
429 + token_cnt = tl.load(tokens_cnts_ptr + off_c + idx)
430 + tl.store(tokens_cnts_ptr + off_c + idx, token_cnt + 1)
431 +
432 +
433 +@triton.jit
434 +def moe_align_block_size_stage2(
435 + tokens_cnts_ptr,
436 + num_experts: tl.constexpr,
437 +):
438 + pid = tl.program_id(0)
439 +
440 + last_cnt = 0
441 + for i in range(1, num_experts + 1):
442 + token_cnt = tl.load(tokens_cnts_ptr + i * num_experts + pid)
443 + last_cnt = last_cnt + token_cnt
444 + tl.store(tokens_cnts_ptr + i * num_experts + pid, last_cnt)
445 +
446 +
447 +@triton.jit
448 +def moe_align_block_size_stage3(
449 + total_tokens_post_pad_ptr,
450 + tokens_cnts_ptr,
451 + cumsum_ptr,
452 + num_experts: tl.constexpr,
453 + block_size: tl.constexpr,
454 +):
455 + last_cumsum = 0
456 + off_cnt = num_experts * num_experts
457 + for i in range(1, num_experts + 1):
458 + token_cnt = tl.load(tokens_cnts_ptr + off_cnt + i - 1)
459 + last_cumsum = last_cumsum + tl.cdiv(token_cnt, block_size) * block_size
460 + tl.store(cumsum_ptr + i, last_cumsum)
461 + tl.store(total_tokens_post_pad_ptr, last_cumsum)
462 +
463 +
464 +@triton.jit
465 +def moe_align_block_size_stage4(
466 + topk_ids_ptr,
467 + sorted_token_ids_ptr,
468 + expert_ids_ptr,
469 + tokens_cnts_ptr,
470 + cumsum_ptr,
471 + num_experts: tl.constexpr,
472 + block_size: tl.constexpr,
473 + numel: tl.constexpr,
474 + tokens_per_thread: tl.constexpr,
475 +):
476 + pid = tl.program_id(0)
477 + start_idx = tl.load(cumsum_ptr + pid)
478 + end_idx = tl.load(cumsum_ptr + pid + 1)
479 +
480 + for i in range(start_idx, end_idx, block_size):
481 + tl.store(expert_ids_ptr + i // block_size, pid)
482 +
483 + start_idx = pid * tokens_per_thread
484 + off_t = pid * num_experts
485 +
486 + for i in range(start_idx, tl.minimum(start_idx + tokens_per_thread,
487 + numel)):
488 + expert_id = tl.load(topk_ids_ptr + i)
489 + token_cnt = tl.load(tokens_cnts_ptr + off_t + expert_id)
490 + rank_post_pad = token_cnt + tl.load(cumsum_ptr + expert_id)
491 + tl.store(sorted_token_ids_ptr + rank_post_pad, i)
492 + tl.store(tokens_cnts_ptr + off_t + expert_id, token_cnt + 1)
493 +
494 +
495 +# Triton implementation based on:
496 +# https://github.com/sgl-project/sglang/commit/ba5112ff691d791a9e38c6c71f59324a5fcb49d0
497 +def moe_align_block_size_triton(
498 + topk_ids: torch.Tensor,
499 + num_experts: int,
500 + block_size: int,
501 + sorted_token_ids: torch.Tensor,
502 + expert_ids: torch.Tensor,
503 + num_tokens_post_pad: torch.Tensor,
504 +) -> None:
505 + numel = topk_ids.numel()
506 + grid = (num_experts, )
507 + tokens_cnts = torch.zeros((num_experts + 1, num_experts),
508 + dtype=torch.int32,
509 + device=topk_ids.device)
510 + cumsum = torch.zeros((num_experts + 1, ),
511 + dtype=torch.int32,
512 + device=topk_ids.device)
513 + tokens_per_thread = ceil_div(numel, num_experts)
514 +
515 + moe_align_block_size_stage1[grid](
516 + topk_ids,
517 + tokens_cnts,
518 + num_experts,
519 + numel,
520 + tokens_per_thread,
521 + )
522 + moe_align_block_size_stage2[grid](
523 + tokens_cnts,
524 + num_experts,
525 + )
526 + moe_align_block_size_stage3[(1, )](
527 + num_tokens_post_pad,
528 + tokens_cnts,
529 + cumsum,
530 + num_experts,
531 + block_size,
532 + )
533 + moe_align_block_size_stage4[grid](
534 + topk_ids,
535 + sorted_token_ids,
536 + expert_ids,
537 + tokens_cnts,
538 + cumsum,
539 + num_experts,
540 + block_size,
541 + numel,
542 + tokens_per_thread,
543 + )
544 +
545 +
546 def moe_align_block_size(
547 topk_ids: torch.Tensor, block_size: int,
548 num_experts: int) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
@@ -457,8 +595,28 @@ def moe_align_block_size(
595 num_tokens_post_pad = torch.empty((1),
596 dtype=torch.int32,
597 device=topk_ids.device)
460 - ops.moe_align_block_size(topk_ids, num_experts, block_size, sorted_ids,
461 - expert_ids, num_tokens_post_pad)
598 + if num_experts >= 224:
599 + if envs.VLLM_ENABLE_MOE_ALIGN_BLOCK_SIZE_TRITON:
600 + moe_align_block_size_triton(
601 + topk_ids,
602 + num_experts,
603 + block_size,
604 + sorted_ids,
605 + expert_ids,
606 + num_tokens_post_pad,
607 + )
608 + else:
609 + ops.sgl_moe_align_block_size(
610 + topk_ids,
611 + num_experts,
612 + block_size,
613 + sorted_ids,
614 + expert_ids,
615 + num_tokens_post_pad,
616 + )
617 + else:
618 + ops.moe_align_block_size(topk_ids, num_experts, block_size, sorted_ids,
619 + expert_ids, num_tokens_post_pad)
620 return sorted_ids, expert_ids, num_tokens_post_pad
621
622