@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 59449095ab
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Performance][Kernel] Fused_moe Performance Improvement (#9384)

Signed-off-by: charlifu <charlifu@amd.com>

Charlie Fu committed Oct 24, 2024 at 17:37 UTC 59449095ab536febe9ff341b2a88a4fed572a70f
9 files changed +118 -38
CMakeLists.txt
+1 -1
@@ -195,7 +195,6 @@ set(VLLM_EXT_SRC
195 "csrc/quantization/compressed_tensors/int8_quant_kernels.cu"
196 "csrc/quantization/fp8/common.cu"
197 "csrc/cuda_utils_kernels.cu"
198 - "csrc/moe_align_block_size_kernels.cu"
198 "csrc/prepare_inputs/advance_step.cu"
199 "csrc/torch_bindings.cpp")
200
@@ -405,6 +404,7 @@ target_compile_definitions(_C PRIVATE CUTLASS_ENABLE_DIRECT_CUDA_DRIVER_CALL=1)
404
405 set(VLLM_MOE_EXT_SRC
406 "csrc/moe/torch_bindings.cpp"
407 + "csrc/moe/moe_align_sum_kernels.cu"
408 "csrc/moe/topk_softmax_kernels.cu")
409
410 set_gencode_flags_for_srcs(
csrc/moe/moe_align_sum_kernels.cu renamed
+83 -15
@@ -1,15 +1,17 @@
1 #include <torch/all.h>
2 #include <ATen/cuda/CUDAContext.h>
3 +#include <c10/cuda/CUDAGuard.h>
4
5 #include <ATen/ATen.h>
6 #include <THC/THCAtomics.cuh>
7
7 -#include "cuda_compat.h"
8 -#include "dispatch_utils.h"
8 +#include "../cuda_compat.h"
9 +#include "../dispatch_utils.h"
10
11 #define CEILDIV(x, y) (((x) + (y) - 1) / (y))
12
13 namespace vllm {
14 +namespace moe {
15
16 namespace {
17 __device__ __forceinline__ int32_t index(int32_t total_col, int32_t row,
@@ -32,10 +34,10 @@ __global__ void moe_align_block_size_kernel(scalar_t* __restrict__ topk_ids,
34 extern __shared__ int32_t shared_mem[];
35
36 int32_t* tokens_cnts =
35 - shared_mem; // 2d tensor with shape (num_experts + 1, num_experts)
37 + shared_mem; // 2d tensor with shape (blockDim.x + 1, num_experts)
38 int32_t* cumsum =
37 - shared_mem + (num_experts + 1) *
38 - num_experts; // 1d tensor with shape (num_experts + 1)
39 + shared_mem +
40 + (blockDim.x + 1) * num_experts; // 1d tensor with shape (num_experts + 1)
41
42 for (int i = 0; i < num_experts; ++i) {
43 tokens_cnts[index(num_experts, threadIdx.x + 1, i)] = 0;
@@ -53,10 +55,12 @@ __global__ void moe_align_block_size_kernel(scalar_t* __restrict__ topk_ids,
55 __syncthreads();
56
57 // For each expert we accumulate the token counts from the different threads.
56 - tokens_cnts[index(num_experts, 0, threadIdx.x)] = 0;
57 - for (int i = 1; i <= blockDim.x; ++i) {
58 - tokens_cnts[index(num_experts, i, threadIdx.x)] +=
59 - tokens_cnts[index(num_experts, i - 1, threadIdx.x)];
58 + if (threadIdx.x < num_experts) {
59 + tokens_cnts[index(num_experts, 0, threadIdx.x)] = 0;
60 + for (int i = 1; i <= blockDim.x; ++i) {
61 + tokens_cnts[index(num_experts, i, threadIdx.x)] +=
62 + tokens_cnts[index(num_experts, i - 1, threadIdx.x)];
63 + }
64 }
65
66 __syncthreads();
@@ -79,9 +83,11 @@ __global__ void moe_align_block_size_kernel(scalar_t* __restrict__ topk_ids,
83 * For each expert, each thread processes the tokens of the corresponding
84 * blocks and stores the corresponding expert_id for each block.
85 */
82 - for (int i = cumsum[threadIdx.x]; i < cumsum[threadIdx.x + 1];
83 - i += block_size) {
84 - expert_ids[i / block_size] = threadIdx.x;
86 + if (threadIdx.x < num_experts) {
87 + for (int i = cumsum[threadIdx.x]; i < cumsum[threadIdx.x + 1];
88 + i += block_size) {
89 + expert_ids[i / block_size] = threadIdx.x;
90 + }
91 }
92
93 /**
@@ -106,6 +112,24 @@ __global__ void moe_align_block_size_kernel(scalar_t* __restrict__ topk_ids,
112 ++tokens_cnts[index(num_experts, threadIdx.x, expert_id)];
113 }
114 }
115 +
116 +template <typename scalar_t, int TOPK>
117 +__global__ void moe_sum_kernel(
118 + scalar_t* __restrict__ out, // [..., d]
119 + const scalar_t* __restrict__ input, // [..., topk, d]
120 + const int d) {
121 + const int64_t token_idx = blockIdx.x;
122 + for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
123 + scalar_t x = 0.0;
124 +#pragma unroll
125 + for (int k = 0; k < TOPK; ++k) {
126 + x += VLLM_LDG(&input[token_idx * TOPK * d + k * d + idx]);
127 + }
128 + out[token_idx * d + idx] = x;
129 + }
130 +}
131 +
132 +} // namespace moe
133 } // namespace vllm
134
135 void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
@@ -117,18 +141,62 @@ void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
141 topk_ids.scalar_type(), "moe_align_block_size_kernel", [&] {
142 // calc needed amount of shared mem for `tokens_cnts` and `cumsum`
143 // tensors
144 + const int32_t num_thread = max((int32_t)num_experts, WARP_SIZE);
145 const int32_t shared_mem =
121 - ((num_experts + 1) * num_experts + (num_experts + 1)) *
146 + ((num_thread + 1) * num_experts + (num_experts + 1)) *
147 sizeof(int32_t);
148
149 // set dynamic shared mem
125 - auto kernel = vllm::moe_align_block_size_kernel<scalar_t>;
150 + auto kernel = vllm::moe::moe_align_block_size_kernel<scalar_t>;
151 AT_CUDA_CHECK(VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize(
152 (void*)kernel, shared_mem));
128 - kernel<<<1, num_experts, shared_mem, stream>>>(
153 + kernel<<<1, num_thread, shared_mem, stream>>>(
154 topk_ids.data_ptr<scalar_t>(), sorted_token_ids.data_ptr<int32_t>(),
155 experts_ids.data_ptr<int32_t>(),
156 num_tokens_post_pad.data_ptr<int32_t>(), num_experts, block_size,
157 topk_ids.numel());
158 });
159 }
160 +
161 +void moe_sum(torch::Tensor& input, // [num_tokens, topk, hidden_size]
162 + torch::Tensor& output) // [num_tokens, hidden_size]
163 +{
164 + const int hidden_size = input.size(-1);
165 + const int num_tokens = output.numel() / hidden_size;
166 + const int topk = input.size(1);
167 +
168 + dim3 grid(num_tokens);
169 + dim3 block(std::min(hidden_size, 1024));
170 + const at::cuda::OptionalCUDAGuard device_guard(device_of(output));
171 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
172 +
173 + switch (topk) {
174 + case 2:
175 + VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "moe_sum_kernel", [&] {
176 + vllm::moe::moe_sum_kernel<scalar_t, 2><<<grid, block, 0, stream>>>(
177 + output.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(),
178 + hidden_size);
179 + });
180 + break;
181 +
182 + case 3:
183 + VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "moe_sum_kernel", [&] {
184 + vllm::moe::moe_sum_kernel<scalar_t, 3><<<grid, block, 0, stream>>>(
185 + output.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(),
186 + hidden_size);
187 + });
188 + break;
189 +
190 + case 4:
191 + VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "moe_sum_kernel", [&] {
192 + vllm::moe::moe_sum_kernel<scalar_t, 4><<<grid, block, 0, stream>>>(
193 + output.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(),
194 + hidden_size);
195 + });
196 + break;
197 +
198 + default:
199 + at::sum_out(output, input, 1);
200 + break;
201 + }
202 +}
csrc/moe/moe_ops.h
+7
@@ -5,3 +5,10 @@
5 void topk_softmax(torch::Tensor& topk_weights, torch::Tensor& topk_indices,
6 torch::Tensor& token_expert_indices,
7 torch::Tensor& gating_output);
8 +
9 +void moe_sum(torch::Tensor& input, torch::Tensor& output);
10 +
11 +void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
12 + int64_t block_size, torch::Tensor sorted_token_ids,
13 + torch::Tensor experts_ids,
14 + torch::Tensor num_tokens_post_pad);
csrc/moe/torch_bindings.cpp
+14
@@ -8,6 +8,20 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
8 "token_expert_indices, Tensor gating_output) -> ()");
9 m.impl("topk_softmax", torch::kCUDA, &topk_softmax);
10
11 + // Calculate the result of moe by summing up the partial results
12 + // from all selected experts.
13 + m.def("moe_sum(Tensor! input, Tensor output) -> ()");
14 + m.impl("moe_sum", torch::kCUDA, &moe_sum);
15 +
16 + // Aligning the number of tokens to be processed by each expert such
17 + // that it is divisible by the block size.
18 + m.def(
19 + "moe_align_block_size(Tensor topk_ids, int num_experts,"
20 + " int block_size, Tensor! sorted_token_ids,"
21 + " Tensor! experts_ids,"
22 + " Tensor! num_tokens_post_pad) -> ()");
23 + m.impl("moe_align_block_size", torch::kCUDA, &moe_align_block_size);
24 +
25 #ifndef USE_ROCM
26 m.def(
27 "marlin_gemm_moe(Tensor! a, Tensor! b_q_weights, Tensor! sorted_ids, "
csrc/ops.h
-5
@@ -145,11 +145,6 @@ void dynamic_per_token_scaled_fp8_quant(
145 torch::Tensor& out, torch::Tensor const& input, torch::Tensor& scale,
146 c10::optional<torch::Tensor> const& scale_ub);
147
148 -void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
149 - int64_t block_size, torch::Tensor sorted_token_ids,
150 - torch::Tensor experts_ids,
151 - torch::Tensor num_tokens_post_pad);
152 -
148 void selective_scan_fwd(const torch::Tensor& u, const torch::Tensor& delta,
149 const torch::Tensor& A, const torch::Tensor& B,
150 const torch::Tensor& C,
csrc/torch_bindings.cpp
-9
@@ -336,15 +336,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
336 ops.impl("dynamic_per_token_scaled_fp8_quant", torch::kCUDA,
337 &dynamic_per_token_scaled_fp8_quant);
338
339 - // Aligning the number of tokens to be processed by each expert such
340 - // that it is divisible by the block size.
341 - ops.def(
342 - "moe_align_block_size(Tensor topk_ids, int num_experts,"
343 - " int block_size, Tensor! sorted_token_ids,"
344 - " Tensor! experts_ids,"
345 - " Tensor! num_tokens_post_pad) -> ()");
346 - ops.impl("moe_align_block_size", torch::kCUDA, &moe_align_block_size);
347 -
339 // Compute int8 quantized tensor for given scaling factor.
340 ops.def(
341 "static_scaled_int8_quant(Tensor! out, Tensor input, Tensor scale,"
tests/kernels/test_moe.py
+4 -2
@@ -19,7 +19,7 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
19 marlin_quantize)
20 from vllm.model_executor.models.mixtral import MixtralMoE
21 from vllm.scalar_type import scalar_types
22 -from vllm.utils import seed_everything
22 +from vllm.utils import is_hip, seed_everything
23
24
25 @pytest.mark.parametrize("m", [1024 * 128, 512, 222, 33, 1])
@@ -103,6 +103,7 @@ def test_mixtral_moe(dtype: torch.dtype):
103 @pytest.mark.parametrize("act_order", [True, False])
104 @pytest.mark.parametrize("num_bits", [4, 8])
105 @pytest.mark.parametrize("is_k_full", [True, False])
106 +@pytest.mark.skipif(is_hip(), reason="Skip for rocm")
107 def test_fused_marlin_moe(
108 m: int,
109 n: int,
@@ -255,6 +256,7 @@ def test_fused_marlin_moe(
256 @pytest.mark.parametrize("act_order", [True, False])
257 @pytest.mark.parametrize("num_bits", [4, 8])
258 @pytest.mark.parametrize("is_k_full", [True, False])
259 +@pytest.mark.skipif(is_hip(), reason="Skip for rocm")
260 def test_single_marlin_moe_multiply(
261 m: int,
262 n: int,
@@ -345,6 +347,6 @@ def test_moe_align_block_size_opcheck():
347 dtype=torch.int32,
348 device=topk_ids.device)
349
348 - opcheck(torch.ops._C.moe_align_block_size,
350 + opcheck(torch.ops._moe_C.moe_align_block_size,
351 (topk_ids, num_experts, block_size, sorted_ids, expert_ids,
352 num_tokens_post_pad))
vllm/_custom_ops.py
+7 -3
@@ -813,13 +813,17 @@ def selective_scan_fwd(u: torch.Tensor, delta: torch.Tensor, A: torch.Tensor,
813
814
815 # moe
816 +def moe_sum(input: torch.Tensor, output: torch.Tensor):
817 + torch.ops._moe_C.moe_sum(input, output)
818 +
819 +
820 def moe_align_block_size(topk_ids: torch.Tensor, num_experts: int,
821 block_size: int, sorted_token_ids: torch.Tensor,
822 experts_ids: torch.Tensor,
823 num_tokens_post_pad: torch.Tensor) -> None:
820 - torch.ops._C.moe_align_block_size(topk_ids, num_experts, block_size,
821 - sorted_token_ids, experts_ids,
822 - num_tokens_post_pad)
824 + torch.ops._moe_C.moe_align_block_size(topk_ids, num_experts, block_size,
825 + sorted_token_ids, experts_ids,
826 + num_tokens_post_pad)
827
828
829 def topk_softmax(topk_weights: torch.Tensor, topk_ids: torch.Tensor,
vllm/model_executor/layers/fused_moe/fused_moe.py
+2 -3
@@ -589,9 +589,8 @@ def fused_experts(hidden_states: torch.Tensor,
589 use_fp8_w8a8=use_fp8_w8a8,
590 use_int8_w8a16=use_int8_w8a16)
591
592 - torch.sum(intermediate_cache3.view(*intermediate_cache3.shape),
593 - dim=1,
594 - out=out_hidden_states[begin_chunk_idx:end_chunk_idx])
592 + ops.moe_sum(intermediate_cache3.view(*intermediate_cache3.shape),
593 + out_hidden_states[begin_chunk_idx:end_chunk_idx])
594 return out_hidden_states
595
596