A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Performance][Kernel] Fused_moe Performance Improvement (#9384)
Signed-off-by: charlifu <charlifu@amd.com>
Charlie Fu committed
Oct 24, 2024 at 17:37 UTC
59449095ab536febe9ff341b2a88a4fed572a70f
9 files changed
+118
-38
CMakeLists.txt
+1
-1
@@ -195,7 +195,6 @@ set(VLLM_EXT_SRC
195
"csrc/quantization/compressed_tensors/int8_quant_kernels.cu"
196
"csrc/quantization/fp8/common.cu"
197
"csrc/cuda_utils_kernels.cu"
198
- "csrc/moe_align_block_size_kernels.cu"
198
"csrc/prepare_inputs/advance_step.cu"
199
"csrc/torch_bindings.cpp")
200
@@ -405,6 +404,7 @@ target_compile_definitions(_C PRIVATE CUTLASS_ENABLE_DIRECT_CUDA_DRIVER_CALL=1)
404
405
set(VLLM_MOE_EXT_SRC
406
"csrc/moe/torch_bindings.cpp"
407
+ "csrc/moe/moe_align_sum_kernels.cu"
408
"csrc/moe/topk_softmax_kernels.cu")
409
410
set_gencode_flags_for_srcs(
csrc/moe/moe_align_sum_kernels.cu
renamed
+83
-15
@@ -1,15 +1,17 @@
1
#include <torch/all.h>
2
#include <ATen/cuda/CUDAContext.h>
3
+#include <c10/cuda/CUDAGuard.h>
4
5
#include <ATen/ATen.h>
6
#include <THC/THCAtomics.cuh>
7
7
-#include "cuda_compat.h"
8
-#include "dispatch_utils.h"
8
+#include "../cuda_compat.h"
9
+#include "../dispatch_utils.h"
10
11
#define CEILDIV(x, y) (((x) + (y) - 1) / (y))
12
13
namespace vllm {
14
+namespace moe {
15
16
namespace {
17
__device__ __forceinline__ int32_t index(int32_t total_col, int32_t row,
@@ -32,10 +34,10 @@ __global__ void moe_align_block_size_kernel(scalar_t* __restrict__ topk_ids,
34
extern __shared__ int32_t shared_mem[];
35
36
int32_t* tokens_cnts =
35
- shared_mem; // 2d tensor with shape (num_experts + 1, num_experts)
37
+ shared_mem; // 2d tensor with shape (blockDim.x + 1, num_experts)
38
int32_t* cumsum =
37
- shared_mem + (num_experts + 1) *
38
- num_experts; // 1d tensor with shape (num_experts + 1)
39
+ shared_mem +
40
+ (blockDim.x + 1) * num_experts; // 1d tensor with shape (num_experts + 1)
41
42
for (int i = 0; i < num_experts; ++i) {
43
tokens_cnts[index(num_experts, threadIdx.x + 1, i)] = 0;
@@ -53,10 +55,12 @@ __global__ void moe_align_block_size_kernel(scalar_t* __restrict__ topk_ids,
55
__syncthreads();
56
57
// For each expert we accumulate the token counts from the different threads.
56
- tokens_cnts[index(num_experts, 0, threadIdx.x)] = 0;
57
- for (int i = 1; i <= blockDim.x; ++i) {
58
- tokens_cnts[index(num_experts, i, threadIdx.x)] +=
59
- tokens_cnts[index(num_experts, i - 1, threadIdx.x)];
58
+ if (threadIdx.x < num_experts) {
59
+ tokens_cnts[index(num_experts, 0, threadIdx.x)] = 0;
60
+ for (int i = 1; i <= blockDim.x; ++i) {
61
+ tokens_cnts[index(num_experts, i, threadIdx.x)] +=
62
+ tokens_cnts[index(num_experts, i - 1, threadIdx.x)];
63
+ }
64
}
65
66
__syncthreads();
@@ -79,9 +83,11 @@ __global__ void moe_align_block_size_kernel(scalar_t* __restrict__ topk_ids,
83
* For each expert, each thread processes the tokens of the corresponding
84
* blocks and stores the corresponding expert_id for each block.
85
*/
82
- for (int i = cumsum[threadIdx.x]; i < cumsum[threadIdx.x + 1];
83
- i += block_size) {
84
- expert_ids[i / block_size] = threadIdx.x;
86
+ if (threadIdx.x < num_experts) {
87
+ for (int i = cumsum[threadIdx.x]; i < cumsum[threadIdx.x + 1];
88
+ i += block_size) {
89
+ expert_ids[i / block_size] = threadIdx.x;
90
+ }
91
}
92
93
/**
@@ -106,6 +112,24 @@ __global__ void moe_align_block_size_kernel(scalar_t* __restrict__ topk_ids,
112
++tokens_cnts[index(num_experts, threadIdx.x, expert_id)];
113
}
114
}
115
+
116
+template <typename scalar_t, int TOPK>
117
+__global__ void moe_sum_kernel(
118
+ scalar_t* __restrict__ out, // [..., d]
119
+ const scalar_t* __restrict__ input, // [..., topk, d]
120
+ const int d) {
121
+ const int64_t token_idx = blockIdx.x;
122
+ for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
123
+ scalar_t x = 0.0;
124
+#pragma unroll
125
+ for (int k = 0; k < TOPK; ++k) {
126
+ x += VLLM_LDG(&input[token_idx * TOPK * d + k * d + idx]);
127
+ }
128
+ out[token_idx * d + idx] = x;
129
+ }
130
+}
131
+
132
+} // namespace moe
133
} // namespace vllm
134
135
void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
@@ -117,18 +141,62 @@ void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
141
topk_ids.scalar_type(), "moe_align_block_size_kernel", [&] {
142
// calc needed amount of shared mem for `tokens_cnts` and `cumsum`
143
// tensors
144
+ const int32_t num_thread = max((int32_t)num_experts, WARP_SIZE);
145
const int32_t shared_mem =
121
- ((num_experts + 1) * num_experts + (num_experts + 1)) *
146
+ ((num_thread + 1) * num_experts + (num_experts + 1)) *
147
sizeof(int32_t);
148
149
// set dynamic shared mem
125
- auto kernel = vllm::moe_align_block_size_kernel<scalar_t>;
150
+ auto kernel = vllm::moe::moe_align_block_size_kernel<scalar_t>;
151
AT_CUDA_CHECK(VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize(
152
(void*)kernel, shared_mem));
128
- kernel<<<1, num_experts, shared_mem, stream>>>(
153
+ kernel<<<1, num_thread, shared_mem, stream>>>(
154
topk_ids.data_ptr<scalar_t>(), sorted_token_ids.data_ptr<int32_t>(),
155
experts_ids.data_ptr<int32_t>(),
156
num_tokens_post_pad.data_ptr<int32_t>(), num_experts, block_size,
157
topk_ids.numel());
158
});
159
}
160
+
161
+void moe_sum(torch::Tensor& input, // [num_tokens, topk, hidden_size]
162
+ torch::Tensor& output) // [num_tokens, hidden_size]
163
+{
164
+ const int hidden_size = input.size(-1);
165
+ const int num_tokens = output.numel() / hidden_size;
166
+ const int topk = input.size(1);
167
+
168
+ dim3 grid(num_tokens);
169
+ dim3 block(std::min(hidden_size, 1024));
170
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(output));
171
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
172
+
173
+ switch (topk) {
174
+ case 2:
175
+ VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "moe_sum_kernel", [&] {
176
+ vllm::moe::moe_sum_kernel<scalar_t, 2><<<grid, block, 0, stream>>>(
177
+ output.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(),
178
+ hidden_size);
179
+ });
180
+ break;
181
+
182
+ case 3:
183
+ VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "moe_sum_kernel", [&] {
184
+ vllm::moe::moe_sum_kernel<scalar_t, 3><<<grid, block, 0, stream>>>(
185
+ output.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(),
186
+ hidden_size);
187
+ });
188
+ break;
189
+
190
+ case 4:
191
+ VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "moe_sum_kernel", [&] {
192
+ vllm::moe::moe_sum_kernel<scalar_t, 4><<<grid, block, 0, stream>>>(
193
+ output.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(),
194
+ hidden_size);
195
+ });
196
+ break;
197
+
198
+ default:
199
+ at::sum_out(output, input, 1);
200
+ break;
201
+ }
202
+}
csrc/moe/moe_ops.h
+7
@@ -5,3 +5,10 @@
5
void topk_softmax(torch::Tensor& topk_weights, torch::Tensor& topk_indices,
6
torch::Tensor& token_expert_indices,
7
torch::Tensor& gating_output);
8
+
9
+void moe_sum(torch::Tensor& input, torch::Tensor& output);
10
+
11
+void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
12
+ int64_t block_size, torch::Tensor sorted_token_ids,
13
+ torch::Tensor experts_ids,
14
+ torch::Tensor num_tokens_post_pad);
csrc/moe/torch_bindings.cpp
+14
@@ -8,6 +8,20 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
8
"token_expert_indices, Tensor gating_output) -> ()");
9
m.impl("topk_softmax", torch::kCUDA, &topk_softmax);
10
11
+ // Calculate the result of moe by summing up the partial results
12
+ // from all selected experts.
13
+ m.def("moe_sum(Tensor! input, Tensor output) -> ()");
14
+ m.impl("moe_sum", torch::kCUDA, &moe_sum);
15
+
16
+ // Aligning the number of tokens to be processed by each expert such
17
+ // that it is divisible by the block size.
18
+ m.def(
19
+ "moe_align_block_size(Tensor topk_ids, int num_experts,"
20
+ " int block_size, Tensor! sorted_token_ids,"
21
+ " Tensor! experts_ids,"
22
+ " Tensor! num_tokens_post_pad) -> ()");
23
+ m.impl("moe_align_block_size", torch::kCUDA, &moe_align_block_size);
24
+
25
#ifndef USE_ROCM
26
m.def(
27
"marlin_gemm_moe(Tensor! a, Tensor! b_q_weights, Tensor! sorted_ids, "
csrc/ops.h
-5
@@ -145,11 +145,6 @@ void dynamic_per_token_scaled_fp8_quant(
145
torch::Tensor& out, torch::Tensor const& input, torch::Tensor& scale,
146
c10::optional<torch::Tensor> const& scale_ub);
147
148
-void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
149
- int64_t block_size, torch::Tensor sorted_token_ids,
150
- torch::Tensor experts_ids,
151
- torch::Tensor num_tokens_post_pad);
152
-
148
void selective_scan_fwd(const torch::Tensor& u, const torch::Tensor& delta,
149
const torch::Tensor& A, const torch::Tensor& B,
150
const torch::Tensor& C,
csrc/torch_bindings.cpp
-9
@@ -336,15 +336,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
336
ops.impl("dynamic_per_token_scaled_fp8_quant", torch::kCUDA,
337
&dynamic_per_token_scaled_fp8_quant);
338
339
- // Aligning the number of tokens to be processed by each expert such
340
- // that it is divisible by the block size.
341
- ops.def(
342
- "moe_align_block_size(Tensor topk_ids, int num_experts,"
343
- " int block_size, Tensor! sorted_token_ids,"
344
- " Tensor! experts_ids,"
345
- " Tensor! num_tokens_post_pad) -> ()");
346
- ops.impl("moe_align_block_size", torch::kCUDA, &moe_align_block_size);
347
-
339
// Compute int8 quantized tensor for given scaling factor.
340
ops.def(
341
"static_scaled_int8_quant(Tensor! out, Tensor input, Tensor scale,"
tests/kernels/test_moe.py
+4
-2
@@ -19,7 +19,7 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
19
marlin_quantize)
20
from vllm.model_executor.models.mixtral import MixtralMoE
21
from vllm.scalar_type import scalar_types
22
-from vllm.utils import seed_everything
22
+from vllm.utils import is_hip, seed_everything
23
24
25
@pytest.mark.parametrize("m", [1024 * 128, 512, 222, 33, 1])
@@ -103,6 +103,7 @@ def test_mixtral_moe(dtype: torch.dtype):
103
@pytest.mark.parametrize("act_order", [True, False])
104
@pytest.mark.parametrize("num_bits", [4, 8])
105
@pytest.mark.parametrize("is_k_full", [True, False])
106
+@pytest.mark.skipif(is_hip(), reason="Skip for rocm")
107
def test_fused_marlin_moe(
108
m: int,
109
n: int,
@@ -255,6 +256,7 @@ def test_fused_marlin_moe(
256
@pytest.mark.parametrize("act_order", [True, False])
257
@pytest.mark.parametrize("num_bits", [4, 8])
258
@pytest.mark.parametrize("is_k_full", [True, False])
259
+@pytest.mark.skipif(is_hip(), reason="Skip for rocm")
260
def test_single_marlin_moe_multiply(
261
m: int,
262
n: int,
@@ -345,6 +347,6 @@ def test_moe_align_block_size_opcheck():
347
dtype=torch.int32,
348
device=topk_ids.device)
349
348
- opcheck(torch.ops._C.moe_align_block_size,
350
+ opcheck(torch.ops._moe_C.moe_align_block_size,
351
(topk_ids, num_experts, block_size, sorted_ids, expert_ids,
352
num_tokens_post_pad))
vllm/_custom_ops.py
+7
-3
@@ -813,13 +813,17 @@ def selective_scan_fwd(u: torch.Tensor, delta: torch.Tensor, A: torch.Tensor,
813
814
815
# moe
816
+def moe_sum(input: torch.Tensor, output: torch.Tensor):
817
+ torch.ops._moe_C.moe_sum(input, output)
818
+
819
+
820
def moe_align_block_size(topk_ids: torch.Tensor, num_experts: int,
821
block_size: int, sorted_token_ids: torch.Tensor,
822
experts_ids: torch.Tensor,
823
num_tokens_post_pad: torch.Tensor) -> None:
820
- torch.ops._C.moe_align_block_size(topk_ids, num_experts, block_size,
821
- sorted_token_ids, experts_ids,
822
- num_tokens_post_pad)
824
+ torch.ops._moe_C.moe_align_block_size(topk_ids, num_experts, block_size,
825
+ sorted_token_ids, experts_ids,
826
+ num_tokens_post_pad)
827
828
829
def topk_softmax(topk_weights: torch.Tensor, topk_ids: torch.Tensor,
vllm/model_executor/layers/fused_moe/fused_moe.py
+2
-3
@@ -589,9 +589,8 @@ def fused_experts(hidden_states: torch.Tensor,
589
use_fp8_w8a8=use_fp8_w8a8,
590
use_int8_w8a16=use_int8_w8a16)
591
592
- torch.sum(intermediate_cache3.view(*intermediate_cache3.shape),
593
- dim=1,
594
- out=out_hidden_states[begin_chunk_idx:end_chunk_idx])
592
+ ops.moe_sum(intermediate_cache3.view(*intermediate_cache3.shape),
593
+ out_hidden_states[begin_chunk_idx:end_chunk_idx])
594
return out_hidden_states
595
596