A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Dynamically configure shared memory size for moe_align_block_size_kernel (#3376)
akhoroshev committed
Mar 15, 2024 at 04:18 UTC
78b6c4845ac9aa57ccf7e42cf4c7d3c4cdef14cf
1 file changed
+29
-13
csrc/moe_align_block_size_kernels.cu
+29
-13
@@ -7,10 +7,17 @@
7
#include "cuda_compat.h"
8
#include "dispatch_utils.h"
9
10
-const static size_t NUM_MAX_EXPERTS = 64;
10
#define CEILDIV(x,y) (((x) + (y) - 1) / (y))
11
12
namespace vllm {
13
+
14
+namespace {
15
+__device__ __forceinline__ int32_t index(int32_t total_col, int32_t row, int32_t col) {
16
+ // don't worry about overflow because num_experts is relatively small
17
+ return row * total_col + col;
18
+}
19
+}
20
+
21
template <typename scalar_t>
22
__global__ void moe_align_block_size_kernel(scalar_t *__restrict__ topk_ids,
23
int32_t *sorted_token_ids,
@@ -21,10 +28,14 @@ __global__ void moe_align_block_size_kernel(scalar_t *__restrict__ topk_ids,
28
size_t numel) {
29
const size_t tokens_per_thread = CEILDIV(numel, blockDim.x);
30
const size_t start_idx = threadIdx.x * tokens_per_thread;
24
- __shared__ int32_t tokens_cnts[NUM_MAX_EXPERTS + 1][NUM_MAX_EXPERTS];
25
- __shared__ int32_t cumsum[NUM_MAX_EXPERTS + 1];
31
+
32
+ extern __shared__ int32_t shared_mem[];
33
+
34
+ int32_t* tokens_cnts = shared_mem; // 2d tensor with shape (num_experts + 1, num_experts)
35
+ int32_t* cumsum = shared_mem + (num_experts + 1) * num_experts; // 1d tensor with shape (num_experts + 1)
36
+
37
for (int i = 0; i < num_experts; ++i) {
27
- tokens_cnts[threadIdx.x + 1][i] = 0;
38
+ tokens_cnts[index(num_experts, threadIdx.x + 1, i)] = 0;
39
}
40
41
/**
@@ -33,15 +44,15 @@ __global__ void moe_align_block_size_kernel(scalar_t *__restrict__ topk_ids,
44
* to expert expert_index.
45
*/
46
for (int i = start_idx; i < numel && i < start_idx + tokens_per_thread; ++i) {
36
- ++tokens_cnts[threadIdx.x + 1][topk_ids[i]];
47
+ ++tokens_cnts[index(num_experts, threadIdx.x + 1, topk_ids[i])];
48
}
49
50
__syncthreads();
51
52
// For each expert we accumulate the token counts from the different threads.
42
- tokens_cnts[0][threadIdx.x] = 0;
53
+ tokens_cnts[index(num_experts, 0, threadIdx.x)] = 0;
54
for (int i = 1; i <= blockDim.x; ++i) {
44
- tokens_cnts[i][threadIdx.x] += tokens_cnts[i-1][threadIdx.x];
55
+ tokens_cnts[index(num_experts, i, threadIdx.x)] += tokens_cnts[index(num_experts, i-1, threadIdx.x)];
56
}
57
58
__syncthreads();
@@ -50,7 +61,7 @@ __global__ void moe_align_block_size_kernel(scalar_t *__restrict__ topk_ids,
61
if (threadIdx.x == 0) {
62
cumsum[0] = 0;
63
for (int i = 1; i <= num_experts; ++i) {
53
- cumsum[i] = cumsum[i-1] + CEILDIV(tokens_cnts[blockDim.x][i - 1], block_size) * block_size;
64
+ cumsum[i] = cumsum[i-1] + CEILDIV(tokens_cnts[index(num_experts, blockDim.x, i - 1)], block_size) * block_size;
65
}
66
*total_tokens_post_pad = cumsum[num_experts];
67
}
@@ -78,9 +89,9 @@ __global__ void moe_align_block_size_kernel(scalar_t *__restrict__ topk_ids,
89
* stores the indices of the tokens processed by the expert with expert_id within
90
* the current thread's token shard.
91
*/
81
- int32_t rank_post_pad = tokens_cnts[threadIdx.x][expert_id] + cumsum[expert_id];
92
+ int32_t rank_post_pad = tokens_cnts[index(num_experts, threadIdx.x, expert_id)] + cumsum[expert_id];
93
sorted_token_ids[rank_post_pad] = i;
83
- ++tokens_cnts[threadIdx.x][expert_id];
94
+ ++tokens_cnts[index(num_experts, threadIdx.x, expert_id)];
95
}
96
}
97
}
@@ -93,11 +104,16 @@ void moe_align_block_size(
104
torch::Tensor experts_ids,
105
torch::Tensor num_tokens_post_pad) {
106
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
96
- assert(num_experts <= NUM_MAX_EXPERTS);
107
VLLM_DISPATCH_INTEGRAL_TYPES(
108
topk_ids.scalar_type(), "moe_align_block_size_kernel", [&] {
99
- vllm::moe_align_block_size_kernel<scalar_t><<<1, num_experts, 0, stream>>>(
100
- topk_ids.data_ptr<scalar_t>(),
109
+ // calc needed amount of shared mem for `tokens_cnts` and `cumsum` tensors
110
+ const int32_t shared_mem = ((num_experts + 1) * num_experts + (num_experts + 1)) * sizeof(int32_t);
111
+
112
+ // set dynamic shared mem
113
+ auto kernel = vllm::moe_align_block_size_kernel<scalar_t>;
114
+ AT_CUDA_CHECK(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, shared_mem));
115
+ kernel<<<1, num_experts, shared_mem, stream>>>(
116
+ topk_ids.data_ptr<scalar_t>(),
117
sorted_token_ids.data_ptr<int32_t>(),
118
experts_ids.data_ptr<int32_t>(),
119
num_tokens_post_pad.data_ptr<int32_t>(),