@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 78b6c4845a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Dynamically configure shared memory size for moe_align_block_size_kernel (#3376)

akhoroshev committed Mar 15, 2024 at 04:18 UTC 78b6c4845ac9aa57ccf7e42cf4c7d3c4cdef14cf
1 file changed +29 -13
csrc/moe_align_block_size_kernels.cu
+29 -13
@@ -7,10 +7,17 @@
7 #include "cuda_compat.h"
8 #include "dispatch_utils.h"
9
10 -const static size_t NUM_MAX_EXPERTS = 64;
10 #define CEILDIV(x,y) (((x) + (y) - 1) / (y))
11
12 namespace vllm {
13 +
14 +namespace {
15 +__device__ __forceinline__ int32_t index(int32_t total_col, int32_t row, int32_t col) {
16 + // don't worry about overflow because num_experts is relatively small
17 + return row * total_col + col;
18 +}
19 +}
20 +
21 template <typename scalar_t>
22 __global__ void moe_align_block_size_kernel(scalar_t *__restrict__ topk_ids,
23 int32_t *sorted_token_ids,
@@ -21,10 +28,14 @@ __global__ void moe_align_block_size_kernel(scalar_t *__restrict__ topk_ids,
28 size_t numel) {
29 const size_t tokens_per_thread = CEILDIV(numel, blockDim.x);
30 const size_t start_idx = threadIdx.x * tokens_per_thread;
24 - __shared__ int32_t tokens_cnts[NUM_MAX_EXPERTS + 1][NUM_MAX_EXPERTS];
25 - __shared__ int32_t cumsum[NUM_MAX_EXPERTS + 1];
31 +
32 + extern __shared__ int32_t shared_mem[];
33 +
34 + int32_t* tokens_cnts = shared_mem; // 2d tensor with shape (num_experts + 1, num_experts)
35 + int32_t* cumsum = shared_mem + (num_experts + 1) * num_experts; // 1d tensor with shape (num_experts + 1)
36 +
37 for (int i = 0; i < num_experts; ++i) {
27 - tokens_cnts[threadIdx.x + 1][i] = 0;
38 + tokens_cnts[index(num_experts, threadIdx.x + 1, i)] = 0;
39 }
40
41 /**
@@ -33,15 +44,15 @@ __global__ void moe_align_block_size_kernel(scalar_t *__restrict__ topk_ids,
44 * to expert expert_index.
45 */
46 for (int i = start_idx; i < numel && i < start_idx + tokens_per_thread; ++i) {
36 - ++tokens_cnts[threadIdx.x + 1][topk_ids[i]];
47 + ++tokens_cnts[index(num_experts, threadIdx.x + 1, topk_ids[i])];
48 }
49
50 __syncthreads();
51
52 // For each expert we accumulate the token counts from the different threads.
42 - tokens_cnts[0][threadIdx.x] = 0;
53 + tokens_cnts[index(num_experts, 0, threadIdx.x)] = 0;
54 for (int i = 1; i <= blockDim.x; ++i) {
44 - tokens_cnts[i][threadIdx.x] += tokens_cnts[i-1][threadIdx.x];
55 + tokens_cnts[index(num_experts, i, threadIdx.x)] += tokens_cnts[index(num_experts, i-1, threadIdx.x)];
56 }
57
58 __syncthreads();
@@ -50,7 +61,7 @@ __global__ void moe_align_block_size_kernel(scalar_t *__restrict__ topk_ids,
61 if (threadIdx.x == 0) {
62 cumsum[0] = 0;
63 for (int i = 1; i <= num_experts; ++i) {
53 - cumsum[i] = cumsum[i-1] + CEILDIV(tokens_cnts[blockDim.x][i - 1], block_size) * block_size;
64 + cumsum[i] = cumsum[i-1] + CEILDIV(tokens_cnts[index(num_experts, blockDim.x, i - 1)], block_size) * block_size;
65 }
66 *total_tokens_post_pad = cumsum[num_experts];
67 }
@@ -78,9 +89,9 @@ __global__ void moe_align_block_size_kernel(scalar_t *__restrict__ topk_ids,
89 * stores the indices of the tokens processed by the expert with expert_id within
90 * the current thread's token shard.
91 */
81 - int32_t rank_post_pad = tokens_cnts[threadIdx.x][expert_id] + cumsum[expert_id];
92 + int32_t rank_post_pad = tokens_cnts[index(num_experts, threadIdx.x, expert_id)] + cumsum[expert_id];
93 sorted_token_ids[rank_post_pad] = i;
83 - ++tokens_cnts[threadIdx.x][expert_id];
94 + ++tokens_cnts[index(num_experts, threadIdx.x, expert_id)];
95 }
96 }
97 }
@@ -93,11 +104,16 @@ void moe_align_block_size(
104 torch::Tensor experts_ids,
105 torch::Tensor num_tokens_post_pad) {
106 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
96 - assert(num_experts <= NUM_MAX_EXPERTS);
107 VLLM_DISPATCH_INTEGRAL_TYPES(
108 topk_ids.scalar_type(), "moe_align_block_size_kernel", [&] {
99 - vllm::moe_align_block_size_kernel<scalar_t><<<1, num_experts, 0, stream>>>(
100 - topk_ids.data_ptr<scalar_t>(),
109 + // calc needed amount of shared mem for `tokens_cnts` and `cumsum` tensors
110 + const int32_t shared_mem = ((num_experts + 1) * num_experts + (num_experts + 1)) * sizeof(int32_t);
111 +
112 + // set dynamic shared mem
113 + auto kernel = vllm::moe_align_block_size_kernel<scalar_t>;
114 + AT_CUDA_CHECK(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, shared_mem));
115 + kernel<<<1, num_experts, shared_mem, stream>>>(
116 + topk_ids.data_ptr<scalar_t>(),
117 sorted_token_ids.data_ptr<int32_t>(),
118 experts_ids.data_ptr<int32_t>(),
119 num_tokens_post_pad.data_ptr<int32_t>(),