A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Kernel] optimize moe_align_block_size for cuda graph and large num_experts (e.g. DeepSeek-V3) (#12222)
Signed-off-by: Jinzhen Lin <linjinzhen@hotmail.com> Co-authored-by: Michael Goin <mgoin@redhat.com> Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com>
Jinzhen Lin committed
Jan 21, 2025 at 08:42 UTC
750f4cabfac4bfed679d95074d9550b043e3f8d5
2 files changed
+58
-37
csrc/moe/moe_align_sum_kernels.cu
+57
-36
@@ -21,7 +21,7 @@ __device__ __forceinline__ int32_t index(int32_t total_col, int32_t row,
21
}
22
} // namespace
23
24
-template <typename scalar_t>
24
+template <typename scalar_t, typename token_cnts_t>
25
__global__ void moe_align_block_size_kernel(scalar_t* __restrict__ topk_ids,
26
int32_t* sorted_token_ids,
27
int32_t* expert_ids,
@@ -32,12 +32,8 @@ __global__ void moe_align_block_size_kernel(scalar_t* __restrict__ topk_ids,
32
const size_t start_idx = threadIdx.x * tokens_per_thread;
33
34
extern __shared__ int32_t shared_mem[];
35
-
36
- int32_t* tokens_cnts =
37
- shared_mem; // 2d tensor with shape (blockDim.x + 1, num_experts)
38
- int32_t* cumsum =
39
- shared_mem +
40
- (blockDim.x + 1) * num_experts; // 1d tensor with shape (num_experts + 1)
35
+ int32_t* cumsum = shared_mem; // 1d tensor with shape (num_experts + 1)
36
+ token_cnts_t* tokens_cnts = (token_cnts_t*)(shared_mem + blockDim.x + 1);
37
38
for (int i = 0; i < num_experts; ++i) {
39
tokens_cnts[index(num_experts, threadIdx.x + 1, i)] = 0;
@@ -74,7 +70,7 @@ __global__ void moe_align_block_size_kernel(scalar_t* __restrict__ topk_ids,
70
block_size) *
71
block_size;
72
}
77
- *total_tokens_post_pad = cumsum[num_experts];
73
+ *total_tokens_post_pad = static_cast<int32_t>(cumsum[num_experts]);
74
}
75
76
__syncthreads();
@@ -224,26 +220,44 @@ void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
220
torch::Tensor num_tokens_post_pad) {
221
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
222
227
- // If we have very large number of experts, we can no longer use shared
228
- // memory.
229
- // TODO(simon): the right solution should be calculating the exact right
230
- // amount of shared memory and use that. The num_experts >= 256 is just a
231
- // temporary solution to unblock Deepseek V3.
232
- if (num_experts >= 256) {
223
+ int device_max_shared_mem;
224
+ auto dev = topk_ids.get_device();
225
+ cudaDeviceGetAttribute(&device_max_shared_mem,
226
+ cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
227
+
228
+ const int32_t num_thread = max((int32_t)num_experts, WARP_SIZE);
229
+ const int32_t shared_mem_i32 =
230
+ ((num_thread + 1) * num_experts + (num_experts + 1)) * sizeof(int32_t);
231
+ const int32_t shared_mem_i16 =
232
+ ((num_thread + 1) * num_experts) * sizeof(uint16_t) +
233
+ (num_experts + 1) * sizeof(int32_t);
234
+
235
+ bool use_global_memory = false;
236
+ bool use_i16 = false; // Use uint16_t for shared memory token counts
237
+ if (shared_mem_i16 > device_max_shared_mem) {
238
+ use_global_memory = true;
239
+ } else if (shared_mem_i32 > device_max_shared_mem &&
240
+ topk_ids.numel() <= 65535) {
241
+ // when nelements of topk_ids is smaller than 65535 (max value of uint16),
242
+ // element value of token_cnts would also smaller than 65535,
243
+ // so we can use uint16 as dtype of token_cnts
244
+ use_i16 = true;
245
+ }
246
+
247
+ if (use_global_memory) {
248
VLLM_DISPATCH_INTEGRAL_TYPES(
249
topk_ids.scalar_type(), "moe_align_block_size_global_mem_kernel", [&] {
250
// calc needed amount of shared mem for `tokens_cnts` and `cumsum`
251
// tensors
252
const int32_t num_thread = max((int32_t)num_experts, WARP_SIZE);
253
239
- const int32_t mem_tokens_cnts =
240
- ((num_experts + 1) * num_experts) * sizeof(int32_t);
241
- const int32_t mem_cumsum = (num_experts + 1) * sizeof(int32_t);
242
- // allocate global memory
243
- int32_t* tokens_cnts;
244
- int32_t* cumsum;
245
- cudaMalloc(&tokens_cnts, mem_tokens_cnts);
246
- cudaMalloc(&cumsum, mem_cumsum);
254
+ auto options_int = torch::TensorOptions()
255
+ .dtype(torch::kInt)
256
+ .device(topk_ids.device());
257
+ torch::Tensor token_cnts_buffer =
258
+ torch::empty({(num_experts + 1) * num_experts}, options_int);
259
+ torch::Tensor cumsum_buffer =
260
+ torch::empty({num_experts + 1}, options_int);
261
262
auto kernel =
263
vllm::moe::moe_align_block_size_global_mem_kernel<scalar_t>;
@@ -252,25 +266,32 @@ void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
266
sorted_token_ids.data_ptr<int32_t>(),
267
experts_ids.data_ptr<int32_t>(),
268
num_tokens_post_pad.data_ptr<int32_t>(), num_experts, block_size,
255
- topk_ids.numel(), tokens_cnts, cumsum);
256
- cudaFree(tokens_cnts);
257
- cudaFree(cumsum);
269
+ topk_ids.numel(), token_cnts_buffer.data_ptr<int32_t>(),
270
+ cumsum_buffer.data_ptr<int32_t>());
271
});
259
- } else {
272
+ } else if (use_i16) {
273
VLLM_DISPATCH_INTEGRAL_TYPES(
274
topk_ids.scalar_type(), "moe_align_block_size_kernel", [&] {
262
- // calc needed amount of shared mem for `tokens_cnts` and `cumsum`
263
- // tensors
264
- const int32_t num_thread = max((int32_t)num_experts, WARP_SIZE);
265
- const int32_t shared_mem =
266
- ((num_thread + 1) * num_experts + (num_experts + 1)) *
267
- sizeof(int32_t);
268
-
275
// set dynamic shared mem
270
- auto kernel = vllm::moe::moe_align_block_size_kernel<scalar_t>;
276
+ auto kernel =
277
+ vllm::moe::moe_align_block_size_kernel<scalar_t, uint16_t>;
278
+ AT_CUDA_CHECK(VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize(
279
+ (void*)kernel, shared_mem_i16));
280
+ kernel<<<1, num_thread, shared_mem_i16, stream>>>(
281
+ topk_ids.data_ptr<scalar_t>(),
282
+ sorted_token_ids.data_ptr<int32_t>(),
283
+ experts_ids.data_ptr<int32_t>(),
284
+ num_tokens_post_pad.data_ptr<int32_t>(), num_experts, block_size,
285
+ topk_ids.numel());
286
+ });
287
+ } else {
288
+ VLLM_DISPATCH_INTEGRAL_TYPES(
289
+ topk_ids.scalar_type(), "moe_align_block_size_kernel", [&] {
290
+ auto kernel =
291
+ vllm::moe::moe_align_block_size_kernel<scalar_t, int32_t>;
292
AT_CUDA_CHECK(VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize(
272
- (void*)kernel, shared_mem));
273
- kernel<<<1, num_thread, shared_mem, stream>>>(
293
+ (void*)kernel, shared_mem_i32));
294
+ kernel<<<1, num_thread, shared_mem_i32, stream>>>(
295
topk_ids.data_ptr<scalar_t>(),
296
sorted_token_ids.data_ptr<int32_t>(),
297
experts_ids.data_ptr<int32_t>(),
vllm/config.py
+1
-1
@@ -607,7 +607,7 @@ class ModelConfig:
607
self.max_seq_len_to_capture = min(self.max_seq_len_to_capture,
608
self.max_model_len)
609
610
- MODEL_NOT_SUPPORT_CUDA_GRAPH = ['deepseek_v3', 'mllama']
610
+ MODEL_NOT_SUPPORT_CUDA_GRAPH = ['mllama']
611
if (self.hf_config.model_type in MODEL_NOT_SUPPORT_CUDA_GRAPH
612
and not self.enforce_eager):
613
logger.warning(