A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Bugfix][Kernel] Fix moe align block issue for mixtral (#12413)
ElizaWszola committed
Jan 24, 2025 at 20:49 UTC
221d388cc5a836fa189305785ed7e887cea8b510
1 file changed
+3
-1
csrc/moe/moe_align_sum_kernels.cu
+3
-1
@@ -33,7 +33,9 @@ __global__ void moe_align_block_size_kernel(scalar_t* __restrict__ topk_ids,
33
34
extern __shared__ int32_t shared_mem[];
35
int32_t* cumsum = shared_mem; // 1d tensor with shape (num_experts + 1)
36
- token_cnts_t* tokens_cnts = (token_cnts_t*)(shared_mem + blockDim.x + 1);
36
+ token_cnts_t* tokens_cnts =
37
+ (token_cnts_t*)(shared_mem + num_experts +
38
+ 1); // 2d tensor with shape (blockDim.x + 1, num_experts)
39
40
for (int i = 0; i < num_experts; ++i) {
41
tokens_cnts[index(num_experts, threadIdx.x + 1, i)] = 0;