@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1e60f87bb3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Kernel] fix moe_align_block_size error condition (#12239)

Signed-off-by: Jinzhen Lin <linjinzhen@hotmail.com>

Jinzhen Lin committed Jan 22, 2025 at 02:30 UTC 1e60f87bb37bc28410e6cf6e9030e9a28ad49d12
1 file changed +6 -4
csrc/moe/moe_align_sum_kernels.cu
+6 -4
@@ -233,15 +233,17 @@ void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
233 (num_experts + 1) * sizeof(int32_t);
234
235 bool use_global_memory = false;
236 - bool use_i16 = false; // Use uint16_t for shared memory token counts
237 - if (shared_mem_i16 > device_max_shared_mem) {
238 - use_global_memory = true;
239 - } else if (shared_mem_i32 > device_max_shared_mem &&
236 + bool use_i16 = false; // Use uint16_t for shared memory token counts
237 + if (shared_mem_i32 < device_max_shared_mem) {
238 + // Do nothing in this case. We're all set to use int32_t token counts
239 + } else if (shared_mem_i16 < device_max_shared_mem &&
240 topk_ids.numel() <= 65535) {
241 // when nelements of topk_ids is smaller than 65535 (max value of uint16),
242 // element value of token_cnts would also smaller than 65535,
243 // so we can use uint16 as dtype of token_cnts
244 use_i16 = true;
245 + } else {
246 + use_global_memory = true;
247 }
248
249 if (use_global_memory) {