A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Kernel] fix moe_align_block_size error condition (#12239)
Signed-off-by: Jinzhen Lin <linjinzhen@hotmail.com>
Jinzhen Lin committed
Jan 22, 2025 at 02:30 UTC
1e60f87bb37bc28410e6cf6e9030e9a28ad49d12
1 file changed
+6
-4
csrc/moe/moe_align_sum_kernels.cu
+6
-4
@@ -233,15 +233,17 @@ void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
233
(num_experts + 1) * sizeof(int32_t);
234
235
bool use_global_memory = false;
236
- bool use_i16 = false; // Use uint16_t for shared memory token counts
237
- if (shared_mem_i16 > device_max_shared_mem) {
238
- use_global_memory = true;
239
- } else if (shared_mem_i32 > device_max_shared_mem &&
236
+ bool use_i16 = false; // Use uint16_t for shared memory token counts
237
+ if (shared_mem_i32 < device_max_shared_mem) {
238
+ // Do nothing in this case. We're all set to use int32_t token counts
239
+ } else if (shared_mem_i16 < device_max_shared_mem &&
240
topk_ids.numel() <= 65535) {
241
// when nelements of topk_ids is smaller than 65535 (max value of uint16),
242
// element value of token_cnts would also smaller than 65535,
243
// so we can use uint16 as dtype of token_cnts
244
use_i16 = true;
245
+ } else {
246
+ use_global_memory = true;
247
}
248
249
if (use_global_memory) {