A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[BugFix] fix group_topk (#8430)
Dipika Sikka committed
Sep 12, 2024 at 21:23 UTC
8f44a92d852935c8378eaab85bad47ef3174e02b
1 file changed
+3
-1
vllm/model_executor/layers/fused_moe/fused_moe.py
+3
-1
@@ -410,6 +410,7 @@ def fused_topk(
410
411
if renormalize:
412
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
413
+
414
return topk_weights, topk_ids
415
416
@@ -443,7 +444,8 @@ def grouped_topk(hidden_states: torch.Tensor,
444
445
if renormalize:
446
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
446
- return topk_weights, topk_ids
447
+
448
+ return topk_weights, topk_ids.to(torch.int32)
449
450
451
def get_config_dtype_str(dtype: torch.dtype,