@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8f44a92d85
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[BugFix] fix group_topk (#8430)

Dipika Sikka committed Sep 12, 2024 at 21:23 UTC 8f44a92d852935c8378eaab85bad47ef3174e02b
1 file changed +3 -1
vllm/model_executor/layers/fused_moe/fused_moe.py
+3 -1
@@ -410,6 +410,7 @@ def fused_topk(
410
411 if renormalize:
412 topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
413 +
414 return topk_weights, topk_ids
415
416
@@ -443,7 +444,8 @@ def grouped_topk(hidden_states: torch.Tensor,
444
445 if renormalize:
446 topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
446 - return topk_weights, topk_ids
447 +
448 + return topk_weights, topk_ids.to(torch.int32)
449
450
451 def get_config_dtype_str(dtype: torch.dtype,