A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Bugfix] fused_experts_impl wrong compute type for float32 (#11921)
Signed-off-by: shaochangxu.scx <shaochangxu.scx@antgroup.com> Co-authored-by: shaochangxu.scx <shaochangxu.scx@antgroup.com>
shaochangxu committed
Jan 11, 2025 at 13:49 UTC
c32a7c7c0c688ed81d2f4ad701a09d0edd095ffe
1 file changed
+8
-2
vllm/model_executor/layers/fused_moe/fused_moe.py
+8
-2
@@ -701,8 +701,14 @@ def fused_experts_impl(hidden_states: torch.Tensor,
701
device=hidden_states.device,
702
dtype=hidden_states.dtype)
703
704
- compute_type = (tl.bfloat16
705
- if hidden_states.dtype == torch.bfloat16 else tl.float16)
704
+ if hidden_states.dtype == torch.bfloat16:
705
+ compute_type = tl.bfloat16
706
+ elif hidden_states.dtype == torch.float16:
707
+ compute_type = tl.float16
708
+ elif hidden_states.dtype == torch.float32:
709
+ compute_type = tl.float32
710
+ else:
711
+ raise ValueError(f"Unsupported compute_type: {hidden_states.dtype}")
712
713
if inplace:
714
out_hidden_states = hidden_states