A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[Hardware][Gaudi][BugFix] fix arguments of hpu fused moe (#15945)
Signed-off-by: zhenwei <zhenweiliu@habana.ai>
liuzhenwei committed
Apr 5, 2025 at 00:38 UTC
0812d8dd412087565d542af55c8fedd6957c04f4
1 file changed
+5
-2
vllm/model_executor/layers/fused_moe/layer.py
+5
-2
@@ -254,9 +254,12 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
254
renormalize: bool,
255
topk_group: Optional[int] = None,
256
num_expert_group: Optional[int] = None,
257
+ global_num_experts: int = -1,
258
+ expert_map: Optional[torch.Tensor] = None,
259
custom_routing_function: Optional[Callable] = None,
260
scoring_func: str = "softmax",
259
- e_score_correction_bias: Optional[torch.Tensor] = None
261
+ e_score_correction_bias: Optional[torch.Tensor] = None,
262
+ activation: str = "silu",
263
) -> torch.Tensor:
264
assert not use_grouped_topk
265
assert num_expert_group is None
@@ -472,7 +475,7 @@ class FusedMoE(torch.nn.Module):
475
"non-grouped topk.")
476
if current_platform.is_hpu():
477
from vllm_hpu_extension.ops import DynamicFusedMOE
475
- self.hpu_fused_moe = DynamicFusedMOE(self.num_experts)
478
+ self.hpu_fused_moe = DynamicFusedMOE(self.global_num_experts)
479
480
# Note: get_quant_method will look at the layer's local_num_experts
481
# for heuristic purposes, so it must be initialized first.