A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 10:00 UTC
[Bugfix] Fix some issues with MoERunner PR #32344 (#34371)
Signed-off-by: Bill Nell <bnell@redhat.com>
bnellnm committed
Feb 11, 2026 at 17:33 UTC
31d992d215a05ad2e4f17653ddff0f515f865914
2 files changed
+6
-3
vllm/model_executor/layers/fused_moe/layer.py
+2
-3
@@ -700,7 +700,7 @@ class FusedMoE(CustomOp):
700
701
@property
702
def gate(self) -> torch.nn.Module | None:
703
- return self._gate
703
+ return self._gate if self.use_overlapped else None
704
705
@property
706
def tp_size(self):
@@ -725,7 +725,7 @@ class FusedMoE(CustomOp):
725
@property
726
def is_internal_router(self) -> bool:
727
# By default, router/gate is called before FusedMoE forward pass
728
- return self._gate is not None
728
+ return self.gate is not None
729
730
def _maybe_init_expert_routing_tables(
731
self,
@@ -1457,7 +1457,6 @@ class FusedMoE(CustomOp):
1457
hidden_states: torch.Tensor,
1458
router_logits: torch.Tensor,
1459
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
1460
- self.ensure_moe_quant_config_init()
1460
return self.runner.forward(
1461
hidden_states,
1462
router_logits,
vllm/model_executor/layers/fused_moe/runner/default_moe_runner.py
+4
@@ -63,6 +63,8 @@ def _moe_forward(
63
layer_name: str,
64
) -> torch.Tensor:
65
layer = get_layer_from_name(layer_name)
66
+ # TODO(bnell): this can be removed after MK migration is complete.
67
+ layer.ensure_moe_quant_config_init()
68
return layer.runner.forward_impl(
69
layer, hidden_states, router_logits, shared_experts_input
70
)
@@ -84,6 +86,8 @@ def _moe_forward_shared(
86
layer_name: str,
87
) -> tuple[torch.Tensor, torch.Tensor]:
88
layer = get_layer_from_name(layer_name)
89
+ # TODO(bnell): this can be removed after MK migration is complete.
90
+ layer.ensure_moe_quant_config_init()
91
return layer.runner.forward_impl(
92
layer, hidden_states, router_logits, shared_experts_input
93
)