A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 04:45 UTC
[Bugfix] Mark 'hidden_states' as mutable in moe_forward registration. (#20152)
Signed-off-by: Bill Nell <bnell@redhat.com>
bnellnm committed
Jun 27, 2025 at 11:42 UTC
c6c983053d1c430e9347797094c95bbed37bac2a
1 file changed
+2
-1
vllm/model_executor/layers/fused_moe/layer.py
+2
-1
@@ -1743,7 +1743,8 @@ def moe_forward_fake(hidden_states: torch.Tensor, router_logits: torch.Tensor,
1743
direct_register_custom_op(
1744
op_name="moe_forward",
1745
op_func=moe_forward,
1746
- mutates_args=[],
1746
+ mutates_args=["hidden_states"],
1747
fake_impl=moe_forward_fake,
1748
dispatch_key=current_platform.dispatch_key,
1749
+ tags=(torch.Tag.needs_fixed_stride_order, ),
1750
)