A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 23:45 UTC
[Perf] Enable separate shared_experts stream only for CUDA (#30085)
Signed-off-by: Alexander Matveev <amatveev@redhat.com>
Alexander Matveev committed
Dec 4, 2025 at 19:03 UTC
4470ee2f90661d9eb632687750dfb1a5a2404032
1 file changed
+2
-1
vllm/model_executor/layers/fused_moe/layer.py
+2
-1
@@ -863,7 +863,8 @@ class FusedMoE(CustomOp):
863
use_chunked_impl: bool,
864
) -> tuple[bool, torch.Tensor | None]:
865
use_shared_experts_stream = (
866
- has_separate_shared_experts
866
+ current_platform.is_cuda()
867
+ and has_separate_shared_experts
868
and not use_chunked_impl
869
and self.shared_experts_stream is not None
870
and (