A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 10:45 UTC
[Bugfix] Disable shared expert overlap if Marlin MoE is used (#28410)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Nov 11, 2025 at 18:16 UTC
e5f599d4d1cfd34a5216cf0733d152ea42073f28
6 files changed
+13
-5
vllm/model_executor/layers/fused_moe/layer.py
+4
@@ -678,6 +678,10 @@ class FusedMoE(CustomOp):
678
and self.moe_config.use_flashinfer_cutlass_kernels
679
)
680
681
+ @property
682
+ def use_marlin_kernels(self):
683
+ return getattr(self.quant_method, "use_marlin", False)
684
+
685
@property
686
def use_dp_chunking(self) -> bool:
687
return (
vllm/model_executor/layers/fused_moe/shared_fused_moe.py
+5
-5
@@ -28,17 +28,17 @@ class SharedFusedMoE(FusedMoE):
28
super().__init__(**kwargs)
29
self._shared_experts = shared_experts
30
31
- # Disable shared expert overlap if we are using eplb, because of
32
- # correctness issues, or if using flashinfer with DP, since there
33
- # is nothing to be gained in this case. Disabling the overlap
34
- # optimization also prevents the shared experts from being hidden
35
- # from torch.compile.
31
+ # Disable shared expert overlap if:
32
+ # - we are using eplb, because of correctness issues
33
+ # - we are using flashinfer with DP, since there nothint to gain
34
+ # - we are using marlin kjernels
35
self.use_overlapped = (
36
use_overlapped
37
and not (
38
# TODO(wentao): find the root cause and remove this condition
39
self.enable_eplb
40
or (self.moe_config.use_flashinfer_cutlass_kernels and self.dp_size > 1)
41
+ or self.use_marlin_kernels
42
)
43
and self._shared_experts is not None
44
)
vllm/model_executor/layers/quantization/awq_marlin.py
+1
@@ -424,6 +424,7 @@ class AWQMoEMethod(FusedMoEMethodBase):
424
if self.quant_config.weight_bits != 4:
425
raise ValueError("AWQMoEMethod only supports 4bit now.")
426
self.quant_type = scalar_types.uint4
427
+ self.use_marlin = True
428
429
def create_weights(
430
self,
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+1
@@ -1342,6 +1342,7 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
1342
f"{WNA16_SUPPORTED_BITS}",
1343
)
1344
self.quant_type = WNA16_SUPPORTED_TYPES_MAP[self.num_bits]
1345
+ self.use_marlin = True
1346
1347
def create_weights(
1348
self,
vllm/model_executor/layers/quantization/gptq_marlin.py
+1
@@ -482,6 +482,7 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
482
self.quant_type = scalar_types.uint8b128
483
else:
484
raise ValueError("GPTQMarlinMoEMethod only supports int4 and int8 now.")
485
+ self.use_marlin = True
486
487
def create_weights(
488
self,
vllm/model_executor/layers/quantization/mxfp4.py
+1
@@ -216,6 +216,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
216
def __init__(self, moe: FusedMoEConfig):
217
super().__init__(moe)
218
self.mxfp4_backend = get_mxfp4_backend(moe.is_lora_enabled)
219
+ self.use_marlin = self.mxfp4_backend == Mxfp4Backend.MARLIN
220
self.max_capture_size = (
221
get_current_vllm_config().compilation_config.max_cudagraph_capture_size
222
)