@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 079a4cf399

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 4, 17:30 UTC

[MoE] Move cutlass moe to fused_moe/experts/ (#40574)

Signed-off-by: Jackmin801 <ongjackm@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>

Jackmin801 committed Apr 23, 2026 at 23:05 UTC 079a4cf399ad548d442fd92bfffbfbe460b66133

14 files changed +20 -18

benchmarks/kernels/benchmark_cutlass_moe_fp8.py

+1 -1

@@ -16,7 +16,7 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import (

16 maybe_make_prepare_finalize,

17 )

18 from vllm.model_executor.layers.fused_moe.config import fp8_w8a8_moe_quant_config

19 -from vllm.model_executor.layers.fused_moe.cutlass_moe import CutlassExpertsFp8

19 +from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import CutlassExpertsFp8

20 from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts, fused_topk

21 from vllm.platforms import current_platform

22 from vllm.utils.argparse_utils import FlexibleArgumentParser

benchmarks/kernels/benchmark_cutlass_moe_nvfp4.py

+1 -1

@@ -22,7 +22,7 @@ from vllm.model_executor.layers.fused_moe.config import (

22 fp8_w8a8_moe_quant_config,

23 nvfp4_moe_quant_config,

24 )

25 -from vllm.model_executor.layers.fused_moe.cutlass_moe import (

25 +from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (

26 CutlassExpertsFp4,

27 )

28 from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts, fused_topk

benchmarks/kernels/benchmark_grouped_gemm_cutlass.py

+1 -1

@@ -13,7 +13,7 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import (

13 maybe_make_prepare_finalize,

14 )

15 from vllm.model_executor.layers.fused_moe.config import fp8_w8a8_moe_quant_config

16 -from vllm.model_executor.layers.fused_moe.cutlass_moe import CutlassExpertsFp8

16 +from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import CutlassExpertsFp8

17 from vllm.model_executor.layers.fused_moe.fused_moe import (

18 fused_experts,

19 fused_topk,

docs/design/moe_kernel_features.md

+2 -2

@@ -83,8 +83,8 @@ To be used with a particular `FusedMoEPrepareAndFinalizeModular` subclass, MoE k

83 | triton | standard | all1 | G,A,T | silu, gelu,swigluoai,silu_no_mul,gelu_no_mul | Y | Y | [`fused_experts`][vllm.model_executor.layers.fused_moe.fused_moe.fused_experts],[`TritonExperts`][vllm.model_executor.layers.fused_moe.fused_moe.TritonExperts] |

84 | triton (batched) | batched | all1 | G,A,T | silu, gelu | 6 | Y | [`BatchedTritonExperts`][vllm.model_executor.layers.fused_moe.fused_batched_moe.BatchedTritonExperts] |

85 | deep gemm | standard,batched | fp8 | G(128),A,T | silu, gelu | 6 | Y | [`DeepGemmExperts`][vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe.DeepGemmExperts],[`BatchedDeepGemmExperts`][vllm.model_executor.layers.fused_moe.experts.batched_deep_gemm_moe.BatchedDeepGemmExperts] |

86 -| cutlass_fp4 | standard,batched | nvfp4 | A,T | silu | Y | Y | [`CutlassExpertsFp4`][vllm.model_executor.layers.fused_moe.cutlass_moe.CutlassExpertsFp4] |

87 -| cutlass_fp8 | standard,batched | fp8 | A,T | silu, gelu | Y | Y | [`CutlassExpertsFp8`][vllm.model_executor.layers.fused_moe.cutlass_moe.CutlassExpertsFp8],[`CutlasBatchedExpertsFp8`][vllm.model_executor.layers.fused_moe.cutlass_moe.CutlassBatchedExpertsFp8] |

86 +| cutlass_fp4 | standard,batched | nvfp4 | A,T | silu | Y | Y | [`CutlassExpertsFp4`][vllm.model_executor.layers.fused_moe.experts.cutlass_moe.CutlassExpertsFp4] |

87 +| cutlass_fp8 | standard,batched | fp8 | A,T | silu, gelu | Y | Y | [`CutlassExpertsFp8`][vllm.model_executor.layers.fused_moe.experts.cutlass_moe.CutlassExpertsFp8],[`CutlasBatchedExpertsFp8`][vllm.model_executor.layers.fused_moe.experts.cutlass_moe.CutlassBatchedExpertsFp8] |

88 | flashinfer | standard | nvfp4,fp8 | T | 5 | N | Y | [`FlashInferExperts`][vllm.model_executor.layers.fused_moe.flashinfer_cutlass_moe.FlashInferExperts] |

89 | gpt oss triton | standard | N/A | N/A | 5 | Y | Y | [`triton_kernel_fused_experts`][vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe.triton_kernel_fused_experts],[`OAITritonExperts`][vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe.OAITritonExperts] |

90 | marlin | standard,batched | 3 / N/A | 3 / N/A | silu,swigluoai | Y | Y | [`fused_marlin_moe`][vllm.model_executor.layers.fused_moe.fused_marlin_moe.fused_marlin_moe],[`MarlinExperts`][vllm.model_executor.layers.fused_moe.fused_marlin_moe.MarlinExperts],[`BatchedMarlinExperts`][vllm.model_executor.layers.fused_moe.fused_marlin_moe.BatchedMarlinExperts] |

tests/kernels/moe/modular_kernel_tools/mk_objects.py

+3 -1

@@ -367,7 +367,9 @@ else:

367 CutlassExpertsFp8 = None

368

369 if cutlass_fp4_supported():

370 - from vllm.model_executor.layers.fused_moe.cutlass_moe import CutlassExpertsFp4

370 + from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (

371 + CutlassExpertsFp4,

372 + )

373

374 register_experts(

375 CutlassExpertsFp4,

tests/kernels/moe/test_cutlass_moe.py

+1 -1

@@ -21,7 +21,7 @@ from vllm.model_executor.layers.fused_moe.config import (

21 FusedMoEQuantConfig,

22 fp8_w8a8_moe_quant_config,

23 )

24 -from vllm.model_executor.layers.fused_moe.cutlass_moe import (

24 +from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (

25 CutlassExpertsFp8,

26 run_cutlass_moe_fp8,

27 )

tests/kernels/moe/test_nvfp4_moe.py

+1 -1

@@ -19,7 +19,7 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import (

19 maybe_make_prepare_finalize,

20 )

21 from vllm.model_executor.layers.fused_moe.config import nvfp4_moe_quant_config

22 -from vllm.model_executor.layers.fused_moe.cutlass_moe import (

22 +from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (

23 CutlassExpertsFp4,

24 )

25 from vllm.model_executor.layers.fused_moe.prepare_finalize import (

vllm/model_executor/layers/fused_moe/__init__.py

+4 -4

@@ -73,15 +73,15 @@ __all__ = [

74 if HAS_TRITON:

75 # import to register the custom ops

76 - from vllm.model_executor.layers.fused_moe.cutlass_moe import (

76 + from vllm.model_executor.layers.fused_moe.experts.batched_deep_gemm_moe import (

77 + BatchedDeepGemmExperts,

78 + )

79 + from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (

80 CutlassBatchedExpertsFp8,

81 CutlassExpertsFp8,

82 CutlassExpertsW4A8Fp8,

83 cutlass_moe_w4a8_fp8,

84 )

82 - from vllm.model_executor.layers.fused_moe.experts.batched_deep_gemm_moe import (

83 - BatchedDeepGemmExperts,

84 - )

85 from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import (

86 DeepGemmExperts,

87 )

vllm/model_executor/layers/fused_moe/experts/cutlass_moe.py renamed

vllm/model_executor/layers/fused_moe/oracle/fp8.py

+1 -1

@@ -173,7 +173,7 @@ def backend_to_kernel_cls(

173 return [TritonOrCutlassExperts]

174

175 elif backend == Fp8MoeBackend.BATCHED_VLLM_CUTLASS:

176 - from vllm.model_executor.layers.fused_moe.cutlass_moe import (

176 + from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (

177 CutlassBatchedExpertsFp8,

178 )

179

vllm/model_executor/layers/fused_moe/oracle/nvfp4.py

+1 -1

@@ -107,7 +107,7 @@ def backend_to_kernel_cls(

107 return [FlashInferCuteDSLBatchedExperts]

108

109 elif backend == NvFp4MoeBackend.VLLM_CUTLASS:

110 - from vllm.model_executor.layers.fused_moe.cutlass_moe import (

110 + from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (

111 CutlassExpertsFp4,

112 )

113

vllm/model_executor/layers/fused_moe/triton_cutlass_moe.py

+1 -1

@@ -10,7 +10,7 @@ from vllm.model_executor.layers.fused_moe.config import (

10 FusedMoEConfig,

11 FusedMoEQuantConfig,

12 )

13 -from vllm.model_executor.layers.fused_moe.cutlass_moe import CutlassExpertsFp8

13 +from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import CutlassExpertsFp8

14 from vllm.model_executor.layers.fused_moe.fallback import FallbackExperts

15 from vllm.model_executor.layers.fused_moe.fused_moe import TritonExperts

16 from vllm.platforms import current_platform

vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_mxfp4.py

+2 -2

@@ -14,7 +14,7 @@ from vllm.model_executor.layers.fused_moe.config import (

14 FusedMoEQuantConfig,

15 mxfp4_moe_quant_config,

16 )

17 -from vllm.model_executor.layers.fused_moe.cutlass_moe import (

17 +from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (

18 CutlassExpertsMxfp4,

19 )

20 from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (

@@ -149,7 +149,7 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):

149 if self.use_cutlass_mxfp4:

150 # Swizzle weight scales from flat checkpoint layout [E, N, K//32]

151 # to CUTLASS tiled layout [E, numMTiles*numKTiles*512].

152 - from vllm.model_executor.layers.fused_moe.cutlass_moe import (

152 + from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (

153 swizzle_mxfp4_scales,

154 )

155

vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a8_fp8.py

+1 -1

@@ -315,7 +315,7 @@ class CompressedTensorsW4A8Fp8MoEMethod(CompressedTensorsMoEMethod):

315 )

316 assert self.moe_quant_config is not None

317

318 - from vllm.model_executor.layers.fused_moe.cutlass_moe import (

318 + from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (

319 cutlass_moe_w4a8_fp8,

320 )

321