A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 11:00 UTC
[MoE] Move DEEP_GEMM into experts/ subdirectory (#39005)
Signed-off-by: Jackmin801 <ongjackm@gmail.com> Signed-off-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Jackmin801 committed
Apr 8, 2026 at 12:23 UTC
a776a48b1c753645c547b735ab647867c98a9b0c
14 files changed
+24
-20
benchmarks/kernels/benchmark_silu_mul_fp8_quant.py
+1
-1
@@ -20,7 +20,7 @@ import matplotlib.pyplot as plt
20
import numpy as np
21
import torch
22
23
-from vllm.model_executor.layers.fused_moe.batched_deep_gemm_moe import (
23
+from vllm.model_executor.layers.fused_moe.experts.batched_deep_gemm_moe import (
24
persistent_masked_m_silu_mul_quant,
25
)
26
from vllm.triton_utils import tl, triton
docs/design/moe_kernel_features.md
+1
-1
@@ -82,7 +82,7 @@ To be used with a particular `FusedMoEPrepareAndFinalizeModular` subclass, MoE k
82
| ------ | ----------------- | ------------ | ------------- | ------------------- | --------------------- | ------- | ------ |
83
| triton | standard | all<sup>1</sup> | G,A,T | silu, gelu,</br>swigluoai,</br>silu_no_mul,</br>gelu_no_mul | Y | Y | [`fused_experts`][vllm.model_executor.layers.fused_moe.fused_moe.fused_experts],</br>[`TritonExperts`][vllm.model_executor.layers.fused_moe.fused_moe.TritonExperts] |
84
| triton (batched) | batched | all<sup>1</sup> | G,A,T | silu, gelu | <sup>6</sup> | Y | [`BatchedTritonExperts`][vllm.model_executor.layers.fused_moe.fused_batched_moe.BatchedTritonExperts] |
85
-| deep gemm | standard,</br>batched | fp8 | G(128),A,T | silu, gelu | <sup>6</sup> | Y | </br>[`DeepGemmExperts`][vllm.model_executor.layers.fused_moe.deep_gemm_moe.DeepGemmExperts],</br>[`BatchedDeepGemmExperts`][vllm.model_executor.layers.fused_moe.batched_deep_gemm_moe.BatchedDeepGemmExperts] |
85
+| deep gemm | standard,</br>batched | fp8 | G(128),A,T | silu, gelu | <sup>6</sup> | Y | </br>[`DeepGemmExperts`][vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe.DeepGemmExperts],</br>[`BatchedDeepGemmExperts`][vllm.model_executor.layers.fused_moe.experts.batched_deep_gemm_moe.BatchedDeepGemmExperts] |
86
| cutlass_fp4 | standard,</br>batched | nvfp4 | A,T | silu | Y | Y | [`CutlassExpertsFp4`][vllm.model_executor.layers.fused_moe.cutlass_moe.CutlassExpertsFp4] |
87
| cutlass_fp8 | standard,</br>batched | fp8 | A,T | silu, gelu | Y | Y | [`CutlassExpertsFp8`][vllm.model_executor.layers.fused_moe.cutlass_moe.CutlassExpertsFp8],</br>[`CutlasBatchedExpertsFp8`][vllm.model_executor.layers.fused_moe.cutlass_moe.CutlassBatchedExpertsFp8] |
88
| flashinfer | standard | nvfp4,</br>fp8 | T | <sup>5</sup> | N | Y | [`FlashInferExperts`][vllm.model_executor.layers.fused_moe.flashinfer_cutlass_moe.FlashInferExperts] |
tests/kernels/moe/modular_kernel_tools/mk_objects.py
+4
-4
@@ -7,14 +7,14 @@ import torch
7
# Fused experts and PrepareFinalize imports
8
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
9
from vllm.model_executor.layers.fused_moe import TritonExperts
10
-from vllm.model_executor.layers.fused_moe.batched_deep_gemm_moe import (
11
- BatchedDeepGemmExperts,
12
-)
10
from vllm.model_executor.layers.fused_moe.config import (
11
FusedMoEConfig,
12
FusedMoEQuantConfig,
13
)
17
-from vllm.model_executor.layers.fused_moe.deep_gemm_moe import DeepGemmExperts
14
+from vllm.model_executor.layers.fused_moe.experts.batched_deep_gemm_moe import (
15
+ BatchedDeepGemmExperts,
16
+)
17
+from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import DeepGemmExperts
18
from vllm.model_executor.layers.fused_moe.fused_batched_moe import (
19
BatchedTritonExperts,
20
NaiveBatchedExperts,
tests/kernels/moe/test_batched_deepgemm.py
+2
-2
@@ -5,10 +5,10 @@ import pytest
5
import torch
6
7
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
8
-from vllm.model_executor.layers.fused_moe.batched_deep_gemm_moe import (
8
+from vllm.model_executor.layers.fused_moe.config import fp8_w8a8_moe_quant_config
9
+from vllm.model_executor.layers.fused_moe.experts.batched_deep_gemm_moe import (
10
BatchedDeepGemmExperts,
11
)
11
-from vllm.model_executor.layers.fused_moe.config import fp8_w8a8_moe_quant_config
12
from vllm.model_executor.layers.fused_moe.fused_batched_moe import (
13
BatchedPrepareAndFinalize,
14
BatchedTritonExperts,
tests/kernels/moe/test_block_fp8.py
+1
-1
@@ -28,7 +28,7 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import (
28
from vllm.model_executor.layers.fused_moe.config import (
29
fp8_w8a8_moe_quant_config,
30
)
31
-from vllm.model_executor.layers.fused_moe.deep_gemm_moe import (
31
+from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import (
32
_valid_deep_gemm_shape,
33
)
34
from vllm.model_executor.layers.fused_moe.triton_deep_gemm_moe import (
tests/kernels/moe/test_deepep_deepgemm_moe.py
+4
-2
@@ -47,10 +47,12 @@ if has_deep_ep():
47
from .parallel_utils import DeepEPHTArgs, DeepEPLLArgs, make_deepep_a2a
48
49
if has_deep_gemm():
50
- from vllm.model_executor.layers.fused_moe.batched_deep_gemm_moe import (
50
+ from vllm.model_executor.layers.fused_moe.experts.batched_deep_gemm_moe import (
51
BatchedDeepGemmExperts,
52
)
53
- from vllm.model_executor.layers.fused_moe.deep_gemm_moe import DeepGemmExperts
53
+ from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import (
54
+ DeepGemmExperts,
55
+ )
56
57
requires_deep_ep = pytest.mark.skipif(
58
not has_deep_ep(),
tests/kernels/moe/test_deepgemm.py
+1
-1
@@ -175,7 +175,7 @@ def test_deepgemm_vs_triton(m, n, k, topk, num_experts, monkeypatch, workspace_i
175
mp.setenv("VLLM_USE_DEEP_GEMM", "1")
176
177
_DeepGemmExperts = importlib.import_module(
178
- "vllm.model_executor.layers.fused_moe.deep_gemm_moe"
178
+ "vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe"
179
).DeepGemmExperts
180
181
call_counter = {"cnt": 0}
tests/kernels/moe/test_silu_mul_fp8_quant_deep_gemm.py
+1
-1
@@ -7,7 +7,7 @@ import random
7
import pytest
8
import torch
9
10
-from vllm.model_executor.layers.fused_moe.batched_deep_gemm_moe import (
10
+from vllm.model_executor.layers.fused_moe.experts.batched_deep_gemm_moe import (
11
persistent_masked_m_silu_mul_quant,
12
)
13
from vllm.model_executor.layers.quantization.utils.quant_utils import (
vllm/model_executor/layers/fused_moe/__init__.py
+6
-4
@@ -77,16 +77,18 @@ __all__ = [
77
78
if HAS_TRITON:
79
# import to register the custom ops
80
- from vllm.model_executor.layers.fused_moe.batched_deep_gemm_moe import (
81
- BatchedDeepGemmExperts,
82
- )
80
from vllm.model_executor.layers.fused_moe.cutlass_moe import (
81
CutlassBatchedExpertsFp8,
82
CutlassExpertsFp8,
83
CutlassExpertsW4A8Fp8,
84
cutlass_moe_w4a8_fp8,
85
)
89
- from vllm.model_executor.layers.fused_moe.deep_gemm_moe import DeepGemmExperts
86
+ from vllm.model_executor.layers.fused_moe.experts.batched_deep_gemm_moe import (
87
+ BatchedDeepGemmExperts,
88
+ )
89
+ from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import (
90
+ DeepGemmExperts,
91
+ )
92
from vllm.model_executor.layers.fused_moe.fused_batched_moe import (
93
BatchedTritonExperts,
94
)
vllm/model_executor/layers/fused_moe/experts/batched_deep_gemm_moe.py
renamed
vllm/model_executor/layers/fused_moe/experts/deep_gemm_moe.py
renamed
vllm/model_executor/layers/fused_moe/oracle/fp8.py
+1
-1
@@ -131,7 +131,7 @@ def backend_to_kernel_cls(
131
return [TritonOrDeepGemmExperts]
132
133
elif backend == Fp8MoeBackend.BATCHED_DEEPGEMM:
134
- from vllm.model_executor.layers.fused_moe.batched_deep_gemm_moe import (
134
+ from vllm.model_executor.layers.fused_moe.experts.batched_deep_gemm_moe import (
135
BatchedDeepGemmExperts,
136
)
137
vllm/model_executor/layers/fused_moe/triton_deep_gemm_moe.py
+1
-1
@@ -9,7 +9,7 @@ from vllm.model_executor.layers.fused_moe.config import (
9
FusedMoEConfig,
10
FusedMoEQuantConfig,
11
)
12
-from vllm.model_executor.layers.fused_moe.deep_gemm_moe import (
12
+from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import (
13
DeepGemmExperts,
14
_valid_deep_gemm,
15
_valid_deep_gemm_shape,
vllm/model_executor/warmup/deep_gemm_warmup.py
+1
-1
@@ -11,8 +11,8 @@ from tqdm import tqdm
11
12
import vllm.envs as envs
13
from vllm.distributed.parallel_state import get_dp_group, is_global_first_rank
14
-from vllm.model_executor.layers.fused_moe.deep_gemm_moe import DeepGemmExperts
14
from vllm.model_executor.layers.fused_moe.deep_gemm_utils import compute_aligned_M
15
+from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import DeepGemmExperts
16
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
17
from vllm.model_executor.layers.fused_moe.triton_deep_gemm_moe import (
18
TritonOrDeepGemmExperts,