[MoE Backend] add HPC-Ops MoE backend (#45924)
Signed-off-by: chengvjiang <chengvjiang@tencent.com> Co-authored-by: chengvjiang <chengvjiang@tencent.com> Co-authored-by: youkaichao <youkaichao@gmail.com>
Cheng Jiang committed
Jun 27, 2026 at 11:18 UTC
ddd3855a28a561a5bb54d380c6e6b8b1e883cc4a
5 files changed
+453
-2
docs/design/moe_kernel_features.md
+1
@@ -89,6 +89,7 @@ To be used with a particular `FusedMoEPrepareAndFinalizeModular` subclass, MoE k
89
| gpt oss triton | standard | N/A | N/A | <sup>5</sup> | Y | Y | [`triton_kernel_fused_experts`][vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe.triton_kernel_fused_experts],</br>[`OAITritonExperts`][vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe.OAITritonExperts] |
90
| marlin | standard,</br>batched | <sup>3</sup> / N/A | <sup>3</sup> / N/A | silu,</br>swigluoai | Y | Y | [`fused_marlin_moe`][vllm.model_executor.layers.fused_moe.experts.marlin_moe.fused_marlin_moe],</br>[`MarlinExperts`][vllm.model_executor.layers.fused_moe.experts.marlin_moe.MarlinExperts],</br>[`BatchedMarlinExperts`][vllm.model_executor.layers.fused_moe.experts.marlin_moe.BatchedMarlinExperts] |
91
| trtllm | standard | mxfp4,</br>nvfp4 | G(16),G(32) | <sup>5</sup> | N | Y | [`TrtLlmMxfp4ExpertsMonolithic`][vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe.TrtLlmMxfp4ExpertsMonolithic],</br>[`TrtLlmMxfp4ExpertsModular`][vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe.TrtLlmMxfp4ExpertsModular],</br>[`TrtLlmNvFp4ExpertsMonolithic`][vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe.TrtLlmNvFp4ExpertsMonolithic],</br>[`TrtLlmNvfp4ExpertsModular`][vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe.TrtLlmNvFp4ExpertsModular] |
92
+| hpc | standard | fp8 | G(128),T | silu | Y | Y | [`HPCExperts`][vllm.model_executor.layers.fused_moe.experts.hpc.HPCExperts] |
93
| rocm aiter moe | standard | mxfp4,</br>fp8 | G(32),G(128),A,T | silu, gelu,</br>swigluoai | Y | N | `rocm_aiter_fused_experts`,</br>`AiterExperts` |
94
| cpu_fused_moe | standard | N/A | N/A | silu | N | N | [`CPUFusedMOE`][vllm.model_executor.layers.fused_moe.cpu_fused_moe.CPUFusedMOE] |
95
| naive batched<sup>4</sup> | batched | int8,</br>fp8 | G,A,T | silu, gelu | <sup>6</sup> | Y | [`NaiveBatchedExperts`][vllm.model_executor.layers.fused_moe.experts.fused_batched_moe.NaiveBatchedExperts] |
vllm/config/kernel.py
+2
@@ -134,6 +134,7 @@ MoEBackend = Literal[
134
"triton_unfused",
135
"aiter",
136
"flydsl",
137
+ "hpc",
138
"emulation",
139
]
140
@@ -189,6 +190,7 @@ class KernelConfig:
190
- "triton_unfused": Use Triton unfused MoE kernels
191
- "aiter": Use AMD AITer kernels (ROCm only)
192
- "flydsl": Use AMD FlyDSL kernels (ROCm only)
193
+ - "hpc": Use HPC kernels (FP8 and Hopper only)
194
- "emulation": use BF16/FP16 GEMM, dequantizing weights and
195
running QDQ on activations.
196
"""
vllm/model_executor/layers/fused_moe/hpc_moe.py
new
+211
@@ -0,0 +1,211 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import torch
5
+
6
+import vllm.model_executor.layers.fused_moe.modular_kernel as mk
7
+from vllm.logger import init_logger
8
+from vllm.model_executor.layers.fused_moe.activation import MoEActivation
9
+from vllm.model_executor.layers.fused_moe.config import (
10
+ FusedMoEParallelConfig,
11
+ FusedMoEQuantConfig,
12
+)
13
+from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
14
+ TopKWeightAndReduceNoOP,
15
+)
16
+from vllm.model_executor.layers.quantization.utils.quant_utils import (
17
+ QuantKey,
18
+ kFp8Dynamic128Sym,
19
+ kFp8Static128BlockSym,
20
+ kFp8StaticTensorSym,
21
+)
22
+from vllm.platforms import current_platform
23
+from vllm.utils.hpc import has_hpc, hpc_fuse_moe, hpc_fuse_moe_blockwise
24
+
25
+logger = init_logger(__name__)
26
+
27
+
28
+class HPCExperts(mk.FusedMoEExpertsModular):
29
+ """MoE implementation powered by [HPC](https://github.com/Tencent/hpc-ops).
30
+
31
+ Only supported on NVIDIA Hopper GPUs (e.g. H20, H200), and currently limited to
32
+ FP8 models such as Hy3-FP8, Qwen3-235B-A22B-FP8, etc.
33
+ """
34
+
35
+ def __init__(
36
+ self,
37
+ moe_config: mk.FusedMoEConfig,
38
+ quant_config: FusedMoEQuantConfig,
39
+ ):
40
+ super().__init__(moe_config, quant_config)
41
+
42
+ assert quant_config.weight_quant_dtype in (torch.float8_e4m3fn,), (
43
+ "Only fp8 quantization is currently supported."
44
+ )
45
+
46
+ self.device = moe_config.device
47
+ self.num_experts = moe_config.num_local_experts
48
+ self.ep_rank = moe_config.moe_parallel_config.ep_rank
49
+ self.ep_size = moe_config.moe_parallel_config.ep_size
50
+ self.tp_rank = moe_config.moe_parallel_config.tp_rank
51
+ self.tp_size = moe_config.moe_parallel_config.tp_size
52
+ self.out_dtype = moe_config.in_dtype
53
+
54
+ @property
55
+ def expects_unquantized_inputs(self) -> bool:
56
+ return False
57
+
58
+ @staticmethod
59
+ def activation_format() -> mk.FusedMoEActivationFormat:
60
+ return mk.FusedMoEActivationFormat.Standard
61
+
62
+ @staticmethod
63
+ def _supports_current_device() -> bool:
64
+ p = current_platform
65
+ return (
66
+ p.is_cuda()
67
+ and (p.is_device_capability(90) or p.is_device_capability_family(100))
68
+ and has_hpc()
69
+ )
70
+
71
+ @staticmethod
72
+ def _supports_no_act_and_mul() -> bool:
73
+ return False
74
+
75
+ @staticmethod
76
+ def _supports_quant_scheme(
77
+ weight_key: QuantKey | None,
78
+ activation_key: QuantKey | None,
79
+ ) -> bool:
80
+ scheme = (weight_key, activation_key)
81
+ # The following are supported by HPCExperts:
82
+ return scheme in [
83
+ # fp8 static per-tensor on 9.0+
84
+ (kFp8StaticTensorSym, kFp8StaticTensorSym),
85
+ (kFp8Static128BlockSym, kFp8Dynamic128Sym),
86
+ ]
87
+
88
+ @staticmethod
89
+ def _supports_activation(activation: MoEActivation) -> bool:
90
+ return activation in [
91
+ MoEActivation.SILU,
92
+ ]
93
+
94
+ @staticmethod
95
+ def _supports_shape(hidden_dim: int) -> bool:
96
+ # HPC fused MoE kernels process hidden_size in blocks of 128:
97
+ # block-wise fp8 requires hidden_size % 128 == 0 (per-128 quant), and
98
+ # the group GEMM tiles N by 128. Require 128-alignment to cover all
99
+ # code paths.
100
+ return hidden_dim % 128 == 0
101
+
102
+ @staticmethod
103
+ def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
104
+ return True
105
+
106
+ def supports_expert_map(self) -> bool:
107
+ return False
108
+
109
+ def supports_chunking(self) -> bool:
110
+ # This refers to TP chunking; DP chunking is handled separately.
111
+ return True
112
+
113
+ def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
114
+ return TopKWeightAndReduceNoOP()
115
+
116
+ def workspace_shapes(
117
+ self,
118
+ M: int,
119
+ N: int,
120
+ K: int,
121
+ topk: int,
122
+ global_num_experts: int,
123
+ local_num_experts: int,
124
+ expert_tokens_meta: mk.ExpertTokensMetadata | None,
125
+ activation: MoEActivation,
126
+ ) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
127
+ # We use global_num_experts due to how moe_align_block_size handles
128
+ # expert_maps.
129
+ """
130
+ Compute the shapes for the temporary and final outputs of the two gemms
131
+ and activation in the fused expert function. Since the gemms are
132
+ independent, the workspace for the first gemm can be shared with the
133
+ workspace for the last gemm.
134
+
135
+ Returns a tuple of:
136
+ - workspace13 shape tuple: must be large enough to hold the
137
+ result of either expert gemm.
138
+ - workspace2 shape tuple: must be large enough to hold the
139
+ result of the activation function.
140
+ - output shape tuple: must be exact size of the final gemm output.
141
+ - Workspace type: The dtype to use for the workspace tensors.
142
+ - Note: in order for activation chunking to work, the first dimension
143
+ of each tuple must be the number of tokens.
144
+ """
145
+ workspace1 = (M, K)
146
+ workspace2 = (0,)
147
+ output_shape = (M, K)
148
+ # The workspace is determined by `aq`, since it comes after any
149
+ # potential communication op and is involved in the expert computation.
150
+ return (workspace1, workspace2, output_shape)
151
+
152
+ def apply(
153
+ self,
154
+ output: torch.Tensor,
155
+ hidden_states: torch.Tensor,
156
+ w1: torch.Tensor,
157
+ w2: torch.Tensor,
158
+ topk_weights: torch.Tensor,
159
+ topk_ids: torch.Tensor,
160
+ activation: MoEActivation,
161
+ global_num_experts: int,
162
+ expert_map: torch.Tensor | None,
163
+ a1q_scale: torch.Tensor | None,
164
+ a2_scale: torch.Tensor | None,
165
+ workspace13: torch.Tensor | None,
166
+ workspace2: torch.Tensor | None,
167
+ expert_tokens_meta: mk.ExpertTokensMetadata | None,
168
+ apply_router_weight_on_input: bool | None,
169
+ ):
170
+ assert self._supports_activation(activation), f"{activation=} not supported"
171
+ assert self.quant_config.w1_scale is not None, (
172
+ "w13_weight_scale must be provided"
173
+ )
174
+ assert self.quant_config.w2_scale is not None, (
175
+ "w2_weight_scale must be provided"
176
+ )
177
+
178
+ if self.quant_config.is_block_quantized:
179
+ hpc_fuse_moe_blockwise(
180
+ x=hidden_states,
181
+ x_scale=a1q_scale,
182
+ gate_up_weight=w1,
183
+ gate_up_weight_scale=self.quant_config.w1_scale,
184
+ down_weight=w2,
185
+ down_weight_scale=self.quant_config.w2_scale,
186
+ topk_ids=topk_ids,
187
+ topk_scale=topk_weights,
188
+ rank_ep=self.ep_rank,
189
+ num_expert_total=global_num_experts,
190
+ output=output,
191
+ )
192
+ else:
193
+ assert self.quant_config.a1_scale is not None, (
194
+ "w13_input_scale must be provided"
195
+ )
196
+ assert self.quant_config.a2_scale is not None, (
197
+ "w2_input_scale must be provided"
198
+ )
199
+ hpc_fuse_moe(
200
+ x=hidden_states,
201
+ gate_up_weight=w1,
202
+ down_weight=w2,
203
+ gate_up_scale=self.quant_config.g1_alphas,
204
+ down_scale=self.quant_config.g2_alphas,
205
+ act_and_mul_scale=self.quant_config.a2_gscale,
206
+ topk_ids=topk_ids,
207
+ topk_scale=topk_weights,
208
+ rank_ep=self.ep_rank,
209
+ num_expert_total=global_num_experts,
210
+ output=output,
211
+ )
vllm/model_executor/layers/fused_moe/oracle/fp8.py
+16
-2
@@ -52,6 +52,7 @@ class Fp8MoeBackend(Enum):
52
BATCHED_VLLM_CUTLASS = "BATCHED_VLLM_CUTLASS"
53
XPU = "XPU"
54
CPU = "CPU"
55
+ HPC = "HPC"
56
# Dequantize-to-BF16 emulation for MXFP8 on devices without a native
57
# MXFP8 MoE kernel (e.g. ROCm). Weights pass through unchanged here.
58
EMULATION = "EMULATION"
@@ -85,6 +86,7 @@ def _get_priority_backends(
86
Fp8MoeBackend.BATCHED_TRITON,
87
Fp8MoeBackend.XPU,
88
Fp8MoeBackend.CPU,
89
+ Fp8MoeBackend.HPC,
90
]
91
92
def _move_to_front(backends: list[Fp8MoeBackend], backend: Fp8MoeBackend) -> None:
@@ -216,6 +218,13 @@ def backend_to_kernel_cls(
218
219
return [CPUExpertsFp8]
220
221
+ elif backend == Fp8MoeBackend.HPC:
222
+ from vllm.model_executor.layers.fused_moe.hpc_moe import (
223
+ HPCExperts,
224
+ )
225
+
226
+ return [HPCExperts]
227
+
228
else:
229
raise ValueError(f"Unknown FP8 MoE backend: {backend.value}")
230
@@ -230,6 +239,7 @@ def map_fp8_backend(runner_backend: MoEBackend) -> Fp8MoeBackend:
239
"flashinfer_cutlass": Fp8MoeBackend.FLASHINFER_CUTLASS,
240
"marlin": Fp8MoeBackend.MARLIN,
241
"aiter": Fp8MoeBackend.AITER,
242
+ "hpc": Fp8MoeBackend.HPC,
243
}
244
if backend := mapping.get(runner_backend):
245
return backend
@@ -470,6 +480,7 @@ def convert_to_fp8_moe_kernel_format(
480
Fp8MoeBackend.VLLM_CUTLASS,
481
Fp8MoeBackend.BATCHED_VLLM_CUTLASS,
482
Fp8MoeBackend.XPU,
483
+ Fp8MoeBackend.HPC,
484
# EMULATION dequantizes weights at runtime; NATIVE_MXFP8 consumes
485
# the MXFP8 weights as-is — neither needs a load-time layout change.
486
Fp8MoeBackend.EMULATION,
@@ -521,9 +532,12 @@ def make_fp8_moe_quant_config(
532
gemm1_clamp_limit=swiglu_limit,
533
)
534
524
- # Flashinfer CUTLASS per-tensor uses single dq scale
535
+ # Flashinfer CUTLASS or HPC per-tensor uses single dq scale
536
# (alpha = w_scale * a_scale) and inverse a2 scale.
526
- if fp8_backend == Fp8MoeBackend.FLASHINFER_CUTLASS and block_shape is None:
537
+ if (
538
+ fp8_backend in [Fp8MoeBackend.FLASHINFER_CUTLASS, Fp8MoeBackend.HPC]
539
+ and block_shape is None
540
+ ):
541
assert a1_scale is not None and a2_scale is not None
542
return fp8_w8a8_moe_quant_config(
543
w1_scale=w1_scale,
vllm/utils/hpc.py
new
+223
@@ -0,0 +1,223 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Compatibility wrapper for HPC API changes.
4
+
5
+Users of vLLM should always import **only** these wrappers.
6
+"""
7
+
8
+import functools
9
+import importlib
10
+import importlib.util
11
+
12
+import torch
13
+
14
+from vllm.logger import init_logger
15
+
16
+logger = init_logger(__name__)
17
+
18
+
19
+@functools.cache
20
+def has_hpc() -> bool:
21
+ """Return `True` if hpc package is available."""
22
+ # Use find_spec to check if the module exists without importing it
23
+ # This avoids potential CUDA initialization side effects
24
+ if importlib.util.find_spec("hpc") is None:
25
+ logger.warning_once(
26
+ "HPC attention requires the hpc module to be installed. "
27
+ "Please install it from https://github.com/Tencent/hpc-ops"
28
+ )
29
+ return False
30
+ return True
31
+
32
+
33
+# Remove 'torch._library.custom_ops':
34
+# The output of this custom operator (1) must not also be an input to
35
+# this custom operator and (2) may not alias any inputs to this custom
36
+# operator or other returns. The most common way to trigger this error
37
+# is if we have y = custom_op(x) and y and x are the same Tensor.
38
+# Please instead return a clone of the offending output tensor(s) (e.g.
39
+# return x.clone()) or refactor the custom operator to not return y.
40
+# @torch.library.custom_op(
41
+# "vllm::fuse_moe_impl",
42
+# mutates_args=[],
43
+# device_types="cuda",
44
+# )
45
+def fuse_moe_impl(
46
+ x: torch.Tensor,
47
+ gate_up_weight: torch.Tensor,
48
+ down_weight: torch.Tensor,
49
+ gate_up_scale: torch.Tensor,
50
+ down_scale: torch.Tensor,
51
+ act_and_mul_scale: torch.Tensor,
52
+ topk_ids: torch.Tensor,
53
+ topk_scale: torch.Tensor,
54
+ rank_ep: int,
55
+ num_expert_total: int,
56
+ use_bf16_mul: bool = True,
57
+ shared_output: torch.Tensor = None,
58
+ output: torch.Tensor = None,
59
+) -> torch.Tensor:
60
+ from hpc import fuse_moe as fuse_moe_
61
+
62
+ return fuse_moe_(
63
+ x,
64
+ gate_up_weight,
65
+ down_weight,
66
+ gate_up_scale,
67
+ down_scale,
68
+ act_and_mul_scale,
69
+ topk_ids,
70
+ topk_scale,
71
+ rank_ep,
72
+ num_expert_total,
73
+ use_bf16_mul,
74
+ shared_output,
75
+ output=output,
76
+ )
77
+
78
+
79
+# @torch.library.register_fake(
80
+# "vllm::fuse_moe_impl",
81
+# )
82
+def fuse_moe_impl_fake(
83
+ x: torch.Tensor,
84
+ gate_up_weight: torch.Tensor,
85
+ down_weight: torch.Tensor,
86
+ gate_up_scale: torch.Tensor,
87
+ down_scale: torch.Tensor,
88
+ act_and_mul_scale: torch.Tensor,
89
+ topk_ids: torch.Tensor,
90
+ topk_scale: torch.Tensor,
91
+ rank_ep: int,
92
+ num_expert_total: int,
93
+ use_bf16_mul: bool = True,
94
+ shared_output: torch.Tensor = None,
95
+ output: torch.Tensor = None,
96
+) -> torch.Tensor:
97
+ return torch.empty_like(x)
98
+
99
+
100
+def hpc_fuse_moe(
101
+ x: torch.Tensor,
102
+ gate_up_weight: torch.Tensor,
103
+ down_weight: torch.Tensor,
104
+ gate_up_scale: torch.Tensor,
105
+ down_scale: torch.Tensor,
106
+ act_and_mul_scale: torch.Tensor,
107
+ topk_ids: torch.Tensor,
108
+ topk_scale: torch.Tensor,
109
+ rank_ep: int,
110
+ num_expert_total: int,
111
+ use_bf16_mul: bool = True,
112
+ shared_output: torch.Tensor = None,
113
+ output: torch.Tensor = None,
114
+) -> torch.Tensor:
115
+ return fuse_moe_impl(
116
+ x,
117
+ gate_up_weight,
118
+ down_weight,
119
+ gate_up_scale,
120
+ down_scale,
121
+ act_and_mul_scale,
122
+ topk_ids,
123
+ topk_scale,
124
+ rank_ep,
125
+ num_expert_total,
126
+ use_bf16_mul,
127
+ shared_output,
128
+ output=output,
129
+ )
130
+
131
+
132
+# @torch.library.custom_op(
133
+# "vllm::fuse_moe_blockwise_impl",
134
+# mutates_args=[],
135
+# device_types="cuda",
136
+# )
137
+def fuse_moe_blockwise_impl(
138
+ x: torch.Tensor,
139
+ x_scale: torch.Tensor,
140
+ gate_up_weight: torch.Tensor,
141
+ gate_up_weight_scale: torch.Tensor,
142
+ down_weight: torch.Tensor,
143
+ down_weight_scale: torch.Tensor,
144
+ topk_ids: torch.Tensor,
145
+ topk_scale: torch.Tensor,
146
+ rank_ep: int,
147
+ num_expert_total: int,
148
+ shared_output: torch.Tensor = None,
149
+ output: torch.Tensor = None,
150
+) -> torch.Tensor:
151
+ from hpc import fuse_moe_blockwise as fuse_moe_blockwise_
152
+
153
+ return fuse_moe_blockwise_(
154
+ x,
155
+ x_scale,
156
+ gate_up_weight,
157
+ gate_up_weight_scale,
158
+ down_weight,
159
+ down_weight_scale,
160
+ topk_ids,
161
+ topk_scale,
162
+ rank_ep,
163
+ num_expert_total,
164
+ shared_output,
165
+ output=output,
166
+ )
167
+
168
+
169
+# @torch.library.register_fake(
170
+# "vllm::fuse_moe_blockwise_impl",
171
+# )
172
+def fuse_moe_blockwise_impl_fake(
173
+ x: torch.Tensor,
174
+ x_scale: torch.Tensor,
175
+ gate_up_weight: torch.Tensor,
176
+ gate_up_weight_scale: torch.Tensor,
177
+ down_weight: torch.Tensor,
178
+ down_weight_scale: torch.Tensor,
179
+ topk_ids: torch.Tensor,
180
+ topk_scale: torch.Tensor,
181
+ rank_ep: int,
182
+ num_expert_total: int,
183
+ shared_output: torch.Tensor = None,
184
+ output: torch.Tensor = None,
185
+) -> torch.Tensor:
186
+ return torch.empty_like(x)
187
+
188
+
189
+def hpc_fuse_moe_blockwise(
190
+ x: torch.Tensor,
191
+ x_scale: torch.Tensor,
192
+ gate_up_weight: torch.Tensor,
193
+ gate_up_weight_scale: torch.Tensor,
194
+ down_weight: torch.Tensor,
195
+ down_weight_scale: torch.Tensor,
196
+ topk_ids: torch.Tensor,
197
+ topk_scale: torch.Tensor,
198
+ rank_ep: int,
199
+ num_expert_total: int,
200
+ shared_output: torch.Tensor = None,
201
+ output: torch.Tensor = None,
202
+) -> torch.Tensor:
203
+ return fuse_moe_blockwise_impl(
204
+ x,
205
+ x_scale,
206
+ gate_up_weight,
207
+ gate_up_weight_scale,
208
+ down_weight,
209
+ down_weight_scale,
210
+ topk_ids,
211
+ topk_scale,
212
+ rank_ep,
213
+ num_expert_total,
214
+ shared_output,
215
+ output=output,
216
+ )
217
+
218
+
219
+__all__ = [
220
+ "has_hpc",
221
+ "hpc_fuse_moe",
222
+ "hpc_fuse_moe_blockwise",
223
+]