A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 17:45 UTC
[XPU][CT] Support mxfp8 moe model (#41918)
Signed-off-by: Kunshang Ji <kunshang.ji@intel.com>
Kunshang Ji committed
May 14, 2026 at 09:47 UTC
751b9f14bd5c62276417d6d99cc8ba32441d2505
3 files changed
+48
-1
vllm/model_executor/layers/fused_moe/experts/xpu_moe.py
+44
@@ -17,6 +17,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
17
kFp8DynamicTensorSym,
18
kFp8StaticTensorSym,
19
kMxfp4Static,
20
+ kMxfp8Dynamic,
21
+ kMxfp8Static,
22
)
23
from vllm.platforms import current_platform
24
@@ -47,6 +49,7 @@ class XPUExperts(mk.FusedMoEExpertsModular):
49
)
50
self.is_fp8 = False
51
self.is_mxfp4 = False
52
+ self.is_mxfp8 = False
53
54
@property
55
def expects_unquantized_inputs(self) -> bool:
@@ -148,6 +151,7 @@ class XPUExperts(mk.FusedMoEExpertsModular):
151
output=output,
152
is_fp8=self.is_fp8,
153
is_mxfp4=self.is_mxfp4,
154
+ is_mxfp8=self.is_mxfp8,
155
)
156
157
@@ -167,6 +171,46 @@ class XPUExpertsFp8(XPUExperts):
171
)
172
self.is_fp8 = True
173
174
+ @staticmethod
175
+ def _supports_quant_scheme(
176
+ weight_key: QuantKey | None,
177
+ activation_key: QuantKey | None,
178
+ ) -> bool:
179
+ SUPPORTED_W_A = [
180
+ (kFp8StaticTensorSym, None),
181
+ (kFp8StaticTensorSym, kFp8DynamicTensorSym),
182
+ ]
183
+ return (weight_key, activation_key) in SUPPORTED_W_A
184
+
185
+
186
+class XPUExpertsMxfp8(XPUExpertsFp8):
187
+ def __init__(
188
+ self,
189
+ moe_config: FusedMoEConfig,
190
+ quant_config: FusedMoEQuantConfig,
191
+ max_num_tokens: int | None = None,
192
+ num_dispatchers: int | None = None,
193
+ ):
194
+ super().__init__(
195
+ moe_config,
196
+ quant_config,
197
+ max_num_tokens,
198
+ num_dispatchers,
199
+ )
200
+ assert quant_config.quant_dtype == "mxfp8"
201
+ self.is_mxfp8 = True
202
+
203
+ @staticmethod
204
+ def _supports_quant_scheme(
205
+ weight_key: QuantKey | None,
206
+ activation_key: QuantKey | None,
207
+ ) -> bool:
208
+ SUPPORTED_W_A = [
209
+ (kMxfp8Static, None),
210
+ (kMxfp8Static, kMxfp8Dynamic),
211
+ ]
212
+ return (weight_key, activation_key) in SUPPORTED_W_A
213
+
214
215
class XPUExpertsMXFp4(XPUExperts):
216
def __init__(
vllm/model_executor/layers/fused_moe/oracle/fp8.py
+2
-1
@@ -185,9 +185,10 @@ def backend_to_kernel_cls(
185
elif backend == Fp8MoeBackend.XPU:
186
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import (
187
XPUExpertsFp8,
188
+ XPUExpertsMxfp8,
189
)
190
190
- return [XPUExpertsFp8]
191
+ return [XPUExpertsFp8, XPUExpertsMxfp8]
192
193
elif backend == Fp8MoeBackend.CPU:
194
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
vllm/model_executor/layers/fused_moe/oracle/mxfp8.py
+2
@@ -18,11 +18,13 @@ logger = init_logger(__name__)
18
_SUPPORTED_BACKENDS = (
19
Fp8MoeBackend.FLASHINFER_TRTLLM,
20
Fp8MoeBackend.MARLIN,
21
+ Fp8MoeBackend.XPU,
22
)
23
24
_BACKEND_NAME_MAP: dict[str, Fp8MoeBackend] = {
25
"flashinfer_trtllm": Fp8MoeBackend.FLASHINFER_TRTLLM,
26
"marlin": Fp8MoeBackend.MARLIN,
27
+ "xpu": Fp8MoeBackend.XPU,
28
}
29
30