A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 12:30 UTC
[XPU][7/N] enable xpu fp8 moe (#34202)
Signed-off-by: Zhu, Zufang <zufang.zhu@intel.com>
zofia committed
Feb 11, 2026 at 11:33 UTC
b482f71e9f25ce848c1a53e71e332953d97b0aac
4 files changed
+52
-5
requirements/xpu.txt
+1
-1
@@ -15,4 +15,4 @@ torch==2.10.0+xpu
15
torchaudio
16
torchvision
17
18
-vllm_xpu_kernels @ https://github.com/vllm-project/vllm-xpu-kernels/releases/download/v0.1.1/vllm_xpu_kernels-0.1.1-cp312-cp312-linux_x86_64.whl
\ No newline at end of file
18
+vllm_xpu_kernels @ https://github.com/vllm-project/vllm-xpu-kernels/releases/download/v0.1.2/vllm_xpu_kernels-0.1.2-cp312-cp312-linux_x86_64.whl
vllm/model_executor/layers/fused_moe/__init__.py
+2
@@ -102,6 +102,7 @@ if HAS_TRITON:
102
)
103
from vllm.model_executor.layers.fused_moe.xpu_fused_moe import (
104
XPUExperts,
105
+ XPUExpertsFp8,
106
)
107
108
__all__ += [
@@ -121,6 +122,7 @@ if HAS_TRITON:
122
"BatchedDeepGemmExperts",
123
"TritonOrDeepGemmExperts",
124
"XPUExperts",
125
+ "XPUExpertsFp8",
126
]
127
else:
128
# Some model classes directly use the custom ops. Add placeholders
vllm/model_executor/layers/fused_moe/oracle/fp8.py
+10
@@ -52,6 +52,7 @@ class Fp8MoeBackend(Enum):
52
AITER = "AITER"
53
VLLM_CUTLASS = "VLLM_CUTLASS"
54
BATCHED_VLLM_CUTLASS = "BATCHED_VLLM_CUTLASS"
55
+ XPU = "XPU"
56
57
58
def backend_to_kernel_cls(
@@ -123,6 +124,13 @@ def backend_to_kernel_cls(
124
125
return CutlassBatchedExpertsFp8
126
127
+ elif backend == Fp8MoeBackend.XPU:
128
+ from vllm.model_executor.layers.fused_moe.xpu_fused_moe import (
129
+ XPUExpertsFp8,
130
+ )
131
+
132
+ return XPUExpertsFp8
133
+
134
else:
135
raise ValueError(f"Unknown FP8 MoE backend: {backend.value}")
136
@@ -154,6 +162,7 @@ def select_fp8_moe_backend(
162
Fp8MoeBackend.TRITON,
163
Fp8MoeBackend.BATCHED_TRITON,
164
Fp8MoeBackend.MARLIN,
165
+ Fp8MoeBackend.XPU,
166
]
167
168
# NOTE(rob): We need to peak into the P/F selection to determine
@@ -393,6 +402,7 @@ def convert_to_fp8_moe_kernel_format(
402
Fp8MoeBackend.BATCHED_TRITON,
403
Fp8MoeBackend.VLLM_CUTLASS,
404
Fp8MoeBackend.BATCHED_VLLM_CUTLASS,
405
+ Fp8MoeBackend.XPU,
406
]:
407
raise ValueError(f"Unsupported FP8 MoE backend: {fp8_backend.value}")
408
vllm/model_executor/layers/fused_moe/xpu_fused_moe.py
+39
-4
@@ -4,13 +4,16 @@ import torch
4
5
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
6
from vllm.model_executor.layers.fused_moe.config import (
7
+ FusedMoEConfig,
8
FusedMoEParallelConfig,
9
+ FusedMoEQuantConfig,
10
)
11
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
12
TopKWeightAndReduceNoOP,
13
)
14
from vllm.model_executor.layers.quantization.utils.quant_utils import (
15
QuantKey,
16
+ kFp8DynamicTensorSym,
17
kFp8StaticTensorSym,
18
)
19
from vllm.platforms import current_platform
@@ -20,6 +23,21 @@ if current_platform.is_xpu():
23
24
25
class XPUExperts(mk.FusedMoEPermuteExpertsUnpermute):
26
+ def __init__(
27
+ self,
28
+ moe_config: FusedMoEConfig,
29
+ quant_config: FusedMoEQuantConfig,
30
+ max_num_tokens: int | None = None,
31
+ num_dispatchers: int | None = None,
32
+ ):
33
+ super().__init__(
34
+ moe_config,
35
+ quant_config,
36
+ max_num_tokens,
37
+ num_dispatchers,
38
+ )
39
+ self.is_fp8 = False
40
+
41
@property
42
def expects_unquantized_inputs(self) -> bool:
43
return True
@@ -49,10 +67,10 @@ class XPUExperts(mk.FusedMoEPermuteExpertsUnpermute):
67
weight_key: QuantKey | None,
68
activation_key: QuantKey | None,
69
) -> bool:
52
- # TODO: dispatch based on device.
70
SUPPORTED_W_A = [
71
(None, None),
72
(kFp8StaticTensorSym, None),
73
+ (kFp8StaticTensorSym, kFp8DynamicTensorSym),
74
]
75
return (weight_key, activation_key) in SUPPORTED_W_A
76
@@ -103,10 +121,10 @@ class XPUExperts(mk.FusedMoEPermuteExpertsUnpermute):
121
xpu_fused_moe(
122
hidden_states=hidden_states,
123
w13=w1,
106
- w13_scales=a1q_scale,
124
+ w13_scales=self.w1_scale,
125
w13_bias=self.w1_bias,
126
w2=w2,
109
- w2_scales=a2_scale,
127
+ w2_scales=self.w2_scale,
128
w2_bias=self.w2_bias,
129
topk_weights=topk_weights,
130
topk_ids=topk_ids,
@@ -116,5 +134,22 @@ class XPUExperts(mk.FusedMoEPermuteExpertsUnpermute):
134
ep_rank=self.moe_config.ep_rank,
135
ep_size=self.moe_config.ep_size,
136
output=output,
137
+ is_fp8=self.is_fp8,
138
+ )
139
+
140
+
141
+class XPUExpertsFp8(XPUExperts):
142
+ def __init__(
143
+ self,
144
+ moe_config: FusedMoEConfig,
145
+ quant_config: FusedMoEQuantConfig,
146
+ max_num_tokens: int | None = None,
147
+ num_dispatchers: int | None = None,
148
+ ):
149
+ super().__init__(
150
+ moe_config,
151
+ quant_config,
152
+ max_num_tokens,
153
+ num_dispatchers,
154
)
120
- return
155
+ self.is_fp8 = True