A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 12:15 UTC
[XPU] add scale transpose to prepare_fp8_moe_layer_for_xpu and bump up kernels (#43277)
Signed-off-by: mayuyuace <qiming1.zhang@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
Qiming Zhang committed
May 28, 2026 at 20:22 UTC
96367093722a52c1ccea70fadc9b2b4da30c4016
3 files changed
+17
-4
requirements/xpu.txt
+1
-1
@@ -17,4 +17,4 @@ torchaudio
17
torchvision
18
19
auto_round_lib>=0.13.0
20
-vllm_xpu_kernels @ https://github.com/vllm-project/vllm-xpu-kernels/releases/download/v0.1.8/vllm_xpu_kernels-0.1.8-cp38-abi3-manylinux_2_28_x86_64.whl
20
+vllm_xpu_kernels @ https://github.com/vllm-project/vllm-xpu-kernels/releases/download/v0.1.9/vllm_xpu_kernels-0.1.9-cp38-abi3-manylinux_2_28_x86_64.whl
vllm/model_executor/layers/fused_moe/experts/xpu_moe.py
+13
-2
@@ -29,9 +29,20 @@ if current_platform.is_xpu():
29
30
def prepare_fp8_moe_layer_for_xpu(
31
w13: torch.Tensor,
32
+ w13_scale: torch.Tensor,
33
w2: torch.Tensor,
33
-) -> tuple[torch.Tensor, torch.Tensor]:
34
- return w13.transpose(-1, -2).contiguous(), w2.transpose(-1, -2).contiguous()
34
+ w2_scale: torch.Tensor,
35
+) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
36
+ if w13_scale is not None and w13_scale.ndim == 3:
37
+ w13_scale = w13_scale.transpose(-1, -2).contiguous()
38
+ if w2_scale is not None and w2_scale.ndim == 3:
39
+ w2_scale = w2_scale.transpose(-1, -2).contiguous()
40
+ return (
41
+ w13.transpose(-1, -2).contiguous(),
42
+ w13_scale,
43
+ w2.transpose(-1, -2).contiguous(),
44
+ w2_scale,
45
+ )
46
47
48
class XPUExperts(mk.FusedMoEExpertsModular):
vllm/model_executor/layers/fused_moe/oracle/fp8.py
+3
-1
@@ -483,7 +483,9 @@ def convert_to_fp8_moe_kernel_format(
483
prepare_fp8_moe_layer_for_xpu,
484
)
485
486
- w13, w2 = prepare_fp8_moe_layer_for_xpu(w13, w2)
486
+ w13, w13_scale, w2, w2_scale = prepare_fp8_moe_layer_for_xpu(
487
+ w13, w13_scale, w2, w2_scale
488
+ )
489
elif fp8_backend == Fp8MoeBackend.CPU:
490
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
491
prepare_fp8_moe_layer_for_cpu,