A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 07:15 UTC
[XPU][CT] support per-channel quantization in xpu fp8 linear method (#38316)
Signed-off-by: Yan Ma <yan.ma@intel.com>
Yan Ma committed
Apr 12, 2026 at 10:46 UTC
394ff86965a64f0a337d2a8860eae0d2d70607e6
2 files changed
+14
-1
vllm/model_executor/kernels/linear/__init__.py
+1
-1
@@ -204,7 +204,7 @@ _POSSIBLE_WFP8A16_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]
204
# To be added
205
],
206
PlatformEnum.XPU: [
207
- # To be added
207
+ XPUFP8ScaledMMLinearKernel,
208
],
209
}
210
vllm/model_executor/kernels/linear/scaled_mm/xpu.py
+13
@@ -9,6 +9,11 @@ from vllm.model_executor.kernels.linear import ( # noqa: E501
9
FP8ScaledMMLinearKernel,
10
FP8ScaledMMLinearLayerConfig,
11
)
12
+from vllm.model_executor.layers.quantization.utils.quant_utils import (
13
+ kFp8StaticChannelSym,
14
+ kFp8StaticTensorSym,
15
+)
16
+from vllm.model_executor.utils import replace_parameter
17
from vllm.platforms import current_platform
18
19
@@ -23,6 +28,11 @@ class XPUFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
28
29
@classmethod
30
def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]:
31
+ if c.weight_quant_key not in {kFp8StaticChannelSym, kFp8StaticTensorSym}:
32
+ return (
33
+ False,
34
+ "XPUFP8ScaledMM only support per-channel and per-tensor quantization",
35
+ )
36
if c.weight_quant_key.dtype not in {torch.float8_e5m2, torch.float8_e4m3fn}:
37
return False, "XPUFP8ScaledMM only support FP8 weight dtype"
38
return True, None
@@ -35,6 +45,9 @@ class XPUFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
45
self.config = c
46
self.layer_param_names = layer_param_names
47
48
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
49
+ replace_parameter(layer, "weight", layer.weight.data.t())
50
+
51
def apply_weights(
52
self,
53
layer: torch.nn.Module,