A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 01:00 UTC
[XPU]fake impl for xpu fp8_gemm (#39984)
Signed-off-by: Xinyu Chen <xinyu1.chen@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
Xinyu Chen committed
Apr 18, 2026 at 08:53 UTC
55842a8d6961204f5adbcb5ca07da8cf79d85c33
1 file changed
+17
vllm/_xpu_ops.py
+17
@@ -22,6 +22,23 @@ else:
22
except ImportError:
23
from torch.library import impl_abstract as register_fake
24
25
+if hasattr(torch.ops._xpu_C, "fp8_gemm"):
26
+
27
+ @register_fake("_xpu_C::fp8_gemm")
28
+ def _fp8_gemm_fake(
29
+ q_input: torch.Tensor,
30
+ q_weight: torch.Tensor,
31
+ out_dtype: torch.dtype,
32
+ input_scales: torch.Tensor,
33
+ weight_scale: torch.Tensor,
34
+ bias: torch.Tensor | None = None,
35
+ ) -> torch.Tensor:
36
+ input_2d = q_input.view(-1, q_input.shape[-1])
37
+ M = input_2d.size(0)
38
+ N = q_weight.size(1)
39
+ return torch.empty((M, N), dtype=out_dtype, device=q_input.device)
40
+
41
+
42
if hasattr(torch.ops._xpu_C, "fp8_gemm_w8a16"):
43
44
@register_fake("_xpu_C::fp8_gemm_w8a16")