A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 16:30 UTC
[XPU] Fix double-transpose in XPUFP8ScaledMMLinearKernel for W8A8 quant method (#41689)
Signed-off-by: Libin Tang <libin.tang@intel.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Libin Tang committed
May 14, 2026 at 02:17 UTC
9946c38b7fda2eecd349166a8c68ba410e33afd5
2 files changed
+13
-1
.buildkite/intel_jobs/test-intel.yaml
+1
@@ -36,6 +36,7 @@ steps:
36
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN &&
37
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8 &&
38
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8 &&
39
+ python3 examples/basic/offline_inference/generate.py --model nvidia/Llama-3.1-8B-Instruct-FP8 --block-size 64 --enforce-eager --quantization modelopt --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --max-model-len 4096 &&
40
python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192 &&
41
python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 &&
42
python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel'
vllm/model_executor/kernels/linear/scaled_mm/xpu.py
+12
-1
@@ -46,7 +46,18 @@ class XPUFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
46
self.layer_param_names = layer_param_names
47
48
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
49
- replace_parameter(layer, "weight", layer.weight.data.t())
49
+ # fp8_gemm_w8a16 expects weight in [in, out] layout.
50
+ # Transpose if weight is still in [out, in] layout.
51
+ # For square matrices, use contiguity as tie-breaker:
52
+ # checkpoint weights are contiguous, .t() views are not.
53
+ weight = layer.weight
54
+ out_features, in_features = self.config.weight_shape
55
+
56
+ if weight.shape == (out_features, in_features) and (
57
+ in_features != out_features or weight.is_contiguous()
58
+ ):
59
+ replace_parameter(layer, "weight", weight.data.t())
60
+ # else: already in [in, out] layout — no-op
61
62
def apply_weights(
63
self,