A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 07:15 UTC
Support FP8 KVCache on XPU (#37731)
Signed-off-by: Xinyu Chen <xinyu1.chen@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
Xinyu Chen committed
Apr 12, 2026 at 11:53 UTC
639402f5a24a9176f1049f1f9dd72dd2a19e8e15
4 files changed
+17
-2
.buildkite/intel_jobs/test-intel.yaml
+1
@@ -35,6 +35,7 @@ steps:
35
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend mp &&
36
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN &&
37
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8 &&
38
+ python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8 &&
39
python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192 &&
40
python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 &&
41
python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel'
vllm/_xpu_ops.py
+3
@@ -258,6 +258,9 @@ class xpu_ops:
258
# alibi_slopes = alibi_slopes,
259
# softcap=softcap,
260
return_softmax_lse=return_softmax_lse,
261
+ q_descale=q_descale,
262
+ k_descale=k_descale,
263
+ v_descale=v_descale,
264
)
265
266
@staticmethod
vllm/v1/attention/backends/fa_utils.py
+6
@@ -166,12 +166,18 @@ def is_fa_version_supported(fa_version: int) -> bool:
166
167
168
def flash_attn_supports_fp8() -> bool:
169
+ if current_platform.is_xpu():
170
+ return True
171
return (
172
get_flash_attn_version() == 3
173
and current_platform.is_device_capability_family(90)
174
)
175
176
177
+def flash_attn_supports_quant_query_input() -> bool:
178
+ return not current_platform.is_xpu()
179
+
180
+
181
def flash_attn_supports_sinks() -> bool:
182
if current_platform.is_xpu():
183
return True
vllm/v1/attention/backends/flash_attn.py
+7
-2
@@ -20,6 +20,7 @@ from vllm.v1.attention.backend import (
20
)
21
from vllm.v1.attention.backends.fa_utils import (
22
flash_attn_supports_fp8,
23
+ flash_attn_supports_quant_query_input,
24
get_flash_attn_version,
25
is_fa_version_supported,
26
is_flash_attn_varlen_func_available,
@@ -656,7 +657,7 @@ class FlashAttentionImpl(AttentionImpl):
657
"heads in the layer"
658
)
659
659
- self.supports_quant_query_input = True
660
+ self.supports_quant_query_input = flash_attn_supports_quant_query_input()
661
662
vllm_config = get_current_vllm_config_or_none()
663
dcp_a2a = (
@@ -757,7 +758,11 @@ class FlashAttentionImpl(AttentionImpl):
758
759
descale_shape = (cu_seqlens_q.shape[0] - 1, self.num_kv_heads)
760
760
- q_descale = layer._q_scale.expand(descale_shape)
761
+ q_descale = (
762
+ layer._q_scale.expand(descale_shape)
763
+ if self.supports_quant_query_input
764
+ else None
765
+ )
766
k_descale = layer._k_scale.expand(descale_shape)
767
v_descale = layer._v_scale.expand(descale_shape)
768