A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
[XPU][Bugfix] minor fix for XPU (#15591)
Signed-off-by: yan ma <yan.ma@intel.com>
Yan Ma committed
Apr 22, 2025 at 00:02 UTC
fe3462c774922e3dd1566bcf4bea8c34a6c6b62b
2 files changed
+8
-6
docs/source/getting_started/installation/gpu/xpu.inc.md
+2
@@ -23,6 +23,8 @@ Currently, there are no pre-built XPU wheels.
23
- Second, install Python packages for vLLM XPU backend building:
24
25
```console
26
+git clone https://github.com/vllm-project/vllm.git
27
+cd vllm
28
pip install --upgrade pip
29
pip install -v -r requirements/xpu.txt
30
```
vllm/attention/backends/ipex_attn.py
+6
-6
@@ -220,8 +220,8 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
220
value_cache,
221
attn_metadata.slot_mapping.flatten(),
222
self.kv_cache_dtype,
223
- layer._k_scale,
224
- layer._v_scale,
223
+ layer._k_scale_float,
224
+ layer._v_scale_float,
225
)
226
227
if attn_metadata.is_prompt:
@@ -306,8 +306,8 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
306
max_seq_len,
307
self.alibi_slopes,
308
self.kv_cache_dtype,
309
- layer._k_scale,
310
- layer._v_scale,
309
+ layer._k_scale_float,
310
+ layer._v_scale_float,
311
)
312
else:
313
# Run PagedAttention V2.
@@ -339,8 +339,8 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
339
max_seq_len,
340
self.alibi_slopes,
341
self.kv_cache_dtype,
342
- layer._k_scale,
343
- layer._v_scale,
342
+ layer._k_scale_float,
343
+ layer._v_scale_float,
344
)
345
346
# Reshape the output tensor.