A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Misc][XPU] Upgrade to Pytorch 2.5 for xpu backend (#9823)
Signed-off-by: Kunshang Ji <kunshang.ji@intel.com> Signed-off-by: yan ma <yan.ma@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
Yan Ma committed
Nov 7, 2024 at 09:29 UTC
d3859f18915a1e3c50ee88bcbb0af4f4fe754b4e
4 files changed
+43
-46
Dockerfile.xpu
+11
-1
@@ -30,9 +30,19 @@ COPY requirements-common.txt /workspace/vllm/requirements-common.txt
30
31
RUN --mount=type=cache,target=/root/.cache/pip \
32
pip install --no-cache-dir \
33
- --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ \
33
-r requirements-xpu.txt
34
35
+RUN git clone https://github.com/intel/pti-gpu && \
36
+ cd pti-gpu/sdk && \
37
+ git checkout 6c491f07a777ed872c2654ca9942f1d0dde0a082 && \
38
+ mkdir build && \
39
+ cd build && \
40
+ cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/icpx_toolchain.cmake -DBUILD_TESTING=OFF .. && \
41
+ make -j && \
42
+ cmake --install . --config Release --prefix "/usr/local"
43
+
44
+ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib/"
45
+
46
COPY . .
47
ARG GIT_REPO_CHECK
48
RUN --mount=type=bind,source=.git,target=.git \
requirements-xpu.txt
+4
-4
@@ -8,9 +8,9 @@ packaging
8
setuptools-scm>=8
9
wheel
10
jinja2
11
-# Following pkgs retrieved from https://pytorch-extension.intel.com/release-whl/stable/xpu/us/
12
-torch == 2.3.1+cxx11.abi
13
-intel-extension-for-pytorch == 2.3.110+xpu
14
-oneccl_bind_pt == 2.3.100+xpu
11
+
12
+torch @ https://intel-extension-for-pytorch.s3.us-east-1.amazonaws.com/ipex_dev/xpu/torch-2.5.0a0%2Bgite84e33f-cp310-cp310-linux_x86_64.whl
13
+intel-extension-for-pytorch @ https://intel-extension-for-pytorch.s3.us-east-1.amazonaws.com/ipex_dev/xpu/intel_extension_for_pytorch-2.5.10%2Bgit9d489a8-cp310-cp310-linux_x86_64.whl
14
+oneccl_bind_pt @ https://intel-extension-for-pytorch.s3.us-east-1.amazonaws.com/ipex_dev/xpu/oneccl_bind_pt-2.5.0%2Bxpu-cp310-cp310-linux_x86_64.whl
15
16
triton-xpu == 3.0.0b1
vllm/_ipex_ops.py
+8
-25
@@ -74,20 +74,12 @@ class ipex_ops:
74
assert kv_cache_dtype == "auto"
75
num_heads = out.size(1)
76
num_queries_per_tokens = num_heads // num_kv_heads
77
- head_mapping = torch.arange(
78
- 0,
79
- num_kv_heads,
80
- device=query.device,
81
- dtype=torch.int32,
82
- ).view(num_kv_heads,
83
- 1).repeat_interleave(num_queries_per_tokens).flatten()
84
- # todo: ipex will refactor namespace
85
- torch.xpu.paged_attention_v1( # type: ignore
77
+ ipex.llm.modules.PagedAttention.single_query_kv_attention(
78
out,
79
query.contiguous(),
80
key_cache.view_as(value_cache),
81
value_cache,
90
- head_mapping,
82
+ num_queries_per_tokens,
83
scale,
84
block_tables,
85
context_lens,
@@ -124,26 +116,15 @@ class ipex_ops:
116
assert kv_cache_dtype == "auto"
117
num_heads = out.size(1)
118
num_queries_per_tokens = num_heads // num_kv_heads
127
- head_mapping = torch.arange(
128
- 0,
129
- num_kv_heads,
130
- dtype=torch.int32,
131
- device=query.device,
132
- ).view(num_kv_heads,
133
- 1).repeat_interleave(num_queries_per_tokens).flatten()
134
- # todo: ipex will refactor namespace
135
- torch.xpu.paged_attention_v2( # type: ignore
119
+ ipex.llm.modules.PagedAttention.single_query_kv_attention(
120
out,
137
- exp_sum,
138
- max_logits,
139
- tmp_out,
121
query.contiguous(),
122
key_cache.view_as(value_cache),
123
value_cache,
143
- head_mapping,
124
+ num_queries_per_tokens,
125
+ scale,
126
block_tables,
127
context_lens,
146
- scale,
128
block_size,
129
max_context_len,
130
alibi_slopes,
@@ -202,6 +183,7 @@ class ipex_ops:
183
is_causal: bool,
184
return_softmax: bool,
185
gen_: torch.Generator,
186
+ logits_soft_cap: float,
187
) -> None:
188
ipex.llm.functional.varlen_attention(query.contiguous(),
189
key.contiguous(),
@@ -210,7 +192,8 @@ class ipex_ops:
192
max_seqlen_q, max_seqlen_k,
193
pdropout, softmax_scale,
194
zero_tensors, is_causal,
213
- return_softmax, gen_)
195
+ return_softmax, gen_,
196
+ logits_soft_cap)
197
198
@staticmethod
199
def reshape_and_cache(
vllm/attention/backends/ipex_attn.py
+20
-16
@@ -119,8 +119,6 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
119
if blocksparse_params is not None:
120
raise ValueError(
121
"IPEX backend does not support block-sparse attention.")
122
- if logits_soft_cap is not None:
123
- raise ValueError("IPEX backend does not support logits_soft_cap.")
122
self.num_heads = num_heads
123
self.head_size = head_size
124
self.scale = float(scale)
@@ -135,6 +133,9 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
133
self.num_queries_per_kv = self.num_heads // self.num_kv_heads
134
self.need_mask = (self.alibi_slopes is not None
135
or self.sliding_window is not None)
136
+ if logits_soft_cap is None:
137
+ logits_soft_cap = 0
138
+ self.logits_soft_cap = logits_soft_cap
139
140
supported_head_sizes = PagedAttention.get_supported_head_sizes()
141
if head_size not in supported_head_sizes:
@@ -239,20 +240,23 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
240
(num_tokens, self.num_heads, self.head_size),
241
dtype=query.dtype,
242
device=query.device)
242
- ipex_ops.varlen_attention(query,
243
- key,
244
- value,
245
- output,
246
- attn_metadata.seqlen_q,
247
- attn_metadata.seqlen_q,
248
- attn_metadata.max_seqlen,
249
- attn_metadata.max_seqlen,
250
- pdropout=0.0,
251
- softmax_scale=self.scale,
252
- zero_tensors=False,
253
- is_causal=True,
254
- return_softmax=False,
255
- gen_=None)
243
+ ipex_ops.varlen_attention(
244
+ query,
245
+ key,
246
+ value,
247
+ output,
248
+ attn_metadata.seqlen_q,
249
+ attn_metadata.seqlen_q,
250
+ attn_metadata.max_seqlen,
251
+ attn_metadata.max_seqlen,
252
+ pdropout=0.0,
253
+ softmax_scale=self.scale,
254
+ zero_tensors=False,
255
+ is_causal=True,
256
+ return_softmax=False,
257
+ gen_=None,
258
+ logits_soft_cap=self.logits_soft_cap,
259
+ )
260
else:
261
# prefix-enabled attention
262
raise RuntimeError(