@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d3859f1891
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Misc][XPU] Upgrade to Pytorch 2.5 for xpu backend (#9823)

Signed-off-by: Kunshang Ji <kunshang.ji@intel.com> Signed-off-by: yan ma <yan.ma@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>

Yan Ma committed Nov 7, 2024 at 09:29 UTC d3859f18915a1e3c50ee88bcbb0af4f4fe754b4e
4 files changed +43 -46
Dockerfile.xpu
+11 -1
@@ -30,9 +30,19 @@ COPY requirements-common.txt /workspace/vllm/requirements-common.txt
30
31 RUN --mount=type=cache,target=/root/.cache/pip \
32 pip install --no-cache-dir \
33 - --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ \
33 -r requirements-xpu.txt
34
35 +RUN git clone https://github.com/intel/pti-gpu && \
36 + cd pti-gpu/sdk && \
37 + git checkout 6c491f07a777ed872c2654ca9942f1d0dde0a082 && \
38 + mkdir build && \
39 + cd build && \
40 + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/icpx_toolchain.cmake -DBUILD_TESTING=OFF .. && \
41 + make -j && \
42 + cmake --install . --config Release --prefix "/usr/local"
43 +
44 +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib/"
45 +
46 COPY . .
47 ARG GIT_REPO_CHECK
48 RUN --mount=type=bind,source=.git,target=.git \
requirements-xpu.txt
+4 -4
@@ -8,9 +8,9 @@ packaging
8 setuptools-scm>=8
9 wheel
10 jinja2
11 -# Following pkgs retrieved from https://pytorch-extension.intel.com/release-whl/stable/xpu/us/
12 -torch == 2.3.1+cxx11.abi
13 -intel-extension-for-pytorch == 2.3.110+xpu
14 -oneccl_bind_pt == 2.3.100+xpu
11 +
12 +torch @ https://intel-extension-for-pytorch.s3.us-east-1.amazonaws.com/ipex_dev/xpu/torch-2.5.0a0%2Bgite84e33f-cp310-cp310-linux_x86_64.whl
13 +intel-extension-for-pytorch @ https://intel-extension-for-pytorch.s3.us-east-1.amazonaws.com/ipex_dev/xpu/intel_extension_for_pytorch-2.5.10%2Bgit9d489a8-cp310-cp310-linux_x86_64.whl
14 +oneccl_bind_pt @ https://intel-extension-for-pytorch.s3.us-east-1.amazonaws.com/ipex_dev/xpu/oneccl_bind_pt-2.5.0%2Bxpu-cp310-cp310-linux_x86_64.whl
15
16 triton-xpu == 3.0.0b1
vllm/_ipex_ops.py
+8 -25
@@ -74,20 +74,12 @@ class ipex_ops:
74 assert kv_cache_dtype == "auto"
75 num_heads = out.size(1)
76 num_queries_per_tokens = num_heads // num_kv_heads
77 - head_mapping = torch.arange(
78 - 0,
79 - num_kv_heads,
80 - device=query.device,
81 - dtype=torch.int32,
82 - ).view(num_kv_heads,
83 - 1).repeat_interleave(num_queries_per_tokens).flatten()
84 - # todo: ipex will refactor namespace
85 - torch.xpu.paged_attention_v1( # type: ignore
77 + ipex.llm.modules.PagedAttention.single_query_kv_attention(
78 out,
79 query.contiguous(),
80 key_cache.view_as(value_cache),
81 value_cache,
90 - head_mapping,
82 + num_queries_per_tokens,
83 scale,
84 block_tables,
85 context_lens,
@@ -124,26 +116,15 @@ class ipex_ops:
116 assert kv_cache_dtype == "auto"
117 num_heads = out.size(1)
118 num_queries_per_tokens = num_heads // num_kv_heads
127 - head_mapping = torch.arange(
128 - 0,
129 - num_kv_heads,
130 - dtype=torch.int32,
131 - device=query.device,
132 - ).view(num_kv_heads,
133 - 1).repeat_interleave(num_queries_per_tokens).flatten()
134 - # todo: ipex will refactor namespace
135 - torch.xpu.paged_attention_v2( # type: ignore
119 + ipex.llm.modules.PagedAttention.single_query_kv_attention(
120 out,
137 - exp_sum,
138 - max_logits,
139 - tmp_out,
121 query.contiguous(),
122 key_cache.view_as(value_cache),
123 value_cache,
143 - head_mapping,
124 + num_queries_per_tokens,
125 + scale,
126 block_tables,
127 context_lens,
146 - scale,
128 block_size,
129 max_context_len,
130 alibi_slopes,
@@ -202,6 +183,7 @@ class ipex_ops:
183 is_causal: bool,
184 return_softmax: bool,
185 gen_: torch.Generator,
186 + logits_soft_cap: float,
187 ) -> None:
188 ipex.llm.functional.varlen_attention(query.contiguous(),
189 key.contiguous(),
@@ -210,7 +192,8 @@ class ipex_ops:
192 max_seqlen_q, max_seqlen_k,
193 pdropout, softmax_scale,
194 zero_tensors, is_causal,
213 - return_softmax, gen_)
195 + return_softmax, gen_,
196 + logits_soft_cap)
197
198 @staticmethod
199 def reshape_and_cache(
vllm/attention/backends/ipex_attn.py
+20 -16
@@ -119,8 +119,6 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
119 if blocksparse_params is not None:
120 raise ValueError(
121 "IPEX backend does not support block-sparse attention.")
122 - if logits_soft_cap is not None:
123 - raise ValueError("IPEX backend does not support logits_soft_cap.")
122 self.num_heads = num_heads
123 self.head_size = head_size
124 self.scale = float(scale)
@@ -135,6 +133,9 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
133 self.num_queries_per_kv = self.num_heads // self.num_kv_heads
134 self.need_mask = (self.alibi_slopes is not None
135 or self.sliding_window is not None)
136 + if logits_soft_cap is None:
137 + logits_soft_cap = 0
138 + self.logits_soft_cap = logits_soft_cap
139
140 supported_head_sizes = PagedAttention.get_supported_head_sizes()
141 if head_size not in supported_head_sizes:
@@ -239,20 +240,23 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
240 (num_tokens, self.num_heads, self.head_size),
241 dtype=query.dtype,
242 device=query.device)
242 - ipex_ops.varlen_attention(query,
243 - key,
244 - value,
245 - output,
246 - attn_metadata.seqlen_q,
247 - attn_metadata.seqlen_q,
248 - attn_metadata.max_seqlen,
249 - attn_metadata.max_seqlen,
250 - pdropout=0.0,
251 - softmax_scale=self.scale,
252 - zero_tensors=False,
253 - is_causal=True,
254 - return_softmax=False,
255 - gen_=None)
243 + ipex_ops.varlen_attention(
244 + query,
245 + key,
246 + value,
247 + output,
248 + attn_metadata.seqlen_q,
249 + attn_metadata.seqlen_q,
250 + attn_metadata.max_seqlen,
251 + attn_metadata.max_seqlen,
252 + pdropout=0.0,
253 + softmax_scale=self.scale,
254 + zero_tensors=False,
255 + is_causal=True,
256 + return_softmax=False,
257 + gen_=None,
258 + logits_soft_cap=self.logits_soft_cap,
259 + )
260 else:
261 # prefix-enabled attention
262 raise RuntimeError(