@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 851725202a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Hardware][intel GPU] bump up ipex version to 2.3 (#8365)

Co-authored-by: Yan Ma <yan.ma@intel.com>

Kunshang Ji committed Sep 14, 2024 at 07:54 UTC 851725202af36dafecd47af802db1d465b25b815
6 files changed +60 -87
Dockerfile.xpu
+10 -2
@@ -1,15 +1,23 @@
1 -FROM intel/oneapi-basekit:2024.1.0-devel-ubuntu20.04
1 +FROM intel/oneapi-basekit:2024.2.1-0-devel-ubuntu22.04
2
3 RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/intel-oneapi-archive-keyring.gpg > /dev/null && \
4 echo "deb [signed-by=/usr/share/keyrings/intel-oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main " | tee /etc/apt/sources.list.d/oneAPI.list && \
5 chmod 644 /usr/share/keyrings/intel-oneapi-archive-keyring.gpg && \
6 - rm /etc/apt/sources.list.d/intel-graphics.list && \
6 wget -O- https://repositories.intel.com/graphics/intel-graphics.key | gpg --dearmor | tee /usr/share/keyrings/intel-graphics.gpg > /dev/null && \
7 echo "deb [arch=amd64,i386 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/graphics/ubuntu jammy arc" | tee /etc/apt/sources.list.d/intel.gpu.jammy.list && \
8 chmod 644 /usr/share/keyrings/intel-graphics.gpg
9
10 RUN apt-get update -y \
11 && apt-get install -y curl libicu70 lsb-release git wget vim numactl python3 python3-pip ffmpeg libsm6 libxext6 libgl1
12 +
13 +RUN git clone https://github.com/intel/pti-gpu && \
14 + cd pti-gpu/sdk && \
15 + mkdir build && \
16 + cd build && \
17 + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/icpx_toolchain.cmake -DBUILD_TESTING=OFF .. && \
18 + make -j && \
19 + cmake --install . --config Release --prefix "/usr/local"
20 +
21 COPY ./ /workspace/vllm
22
23 WORKDIR /workspace/vllm
requirements-xpu.txt
+5 -4
@@ -3,9 +3,10 @@
3
4 setuptools < 70.0.0 # IPEX's torch have some dependency. to be removed.
5
6 -torch @ https://intel-extension-for-pytorch.s3.amazonaws.com/ipex_dev/xpu/torch-2.1.0.post1%2Bcxx11.abi-cp310-cp310-linux_x86_64.whl
7 -intel_extension_for_pytorch @ https://intel-extension-for-pytorch.s3.amazonaws.com/ipex_dev/xpu/intel_extension_for_pytorch-2.1.30a0-cp310-cp310-linux_x86_64.whl
8 -oneccl_bind_pt @ https://intel-extension-for-pytorch.s3.amazonaws.com/ipex_stable/xpu/oneccl_bind_pt-2.1.200%2Bxpu-cp310-cp310-linux_x86_64.whl
6 +torch == 2.3.1+cxx11.abi
7 +intel-extension-for-pytorch == 2.3.110+xpu
8 +oneccl_bind_pt == 2.3.100+xpu
9
10 -triton @ https://github.com/intel/intel-xpu-backend-for-triton/releases/download/v2.1.0/triton-2.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
10 +triton-xpu == 3.0.0b2
11
12 +--extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/
vllm/_ipex_ops.py
+29 -69
@@ -27,29 +27,27 @@ class ipex_ops:
27
28 @staticmethod
29 def silu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
30 - x1, x2 = ipex_ops._reshape_activation_tensor(x)
31 - ipex.llm.functional.silu_mul(x1, x2, out)
30 + ipex.llm.functional.silu_and_mul(x, out)
31
32 @staticmethod
33 def gelu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
35 - x1, x2 = ipex_ops._reshape_activation_tensor(x)
36 - ipex.llm.functional.gelu_mul(x1, x2, out, "none")
34 + ipex.llm.functional.gelu_and_mul(x, out)
35
36 @staticmethod
37 def gelu_tanh_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
40 - x1, x2 = ipex_ops._reshape_activation_tensor(x)
41 - ipex.llm.functional.gelu_mul(x1, x2, out, "tanh")
38 + ipex.llm.functional.gelu_and_mul(x, out)
39
40 @staticmethod
44 - def gelu_fast(out: torch.Tensor, x: torch.Tensor) -> None:
45 - out.copy_(torch.nn.functional.gelu(x))
41 + def gelu_fast(x: torch.Tensor) -> torch.Tensor:
42 + return torch.nn.functional.gelu(x)
43
44 @staticmethod
48 - def gelu_new(out: torch.Tensor, x: torch.Tensor) -> None:
49 - out.copy_(torch.nn.functional.gelu(x))
45 + def gelu_new(x: torch.Tensor) -> torch.Tensor:
46 + return torch.nn.functional.gelu(x)
47
51 - # TODO add implementation of gelu_quick here
52 - # def gelu_quick(out: torch.Tensor, x: torch.Tensor) -> None:
48 + @staticmethod
49 + def gelu_quick(out: torch.Tensor, x: torch.Tensor) -> None:
50 + ipex.llm.functional.gelu_quick(x, out)
51
52 @staticmethod
53 def paged_attention_v1(
@@ -160,29 +158,10 @@ class ipex_ops:
158 cos_sin_cache: torch.Tensor, # [cos_sin_dim, rot_dim]
159 is_neox: bool,
160 ) -> None:
163 - if positions.dim() == 1:
164 - positions = positions.unsqueeze(0)
165 - query = query.unsqueeze(0)
166 - key = key.unsqueeze(0)
167 -
168 - rotary_dim = cos_sin_cache.size(1)
169 - query = query.view(*query.shape[:-1], -1, head_size)
170 - key = key.view(*key.shape[:-1], -1, head_size)
171 -
172 - query_rot = query[..., :rotary_dim]
173 - key_rot = key[..., :rotary_dim]
174 -
175 - cos_sin = cos_sin_cache[positions.long()]
176 - cos, sin = cos_sin.chunk(2, dim=-1)
177 -
178 - if is_neox:
179 - cos = cos.repeat(1, 1, 2).unsqueeze(-2)
180 - sin = sin.repeat(1, 1, 2).unsqueeze(-2)
181 - else:
182 - cos = cos.repeat_interleave(2, dim=-1).unsqueeze(-2)
183 - sin = sin.repeat_interleave(2, dim=-1).unsqueeze(-2)
184 - ipex.llm.functional.rotary_embedding(query_rot, key_rot, sin, cos,
185 - rotary_dim, is_neox, positions)
161 + rot_dim = cos_sin_cache.size(1)
162 + ipex.llm.functional.rotary_embedding_batched(positions, query, key,
163 + head_size, cos_sin_cache,
164 + is_neox, rot_dim)
165
166 @staticmethod
167 def batched_rotary_embedding(positions: torch.Tensor, query: torch.Tensor,
@@ -190,37 +169,15 @@ class ipex_ops:
169 cos_sin_cache: torch.Tensor, is_neox: bool,
170 rot_dim: int,
171 cos_sin_cache_offsets: torch.Tensor) -> None:
193 - if positions.dim() == 1:
194 - positions = positions.unsqueeze(0)
195 - query = query.unsqueeze(0)
196 - key = key.unsqueeze(0)
197 - cos_sin_cache_offsets = cos_sin_cache_offsets.view_as(positions)
198 - rotary_dim = cos_sin_cache.size(1)
199 - query = query.view(*query.shape[:-1], -1, head_size)
200 - key = key.view(*key.shape[:-1], -1, head_size)
201 -
202 - query_rot = query[..., :rotary_dim]
203 - key_rot = key[..., :rotary_dim]
204 -
205 - cos_sin = cos_sin_cache[torch.add(positions,
206 - cos_sin_cache_offsets).long()]
207 - cos, sin = cos_sin.chunk(2, dim=-1)
208 -
209 - if is_neox:
210 - cos = cos.repeat(1, 1, 2).unsqueeze(-2)
211 - sin = sin.repeat(1, 1, 2).unsqueeze(-2)
212 - else:
213 - cos = cos.repeat_interleave(2, dim=-1).unsqueeze(-2)
214 - sin = sin.repeat_interleave(2, dim=-1).unsqueeze(-2)
215 -
216 - ipex.llm.functional.rotary_embedding(query_rot, key_rot, sin, cos,
217 - rotary_dim, is_neox, positions)
172 + ipex.llm.functional.rotary_embedding_batched(positions, query, key,
173 + head_size, cos_sin_cache,
174 + is_neox, rot_dim,
175 + cos_sin_cache_offsets)
176
177 @staticmethod
220 - def rms_norm(out: torch.Tensor, input: torch.Tensor, weight: torch.Tensor,
221 - epsilon: float) -> None:
222 - tmp = ipex.llm.functional.rms_norm(input, weight, epsilon)
223 - out.copy_(tmp)
178 + def rms_norm(input: torch.Tensor, weight: torch.Tensor,
179 + epsilon: float) -> torch.Tensor:
180 + return ipex.llm.functional.rms_norm(input, weight, epsilon)
181
182 @staticmethod
183 def fused_add_rms_norm(input: torch.Tensor, residual: torch.Tensor,
@@ -246,11 +203,14 @@ class ipex_ops:
203 return_softmax: bool,
204 gen_: torch.Generator,
205 ) -> None:
249 - ipex.llm.functional.varlen_attention(query, key, value, out, seqlen_q,
250 - seqlen_k, max_seqlen_q,
251 - max_seqlen_k, pdropout,
252 - softmax_scale, zero_tensors,
253 - is_causal, return_softmax, gen_)
206 + ipex.llm.functional.varlen_attention(query.contiguous(),
207 + key.contiguous(),
208 + value.contiguous(), out,
209 + seqlen_q.int(), seqlen_k.int(),
210 + max_seqlen_q, max_seqlen_k,
211 + pdropout, softmax_scale,
212 + zero_tensors, is_causal,
213 + return_softmax, gen_)
214
215 @staticmethod
216 def reshape_and_cache(
vllm/attention/backends/ipex_attn.py
+6 -2
@@ -49,14 +49,18 @@ class IpexAttnBackend(AttentionBackend):
49 dst_kv_cache: torch.Tensor,
50 src_to_dst: torch.Tensor,
51 ) -> None:
52 - PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
52 + from vllm._ipex_ops import ipex_ops as ops
53 + ops.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
54
55 @staticmethod
56 def copy_blocks(
57 kv_caches: List[torch.Tensor],
58 src_to_dists: torch.Tensor,
59 ) -> None:
59 - PagedAttention.copy_blocks(kv_caches, src_to_dists)
60 + from vllm._ipex_ops import ipex_ops as ops
61 + key_caches = [kv_cache[0] for kv_cache in kv_caches]
62 + value_caches = [kv_cache[1] for kv_cache in kv_caches]
63 + ops.copy_blocks(key_caches, value_caches, src_to_dists)
64
65
66 @dataclass
vllm/model_executor/layers/activation.py
+9 -6
@@ -114,9 +114,7 @@ class NewGELU(CustomOp):
114 def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
115 from vllm._ipex_ops import ipex_ops as ops
116
117 - out = torch.empty_like(x)
118 - ops.gelu_new(out, x)
119 - return out
117 + return ops.gelu_new(x)
118
119
120 class FastGELU(CustomOp):
@@ -136,9 +134,7 @@ class FastGELU(CustomOp):
134 def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
135 from vllm._ipex_ops import ipex_ops as ops
136
139 - out = torch.empty_like(x)
140 - ops.gelu_fast(out, x)
141 - return out
137 + return ops.gelu_fast(x)
138
139
140 class QuickGELU(CustomOp):
@@ -155,6 +151,13 @@ class QuickGELU(CustomOp):
151 ops.gelu_quick(out, x)
152 return out
153
154 + def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
155 + from vllm._ipex_ops import ipex_ops as ops
156 +
157 + out = torch.empty_like(x)
158 + ops.gelu_quick(out, x)
159 + return out
160 +
161 # TODO implement forward_xpu for QuickGELU
162 # def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
163
vllm/model_executor/layers/layernorm.py
+1 -4
@@ -82,14 +82,11 @@ class RMSNorm(CustomOp):
82 self.variance_epsilon,
83 )
84 return x, residual
85 - out = torch.empty_like(x)
86 - ops.rms_norm(
87 - out,
85 + return ops.rms_norm(
86 x,
87 self.weight.data,
88 self.variance_epsilon,
89 )
92 - return out
90
91 def extra_repr(self) -> str:
92 s = f"hidden_size={self.weight.data.size(0)}"